Merge pull request #35 from NodeFactoryIo/mpetrunic/custom-length-encoder

Fix length prefix encoding and types

.babelrc

@@ -13,6 +13,7 @@
   "plugins": [
     "@babel/plugin-proposal-object-rest-spread",
     "@babel/plugin-proposal-export-default-from",
-    "@babel/plugin-proposal-async-generator-functions"
+    "@babel/plugin-proposal-async-generator-functions",
+    "@babel/plugin-proposal-class-properties"
   ]
 }

package.json

@@ -41,6 +41,7 @@
     "@babel/preset-typescript": "^7.6.0",
     "@babel/register": "^7.6.2",
     "@babel/runtime": "^7.6.3",
+    "@types/bl": "^2.1.0",
     "@types/chai": "^4.2.4",
     "@types/mocha": "^5.2.7",
     "@typescript-eslint/eslint-plugin": "^2.6.0",
@@ -69,7 +70,7 @@
     "it-buffer": "^0.1.1",
     "it-length-prefixed": "^3.0.0",
     "it-pair": "^1.0.0",
-    "it-pb-rpc": "^0.1.3",
+    "it-pb-rpc": "^0.1.6",
     "it-pipe": "^1.1.0",
     "libp2p-crypto": "^0.17.1",
     "peer-id": "^0.13.5",

src/@types/it-length-prefixed/index.d.ts

@@ -0,0 +1,39 @@
+declare module "it-length-prefixed" {
+  /* eslint-disable @typescript-eslint/interface-name-prefix */
+  import BufferList from "bl";
+  import {Buffer} from "buffer"
+
+  interface LengthDecoderFunction {
+    (data: Buffer | BufferList): number;
+    bytes: number;
+  }
+
+  interface LengthEncoderFunction {
+    (value: Buffer, target: number, offset: number): number|Buffer;
+    bytes: number;
+  }
+
+  interface Encoder {
+    (options?: Partial<{lengthEncoder: LengthEncoderFunction}>): AsyncGenerator<BufferList, Buffer>;
+    single: (chunk: Buffer, options?: Partial<{lengthEncoder: LengthEncoderFunction}>) => BufferList;
+    MIN_POOL_SIZE: number;
+    DEFAULT_POOL_SIZE: number;
+  }
+
+  interface DecoderOptions {
+    lengthDecoder: LengthDecoderFunction;
+    maxLengthLength: number;
+    maxDataLength: number;
+  }
+
+  interface Decoder {
+    (options?: Partial<DecoderOptions>): AsyncGenerator<BufferList, BufferList>;
+    fromReader: (reader: any, options?: Partial<DecoderOptions>) => BufferList;
+    MAX_LENGTH_LENGTH: number;
+    MAX_DATA_LENGTH: number;
+  }
+
+  export const encode: Encoder;
+  export const decode: Decoder;
+
+}

src/@types/it-pb-rpc/index.d.ts

@@ -1,15 +0,0 @@
-declare module "it-pb-rpc" {
-  import { Buffer } from "buffer";
-  import { Duplex } from "it-pair";
-  type WrappedDuplex = {
-    read(bytes?: number): Promise<Buffer>;
-    readLP(): Promise<Buffer>;
-    write(input: Buffer): void;
-    writeLP(input: Buffer): void;
-    unwrap(): Duplex;
-  }
-
-  function Wrap (duplex: any): WrappedDuplex;
-
-  export = Wrap;
-}

src/constants.ts

@@ -0,0 +1,2 @@
+export const NOISE_MSG_MAX_LENGTH_BYTES = 65535;
+

src/encoder.ts

@@ -4,7 +4,8 @@ import {MessageBuffer} from "./@types/handshake";
 
 export const uint16BEEncode = (value, target, offset) => {
   target = target || Buffer.allocUnsafe(2);
-  return target.writeUInt16BE(value, offset);
+  target.writeUInt16BE(value, offset);
+  return target;
 };
 uint16BEEncode.bytes = 2;
 

src/handshake-ik.ts

@@ -54,7 +54,7 @@ export class IKHandshake implements IHandshake {
       logger("IK Stage 0 - Responder receiving message...");
       const receivedMsg = await this.connection.readLP();
       try {
-        const receivedMessageBuffer = decode1(receivedMsg);
+        const receivedMessageBuffer = decode1(receivedMsg.slice());
         const plaintext = this.ik.recvMessage(this.session, receivedMessageBuffer);
         logger("IK Stage 0 - Responder got message, going to verify payload.");
         const decodedPayload = await decodePayload(plaintext);

src/handshake-xx.ts

@@ -57,7 +57,7 @@ export class XXHandshake implements IHandshake {
       logger("Stage 0 - Initiator finished sending first message.");
     } else {
       logger("Stage 0 - Responder waiting to receive first message...");
-      const receivedMessageBuffer = decode0(await this.connection.readLP());
+      const receivedMessageBuffer = decode0((await this.connection.readLP()).slice());
       this.xx.recvMessage(this.session, receivedMessageBuffer);
       logger("Stage 0 - Responder received first message.");
     }
@@ -67,7 +67,7 @@ export class XXHandshake implements IHandshake {
   public async exchange(): Promise<void> {
     if (this.isInitiator) {
       logger('Stage 1 - Initiator waiting to receive first message from responder...');
-      const receivedMessageBuffer = decode1(await this.connection.readLP());
+      const receivedMessageBuffer = decode1((await this.connection.readLP()).slice());
       const plaintext = this.xx.recvMessage(this.session, receivedMessageBuffer);
       logger('Stage 1 - Initiator received the message. Got remote\'s static key.');
 
@@ -97,7 +97,7 @@ export class XXHandshake implements IHandshake {
       logger('Stage 2 - Initiator sent message with signed payload.');
     } else {
       logger('Stage 2 - Responder waiting for third handshake message...');
-      const receivedMessageBuffer = decode1(await this.connection.readLP());
+      const receivedMessageBuffer = decode1((await this.connection.readLP()).slice());
       const plaintext = this.xx.recvMessage(this.session, receivedMessageBuffer);
       logger('Stage 2 - Responder received the message, finished handshake. Got remote\'s static key.');
 

src/noise.ts

@@ -4,7 +4,7 @@ import Wrap from 'it-pb-rpc';
 import DuplexPair from 'it-pair/duplex';
 import ensureBuffer from 'it-buffer';
 import pipe from 'it-pipe';
-import lp from 'it-length-prefixed';
+import {encode, decode} from 'it-length-prefixed';
 
 import {XXHandshake} from "./handshake-xx";
 import {IKHandshake} from "./handshake-ik";
@@ -19,6 +19,7 @@ import {IHandshake} from "./@types/handshake-interface";
 import {KeyCache} from "./keycache";
 import {logger} from "./logger";
 import PeerId from "peer-id";
+import {NOISE_MSG_MAX_LENGTH_BYTES} from "./constants";
 
 export type WrappedConnection = ReturnType<typeof Wrap>;
 
@@ -66,7 +67,14 @@ export class Noise implements INoiseConnection {
    * @returns {Promise<SecureOutbound>}
    */
   public async secureOutbound(localPeer: PeerId, connection: any, remotePeer: PeerId): Promise<SecureOutbound> {
-    const wrappedConnection = Wrap(connection);
+    const wrappedConnection = Wrap(
+      connection,
+      {
+        lengthEncoder: uint16BEEncode,
+        lengthDecoder: uint16BEDecode,
+        maxDataLength: NOISE_MSG_MAX_LENGTH_BYTES
+      }
+    );
     const handshake = await this.performHandshake({
       connection: wrappedConnection,
       isInitiator: true,
@@ -89,7 +97,14 @@ export class Noise implements INoiseConnection {
    * @returns {Promise<SecureOutbound>}
    */
   public async secureInbound(localPeer: PeerId, connection: any, remotePeer?: PeerId): Promise<SecureOutbound> {
-    const wrappedConnection = Wrap(connection);
+    const wrappedConnection = Wrap(
+      connection,
+      {
+        lengthEncoder: uint16BEEncode,
+        lengthDecoder: uint16BEDecode,
+        maxDataLength: NOISE_MSG_MAX_LENGTH_BYTES
+      }
+    );
     const handshake = await this.performHandshake({
       connection: wrappedConnection,
       isInitiator: false,
@@ -213,9 +228,9 @@ export class Noise implements INoiseConnection {
       secure, // write to wrapper
       ensureBuffer, // ensure any type of data is converted to buffer
       encryptStream(handshake), // data is encrypted
-      lp.encode({ lengthEncoder: uint16BEEncode }), // prefix with message length
+      encode({ lengthEncoder: uint16BEEncode }), // prefix with message length
       network, // send to the remote peer
-      lp.decode({ lengthDecoder: uint16BEDecode }), // read message length prefix
+      decode({ lengthDecoder: uint16BEDecode, maxDataLength: NOISE_MSG_MAX_LENGTH_BYTES }), // read message length prefix
       ensureBuffer, // ensure any type of data is converted to buffer
       decryptStream(handshake), // decrypt the incoming data
       secure // pipe to the wrapper

src/utils.ts

@@ -74,7 +74,9 @@ export async function decodePayload(payload: bytes): Promise<INoisePayload> {
   ) as INoisePayload;
 }
 
-export const getHandshakePayload = (publicKey: bytes ) => Buffer.concat([Buffer.from("noise-libp2p-static-key:"), publicKey]);
+export function getHandshakePayload(publicKey: bytes): bytes {
+  return Buffer.concat([Buffer.from("noise-libp2p-static-key:"), publicKey]);
+}
 
 async function isValidPeerId(peerId: bytes, publicKeyProtobuf: bytes) {
   const generatedPeerId = await PeerId.createFromPubKey(publicKeyProtobuf);

test/noise.test.ts

@@ -13,12 +13,13 @@ import {
   getHandshakePayload, getPayload,
   signPayload
 } from "../src/utils";
-import {decode0, decode1, encode1} from "../src/encoder";
+import {decode0, decode1, encode1, uint16BEDecode, uint16BEEncode} from "../src/encoder";
 import {XX} from "../src/handshakes/xx";
 import {Buffer} from "buffer";
 import {getKeyPairFromPeerId} from "./utils";
 import {KeyCache} from "../src/keycache";
-import {XXFallbackHandshake} from "../src/handshake-xx-fallback";
+import {NOISE_MSG_MAX_LENGTH_BYTES} from "../src/constants";
+import BufferList from "bl";
 
 describe("Noise", () => {
   let remotePeer, localPeer;
@@ -60,7 +61,14 @@ describe("Noise", () => {
     const [outbound, { wrapped, handshake }] = await Promise.all([
       noiseInit.secureOutbound(localPeer, outboundConnection, remotePeer),
       (async () => {
-        const wrapped = Wrap(inboundConnection);
+        const wrapped = Wrap(
+          inboundConnection,
+          {
+            lengthEncoder: uint16BEEncode,
+            lengthDecoder: uint16BEDecode,
+            maxDataLength: NOISE_MSG_MAX_LENGTH_BYTES
+          }
+          );
         const prologue = Buffer.alloc(0);
         const staticKeys = generateKeypair();
         const xx = new XX();
@@ -90,7 +98,7 @@ describe("Noise", () => {
 
     try {
       const wrappedOutbound = Wrap(outbound.conn);
-      wrappedOutbound.write(Buffer.from("test"));
+      wrappedOutbound.write(new BufferList([Buffer.from("test")]));
 
       // Check that noise message is prefixed with 16-bit big-endian unsigned integer
       const receivedEncryptedPayload = (await wrapped.read()).slice();

tsconfig.json

@@ -16,6 +16,7 @@
     "**/src/**/*.ts"
   ],
   "exclude": [
-    "node_modules"
+    "node_modules",
+    "./src/@types"
   ]
 }

yarn.lock

@@ -1049,6 +1049,13 @@
   resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz#8da5c6530915653f3a1f38fd5f101d8c3f8079c5"
   integrity sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ==
 
+"@types/bl@^2.1.0":
+  version "2.1.0"
+  resolved "https://registry.yarnpkg.com/@types/bl/-/bl-2.1.0.tgz#45c881c97feae1223d63bbc5b83166153fcb2a15"
+  integrity sha512-1TdA9IXOy4sdqn8vgieQ6GZAiHiPNrOiO1s2GJjuYPw4QVY7gYoVjkW049avj33Ez7IcIvu43hQsMsoUFbCn2g==
+  dependencies:
+    "@types/node" "*"
+
 "@types/chai@^4.2.4":
   version "4.2.4"
   resolved "https://registry.yarnpkg.com/@types/chai/-/chai-4.2.4.tgz#8936cffad3c96ec470a2dc26a38c3ba8b9b6f619"
@@ -1074,6 +1081,11 @@
   resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-5.2.7.tgz#315d570ccb56c53452ff8638738df60726d5b6ea"
   integrity sha512-NYrtPht0wGzhwe9+/idPaBB+TqkY9AhTvOLMkThm0IoEfLaiVQZwBwyJ5puCkO3AUCWrmcoePjp2mbFocKy4SQ==
 
+"@types/node@*":
+  version "13.7.1"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-13.7.1.tgz#238eb34a66431b71d2aaddeaa7db166f25971a0d"
+  integrity sha512-Zq8gcQGmn4txQEJeiXo/KiLpon8TzAl0kmKH4zdWctPj05nWwp1ClMdAVEloqrQKfaC48PNLdgN/aVaLqUrluA==
+
 "@types/node@^10.1.0":
   version "10.17.3"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-10.17.3.tgz#65a8d9a6a0f6af55595a2d0020617959130d6495"
@@ -3600,10 +3612,10 @@ it-pair@^1.0.0:
   dependencies:
     get-iterator "^1.0.2"
 
-it-pb-rpc@^0.1.3:
-  version "0.1.3"
-  resolved "https://registry.yarnpkg.com/it-pb-rpc/-/it-pb-rpc-0.1.3.tgz#3d8e98454cd3fda31a4767b86267b45a7a141aa6"
-  integrity sha512-Zzq7ODzzFSZLsYzQVRqqaOnQENslRt0kY6QQ8ApaciCaX4xXJhtIFK9UrPbWAgXkyTDFJuLH5GgDAph/JN7JQg==
+it-pb-rpc@^0.1.6:
+  version "0.1.6"
+  resolved "https://registry.yarnpkg.com/it-pb-rpc/-/it-pb-rpc-0.1.6.tgz#f6dba92f44a6ce5df8fc71ca08aefe472b552a02"
+  integrity sha512-4srJH8iK976tlAEv0wBnud78hxjQsDvrl71lPG7zquNiaqZ3409oqzukC05f6z0UeYwi4M80Bm+gHy/642lJ/g==
   dependencies:
     it-handshake "^1.0.1"
     it-length-prefixed "^3.0.0"