feat(js-client): Update libp2p ecosystem [fixes DXJ-551] (#393)

* Update libp2p ecosystem

.github/workflows/e2e.yml

@@ -43,7 +43,7 @@ jobs:
     uses: fluencelabs/aqua/.github/workflows/tests.yml@main
     with:
       js-client-snapshots: "${{ needs.js-client.outputs.js-client-snapshots }}"
-      nox-image: "fluencelabs/nox:0.4.2"
+      nox-image: "fluencelabs/nox:unstable"
   flox:
     needs:
       - js-client
@@ -51,4 +51,4 @@ jobs:
     uses: fluencelabs/flox/.github/workflows/tests.yml@main
     with:
       js-client-snapshots: "${{ needs.js-client.outputs.js-client-snapshots }}"
-      nox-image: "fluencelabs/nox:unstable_minimal"
+      nox-image: "fluencelabs/nox:unstable"

packages/core/js-client/package.json

@@ -30,27 +30,28 @@
   "author": "Fluence Labs",
   "license": "Apache-2.0",
   "dependencies": {
-    "@chainsafe/libp2p-noise": "13.0.0",
+    "@chainsafe/libp2p-noise": "14.0.0",
-    "@chainsafe/libp2p-yamux": "5.0.0",
+    "@chainsafe/libp2p-yamux": "6.0.1",
     "@fluencelabs/avm": "0.54.0",
     "@fluencelabs/interfaces": "workspace:*",
     "@fluencelabs/js-client-isomorphic": "workspace:*",
     "@fluencelabs/marine-worker": "0.5.0",
     "@fluencelabs/threads": "^2.0.0",
-    "@libp2p/crypto": "2.0.3",
+    "@libp2p/crypto": "3.0.1",
-    "@libp2p/interface": "0.1.2",
+    "@libp2p/identify": "1.0.4",
-    "@libp2p/peer-id": "3.0.2",
+    "@libp2p/interface": "1.0.1",
-    "@libp2p/peer-id-factory": "3.0.3",
+    "@libp2p/peer-id": "4.0.1",
-    "@libp2p/websockets": "7.0.4",
+    "@libp2p/peer-id-factory": "4.0.0",
+    "@libp2p/ping": "1.0.4",
+    "@libp2p/websockets": "8.0.5",
     "@multiformats/multiaddr": "11.3.0",
     "bs58": "5.0.0",
-    "buffer": "6.0.3",
     "debug": "4.3.4",
-    "it-length-prefixed": "8.0.4",
+    "it-length-prefixed": "9.0.3",
-    "it-map": "2.0.0",
+    "it-map": "3.0.5",
-    "it-pipe": "2.0.5",
+    "it-pipe": "3.0.1",
     "js-base64": "3.7.5",
-    "libp2p": "0.46.6",
+    "libp2p": "1.0.7",
     "multiformats": "11.0.1",
     "rxjs": "7.5.5",
     "uint8arrays": "4.0.3",
@@ -59,7 +60,7 @@
   },
   "devDependencies": {
     "@fluencelabs/aqua-api": "0.13.0",
-    "@rollup/plugin-inject": "5.0.3",
+    "@rollup/plugin-inject": "5.0.5",
     "@types/bs58": "4.0.1",
     "@types/debug": "4.1.7",
     "@types/node": "20.7.0",

packages/core/js-client/src/connection/RelayConnection.ts

@@ -17,9 +17,10 @@
 import { noise } from "@chainsafe/libp2p-noise";
 import { yamux } from "@chainsafe/libp2p-yamux";
 import { PeerIdB58 } from "@fluencelabs/interfaces";
-import type { Stream } from "@libp2p/interface/connection";
+import { identify } from "@libp2p/identify";
-import type { PeerId } from "@libp2p/interface/peer-id";
+import type { PeerId, Stream } from "@libp2p/interface";
 import { peerIdFromString } from "@libp2p/peer-id";
+import { ping } from "@libp2p/ping";
 import { webSockets } from "@libp2p/websockets";
 import { all } from "@libp2p/websockets/filters";
 import { multiaddr, type Multiaddr } from "@multiformats/multiaddr";
@@ -27,8 +28,6 @@ import { decode, encode } from "it-length-prefixed";
 import map from "it-map";
 import { pipe } from "it-pipe";
 import { createLibp2p, Libp2p } from "libp2p";
-import { identifyService } from "libp2p/identify";
-import { pingService } from "libp2p/ping";
 import { Subject } from "rxjs";
 import { fromString } from "uint8arrays/from-string";
 import { toString } from "uint8arrays/to-string";
@@ -139,8 +138,8 @@ export class RelayConnection implements IConnection {
         },
       },
       services: {
-        identify: identifyService(),
+        identify: identify(),
-        ping: pingService(),
+        ping: ping(),
       },
     });
 
@@ -192,7 +191,7 @@ export class RelayConnection implements IConnection {
     log.trace("created stream with id ", stream.id);
     const sink = stream.sink;
 
-    await pipe([fromString(serializeToString(particle))], encode(), sink);
+    await pipe([fromString(serializeToString(particle))], encode, sink);
     log.trace("data written to sink");
   }
 
@@ -260,7 +259,7 @@ export class RelayConnection implements IConnection {
       ({ stream }) => {
         void pipe(
           stream.source,
-          decode(),
+          decode,
           (source) => {
             return map(source, (buf) => {
               return toString(buf.subarray());

packages/core/js-client/src/jsPeer/FluencePeer.ts

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-import { Buffer } from "buffer";
-
 import {
   deserializeAvmResult,
   InterpreterResult,
@@ -165,7 +163,7 @@ export abstract class FluencePeer {
    * @param serviceId - the service id by which the service can be accessed in aqua
    */
   async registerMarineService(
-    wasm: SharedArrayBuffer | Buffer,
+    wasm: ArrayBuffer | SharedArrayBuffer,
     serviceId: string,
   ): Promise<void> {
     if (this.jsServiceHost.hasService(serviceId)) {
@@ -342,10 +340,7 @@ export abstract class FluencePeer {
   private async sendParticleToRelay(
     item: ParticleQueueItem & { result: InterpreterResult },
   ) {
-    const newParticle = cloneWithNewData(
+    const newParticle = cloneWithNewData(item.particle, item.result.data);
-      item.particle,
-      Buffer.from(item.result.data),
-    );
 
     log_particle.debug(
       "id %s. sending particle into network. Next peer ids: %s",
@@ -407,7 +402,10 @@ export abstract class FluencePeer {
           retCode: res.retCode,
         };
 
-        const newParticle = cloneWithNewData(item.particle, Buffer.from([]));
+        const newParticle = cloneWithNewData(
+          item.particle,
+          Uint8Array.from([]),
+        );
 
         this._incomingParticles.next({
           ...item,
@@ -467,7 +465,7 @@ export abstract class FluencePeer {
   private mapParticleGroup(
     group$: GroupedObservable<string, ParticleQueueItem>,
   ) {
-    let prevData: Uint8Array = Buffer.from([]);
+    let prevData = Uint8Array.from([]);
 
     return group$.pipe(
       concatMap(async (item) => {
@@ -515,13 +513,14 @@ export abstract class FluencePeer {
           const res = await this.marineHost.callService("avm", "invoke", args);
 
           avmCallResult = deserializeAvmResult(res);
+          // TODO: This is bug in @fluencelabs/avm package. 'avmCallResult.data' actually number array, not Uint8Array as stated in type.
+          avmCallResult.data = Uint8Array.from(avmCallResult.data);
         } catch (e) {
           avmCallResult = e instanceof Error ? e : new Error(String(e));
         }
 
         if (!(avmCallResult instanceof Error) && avmCallResult.retCode === 0) {
-          const newData = Buffer.from(avmCallResult.data);
+          prevData = avmCallResult.data;
-          prevData = newData;
         }
 
         return {

packages/core/js-client/src/keypair/index.ts

@@ -19,8 +19,7 @@ import {
   generateKeyPair,
   unmarshalPublicKey,
 } from "@libp2p/crypto/keys";
-import type { PrivateKey, PublicKey } from "@libp2p/interface/keys";
+import type { PrivateKey, PublicKey, PeerId } from "@libp2p/interface";
-import type { PeerId } from "@libp2p/interface/peer-id";
 import { createFromPrivKey } from "@libp2p/peer-id-factory";
 import bs58 from "bs58";
 import { toUint8Array } from "js-base64";
@@ -91,11 +90,11 @@ export class KeyPair {
     return this.privateKey.marshal().subarray(0, 32);
   }
 
-  signBytes(data: Uint8Array): Promise<Uint8Array> {
+  async signBytes(data: Uint8Array): Promise<Uint8Array> {
     return this.privateKey.sign(data);
   }
 
-  verify(data: Uint8Array, signature: Uint8Array): Promise<boolean> {
+  async verify(data: Uint8Array, signature: Uint8Array): Promise<boolean> {
     return this.publicKey.verify(data, signature);
   }
 }

packages/core/js-client/src/services/SingleModuleSrv.ts

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-import { Buffer } from "buffer";
-
 import { v4 as uuidv4 } from "uuid";
 
 import { ServiceFnArgs } from "../compilerSupport/types.js";
@@ -53,12 +51,17 @@ export class Srv {
 
     try {
       const newServiceId = uuidv4();
-      const buffer = Buffer.from(wasmContent, "base64");
+
+      const wasmContentBinary = Uint8Array.from(atob(wasmContent), (m) => {
+        // codePointAt cannot return `undefined` value here as callback is called on every symbol
+        return m.codePointAt(0) ?? 0;
+      });
+
       // TODO:: figure out why SharedArrayBuffer is not working here
       // const sab = new SharedArrayBuffer(buffer.length);
       // const tmp = new Uint8Array(sab);
       // tmp.set(buffer, 0);
-      await this.peer.registerMarineService(buffer, newServiceId);
+      await this.peer.registerMarineService(wasmContentBinary, newServiceId);
       this.services.add(newServiceId);
 
       return {

packages/core/js-client/src/services/builtins.ts

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-import { Buffer } from "buffer";
-
 import { JSONValue } from "@fluencelabs/interfaces";
 import bs58 from "bs58";
 import { sha256 } from "multiformats/hashes/sha2";
@@ -332,7 +330,7 @@ export const builtInServices: Record<
     }),
 
     sha256_string: withSchema(z.tuple([z.string()]))(async ([input]) => {
-      const inBuffer = Buffer.from(input);
+      const inBuffer = new TextEncoder().encode(input);
       const multihash = await sha256.digest(inBuffer);
 
       return success(bs58.encode(multihash.bytes));

packages/core/js-client/vite.config.ts

@@ -1,4 +1,4 @@
-/*
+/**
  * Copyright 2023 Fluence Labs Limited
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
@@ -14,11 +14,12 @@
  * limitations under the License.
  */
 
-import inject from "@rollup/plugin-inject";
-import tsconfigPaths from "vite-tsconfig-paths";
 import { createRequire } from "module";
-import { PluginOption, UserConfig } from "vite";
+
+import inject from "@rollup/plugin-inject";
 import { transform } from "esbuild";
+import { PluginOption, UserConfig } from "vite";
+import tsconfigPaths from "vite-tsconfig-paths";
 
 const require = createRequire(import.meta.url);
 const esbuildShim = require.resolve("node-stdlib-browser/helpers/esbuild/shim");
@@ -35,6 +36,7 @@ function minifyEs(): PluginOption {
         ) {
           return await transform(code, { minify: true });
         }
+
         return code;
       },
     },
@@ -48,21 +50,19 @@ const config: UserConfig = {
     lib: {
       entry: "./src/index.ts",
       name: "js-client",
-      fileName: () => "index.min.js",
+      fileName: () => {
+        return "index.min.js";
+      },
       formats: ["es"],
     },
     outDir: "./dist/browser",
     rollupOptions: {
       plugins: [
-        {
+        inject({
-          // @ts-ignore
+          global: [esbuildShim, "global"],
-          ...inject({
+          process: [esbuildShim, "process"],
-            global: [esbuildShim, "global"],
+          Buffer: [esbuildShim, "Buffer"],
-            process: [esbuildShim, "process"],
+        }),
-            Buffer: [esbuildShim, "Buffer"],
-          }),
-          enforce: "post",
-        },
       ],
     },
   },