fluencelabs/js-libp2p
1
0
Fork 0
mirror of https://github.com/fluencelabs/js-libp2p synced 2025-04-25 10:32:14 +00:00
Code Issues Projects Releases Wiki Activity

deps: update it-length-prefix, uint8arraylist etc (#1317)

Browse Source 
In order to support no-copy operations in streams, update all deps
to support streaming Uint8ArrayLists.
This commit is contained in:
Alex Potsides 2022-08-03 14:15:35 +01:00 committed by GitHub
parent 05e8e7ead9
commit 564f4b8aa7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 69 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
examples/auto-relay/test.js
View File

@ -1,5 +1,5 @@
import path from 'path'
import execa from 'execa'
import { execa } from 'execa'
import pDefer from 'p-defer'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { fileURLToPath } from 'url'

2
examples/chat/src/stream.js
View File

@ -28,7 +28,7 @@ export function streamToConsole(stream) {
// Decode length-prefixed data
lp.decode(),
// Turn buffers into strings
(source) => map(source, (buf) => uint8ArrayToString(buf)),
(source) => map(source, (buf) => uint8ArrayToString(buf.subarray())),
// Sink function
async function (source) {
// For each chunk of data

2
examples/chat/test.js
View File

@ -1,5 +1,5 @@
import path from 'path'
import execa from 'execa'
import { execa } from 'execa'
import pDefer from 'p-defer'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { fileURLToPath } from 'url'

2
examples/discovery-mechanisms/test-2.js
View File

@ -1,5 +1,5 @@
import path from 'path'
import execa from 'execa'
import { execa } from 'execa'
import pWaitFor from 'p-wait-for'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { fileURLToPath } from 'url'

2
examples/discovery-mechanisms/test-3.js
View File

@ -1,5 +1,5 @@
import path from 'path'
import execa from 'execa'
import { execa } from 'execa'
import pWaitFor from 'p-wait-for'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { fileURLToPath } from 'url'

2
examples/echo/test.js
View File

@ -1,5 +1,5 @@
import path from 'path'
import execa from 'execa'
import { execa } from 'execa'
import pDefer from 'p-defer'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { fileURLToPath } from 'url'

8
examples/libp2p-in-the-browser/README.md
View File

@ -1,6 +1,6 @@
# libp2p in the browser
This example leverages the [Parcel.js bundler](https://parceljs.org/) to compile and serve the libp2p code in the browser. Parcel uses [Babel](https://babeljs.io/) to handle transpilation of the code. You can use other bundlers such as Webpack or Browserify, but we will not be covering them here.
This example leverages the [vite bundler](https://vitejs.dev/) to compile and serve the libp2p code in the browser. You can use other bundlers such as Webpack, but we will not be covering them here.
## Setup
@ -18,7 +18,7 @@ npm install
## Running the examples
Start by running the Parcel server:
Start by running the vite server:
```
npm start
@ -30,7 +30,7 @@ The output should look something like this:
$ npm start
> libp2p-in-browser@1.0.0 start
> parcel index.html
> vite index.html
Server running at http://localhost:1234
✨ Built in 1000ms.
@ -40,7 +40,7 @@ This will compile the code and start a server listening on port [http://localhos
Now, if you open a second browser tab to `http://localhost:1234`, you should discover your node from the previous tab. This is due to the fact that the `libp2p-webrtc-star` transport also acts as a Peer Discovery interface. Your node will be notified of any peer that connects to the same signaling server you are connected to. Once libp2p discovers this new peer, it will attempt to establish a direct WebRTC connection.
**Note**: In the example we assign libp2p to `window.libp2p`, in case you would like to play around with the API directly in the browser. You can of course make changes to `index.js` and Parcel will automatically rebuild and reload the browser tabs.
**Note**: In the example we assign libp2p to `window.libp2p`, in case you would like to play around with the API directly in the browser. You can of course make changes to `index.js` and vite will automatically rebuild and reload the browser tabs.
## Going to production?

2
examples/libp2p-in-the-browser/index.html
View File

@ -4,7 +4,7 @@
<head>
<meta charset="utf-8">
<title>js-libp2p parcel.js browser example</title>
<title>js-libp2p vite browser example</title>
</head>
<body>

2
examples/libp2p-in-the-browser/test.js
View File

@ -1,4 +1,4 @@
import execa from 'execa'
import { execa } from 'execa'
import { chromium } from 'playwright'
import path from 'path'
import { fileURLToPath } from 'url'

10
examples/package.json
View File

@ -9,13 +9,13 @@
},
"license": "MIT",
"dependencies": {
"@libp2p/pubsub-peer-discovery": "^6.0.0",
"@libp2p/floodsub": "^3.0.0",
"@libp2p/pubsub-peer-discovery": "^6.0.1",
"@libp2p/floodsub": "^3.0.3",
"@nodeutils/defaults-deep": "^1.1.0",
"execa": "^2.1.0",
"fs-extra": "^8.1.0",
"execa": "^6.1.0",
"fs-extra": "^10.1.0",
"libp2p": "../",
"p-defer": "^3.0.0",
"p-defer": "^4.0.0",
"uint8arrays": "^3.0.0",
"which": "^2.0.1"
},

2
examples/pubsub/message-filtering/test.js
View File

@ -1,5 +1,5 @@
import path from 'path'
import execa from 'execa'
import { execa } from 'execa'
import pDefer from 'p-defer'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { fileURLToPath } from 'url'

2
examples/pubsub/test-1.js
View File

@ -1,5 +1,5 @@
import path from 'path'
import execa from 'execa'
import { execa } from 'execa'
import pDefer from 'p-defer'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { fileURLToPath } from 'url'

6
examples/test.js
View File

@ -3,7 +3,7 @@ process.env.CI = true // needed for some "clever" build tools
import fs from 'fs-extra'
import path from 'path'
import execa from 'execa'
import { execa } from 'execa'
import { fileURLToPath } from 'url'
const __dirname = path.dirname(fileURLToPath(import.meta.url))
@ -36,7 +36,8 @@ async function installDeps (dir) {
return
}
const proc = execa.command('npm install', {
const proc = execa('npm', ['install'], {
all: true,
cwd: dir
})
proc.all.on('data', (data) => {
@ -71,6 +72,7 @@ async function build (dir) {
}
const proc = execa('npm', ['run', build], {
all: true,
cwd: dir
})
proc.all.on('data', (data) => {

7
examples/utils.js
View File

@ -1,4 +1,4 @@
import execa from 'execa'
import { execa } from 'execa'
import fs from 'fs-extra'
import which from 'which'
@ -26,7 +26,10 @@ export async function waitForOutput (expectedOutput, command, args = [], opts =
command = 'node'
}
const proc = execa(command, args, opts)
const proc = execa(command, args, {
...opts,
all: true
})
let output = ''
let time = 600000

2
examples/webrtc-direct/index.html
View File

@ -2,7 +2,7 @@
<html lang="en">
<head>
<meta charset="UTF-8">
<title>js-libp2p parcel.js browser example</title>
<title>js-libp2p vite browser example</title>
</head>
<body>

4
examples/webrtc-direct/package.json
View File

@ -10,9 +10,9 @@
"license": "ISC",
"dependencies": {
"@libp2p/webrtc-direct": "^2.0.0",
"@chainsafe/libp2p-noise": "^7.0.1",
"@chainsafe/libp2p-noise": "^7.0.3",
"@libp2p/bootstrap": "^2.0.0",
"@libp2p/mplex": "^4.0.2",
"@libp2p/mplex": "^4.0.3",
"libp2p": "../../",
"wrtc": "^0.4.7"
},

2
examples/webrtc-direct/test.js
View File

@ -1,5 +1,5 @@
import path from 'path'
import execa from 'execa'
import { execa } from 'execa'
import pDefer from 'p-defer'
import { toString as uint8ArrayToString } from 'uint8arrays/to-string'
import { chromium } from 'playwright'

16
package.json
View File

@ -87,7 +87,6 @@
"generate:proto:fetch": "protons ./src/fetch/pb/proto.proto",
"generate:proto:identify": "protons ./src/identify/pb/message.proto",
"generate:proto:plaintext": "protons ./src/insecure/pb/proto.proto",
"generate:proto:tags": "protons ./src/connection-manager/tags/tags.proto",
"test": "aegir test",
"test:node": "aegir test -t node -f \"./dist/test/**/*.{node,spec}.js\" --cov",
"test:chrome": "aegir test -t browser -f \"./dist/test/**/*.spec.js\" --cov",
@ -113,7 +112,7 @@
"@libp2p/interface-peer-info": "^1.0.1",
"@libp2p/interface-peer-routing": "^1.0.0",
"@libp2p/interface-peer-store": "^1.2.0",
"@libp2p/interface-pubsub": "^1.0.3",
"@libp2p/interface-pubsub": "^2.0.0",
"@libp2p/interface-registrar": "^2.0.0",
"@libp2p/interface-stream-muxer": "^2.0.1",
"@libp2p/interface-transport": "^1.0.0",
@ -123,7 +122,7 @@
"@libp2p/peer-collections": "^2.0.0",
"@libp2p/peer-id": "^1.1.10",
"@libp2p/peer-id-factory": "^1.0.9",
"@libp2p/peer-record": "^3.0.0",
"@libp2p/peer-record": "^4.0.0",
"@libp2p/peer-store": "^3.0.0",
"@libp2p/tracked-map": "^2.0.1",
"@libp2p/utils": "^3.0.0",
@ -142,7 +141,7 @@
"it-first": "^1.0.6",
"it-foreach": "^0.1.1",
"it-handshake": "^4.0.0",
"it-length-prefixed": "^7.0.1",
"it-length-prefixed": "^8.0.2",
"it-map": "^1.0.6",
"it-merge": "^1.0.3",
"it-pair": "^2.0.2",
@ -157,17 +156,18 @@
"p-retry": "^5.0.0",
"p-settle": "^5.0.0",
"private-ip": "^2.3.3",
"protons-runtime": "^1.0.4",
"protons-runtime": "^2.0.2",
"retimer": "^3.0.0",
"sanitize-filename": "^1.6.3",
"set-delayed-interval": "^1.0.0",
"timeout-abort-controller": "^3.0.0",
"uint8arraylist": "^2.0.0",
"uint8arrays": "^3.0.0",
"wherearewe": "^1.0.0",
"xsalsa20": "^1.1.0"
},
"devDependencies": {
"@chainsafe/libp2p-noise": "^7.0.1",
"@chainsafe/libp2p-noise": "^7.0.2",
"@libp2p/bootstrap": "^2.0.0",
"@libp2p/daemon-client": "^2.0.0",
"@libp2p/daemon-server": "^2.0.0",
@ -204,8 +204,8 @@
"p-defer": "^4.0.0",
"p-event": "^5.0.1",
"p-times": "^4.0.0",
"p-wait-for": "^4.1.0",
"protons": "^3.0.4",
"p-wait-for": "^5.0.0",
"protons": "^4.0.1",
"rimraf": "^3.0.2",
"sinon": "^14.0.0",
"ts-sinon": "^2.0.2"

5
src/circuit/circuit/stream-handler.ts
View File

@ -4,6 +4,7 @@ import { Handshake, handshake } from 'it-handshake'
import { CircuitRelay } from '../pb/index.js'
import type { Stream } from '@libp2p/interface-connection'
import type { Source } from 'it-stream-types'
import type { Uint8ArrayList } from 'uint8arraylist'
const log = logger('libp2p:circuit:stream-handler')
@ -22,7 +23,7 @@ export interface StreamHandlerOptions {
export class StreamHandler {
private readonly stream: Stream
private readonly shake: Handshake
private readonly decoder: Source<Uint8Array>
private readonly decoder: Source<Uint8ArrayList>
constructor (options: StreamHandlerOptions) {
const { stream, maxLength = 4096 } = options
@ -40,7 +41,7 @@ export class StreamHandler {
const msg = await this.decoder.next()
if (msg.value != null) {
const value = CircuitRelay.decode(msg.value.slice())
const value = CircuitRelay.decode(msg.value)
log('read message type', value.type)
return value
}

9
src/circuit/pb/index.ts
View File

@ -3,6 +3,7 @@
import { enumeration, encodeMessage, decodeMessage, message, bytes } from 'protons-runtime'
import type { Codec } from 'protons-runtime'
import type { Uint8ArrayList } from 'uint8arraylist'
export interface CircuitRelay {
type?: CircuitRelay.Type
@ -89,11 +90,11 @@ export namespace CircuitRelay {
})
}
export const encode = (obj: Peer): Uint8Array => {
export const encode = (obj: Peer): Uint8ArrayList => {
return encodeMessage(obj, Peer.codec())
}
export const decode = (buf: Uint8Array): Peer => {
export const decode = (buf: Uint8Array | Uint8ArrayList): Peer => {
return decodeMessage(buf, Peer.codec())
}
}
@ -107,11 +108,11 @@ export namespace CircuitRelay {
})
}
export const encode = (obj: CircuitRelay): Uint8Array => {
export const encode = (obj: CircuitRelay): Uint8ArrayList => {
return encodeMessage(obj, CircuitRelay.codec())
}
export const decode = (buf: Uint8Array): CircuitRelay => {
export const decode = (buf: Uint8Array | Uint8ArrayList): CircuitRelay => {
return decodeMessage(buf, CircuitRelay.codec())
}
}

9
src/fetch/pb/proto.ts
View File

@ -3,6 +3,7 @@
import { encodeMessage, decodeMessage, message, string, enumeration, bytes } from 'protons-runtime'
import type { Codec } from 'protons-runtime'
import type { Uint8ArrayList } from 'uint8arraylist'
export interface FetchRequest {
identifier: string
@ -15,11 +16,11 @@ export namespace FetchRequest {
})
}
export const encode = (obj: FetchRequest): Uint8Array => {
export const encode = (obj: FetchRequest): Uint8ArrayList => {
return encodeMessage(obj, FetchRequest.codec())
}
export const decode = (buf: Uint8Array): FetchRequest => {
export const decode = (buf: Uint8Array | Uint8ArrayList): FetchRequest => {
return decodeMessage(buf, FetchRequest.codec())
}
}
@ -55,11 +56,11 @@ export namespace FetchResponse {
})
}
export const encode = (obj: FetchResponse): Uint8Array => {
export const encode = (obj: FetchResponse): Uint8ArrayList => {
return encodeMessage(obj, FetchResponse.codec())
}
export const decode = (buf: Uint8Array): FetchResponse => {
export const decode = (buf: Uint8Array | Uint8ArrayList): FetchResponse => {
return decodeMessage(buf, FetchResponse.codec())
}
}

2
src/identify/index.ts
View File

@ -404,7 +404,7 @@ export class IdentifyService implements Startable {
const envelope = await RecordEnvelope.seal(peerRecord, this.components.getPeerId())
await this.components.getPeerStore().addressBook.consumePeerRecord(envelope)
signedPeerRecord = envelope.marshal()
signedPeerRecord = envelope.marshal().subarray()
}
const message = Identify.encode({

5
src/identify/pb/message.ts
View File

@ -3,6 +3,7 @@
import { encodeMessage, decodeMessage, message, string, bytes } from 'protons-runtime'
import type { Codec } from 'protons-runtime'
import type { Uint8ArrayList } from 'uint8arraylist'
export interface Identify {
protocolVersion?: string
@ -27,11 +28,11 @@ export namespace Identify {
})
}
export const encode = (obj: Identify): Uint8Array => {
export const encode = (obj: Identify): Uint8ArrayList => {
return encodeMessage(obj, Identify.codec())
}
export const decode = (buf: Uint8Array): Identify => {
export const decode = (buf: Uint8Array | Uint8ArrayList): Identify => {
return decodeMessage(buf, Identify.codec())
}
}

2
src/insecure/index.ts
View File

@ -39,7 +39,7 @@ async function encrypt (localId: PeerId, conn: Duplex<Uint8Array>, remoteId?: Pe
Type: type,
Data: localId.publicKey ?? new Uint8Array(0)
}
}).slice()
}).subarray()
)
log('write pubkey exchange to peer %p', remoteId)

9
src/insecure/pb/proto.ts
View File

@ -3,6 +3,7 @@
import { encodeMessage, decodeMessage, message, bytes, enumeration } from 'protons-runtime'
import type { Codec } from 'protons-runtime'
import type { Uint8ArrayList } from 'uint8arraylist'
export interface Exchange {
id?: Uint8Array
@ -17,11 +18,11 @@ export namespace Exchange {
})
}
export const encode = (obj: Exchange): Uint8Array => {
export const encode = (obj: Exchange): Uint8ArrayList => {
return encodeMessage(obj, Exchange.codec())
}
export const decode = (buf: Uint8Array): Exchange => {
export const decode = (buf: Uint8Array | Uint8ArrayList): Exchange => {
return decodeMessage(buf, Exchange.codec())
}
}
@ -58,11 +59,11 @@ export namespace PublicKey {
})
}
export const encode = (obj: PublicKey): Uint8Array => {
export const encode = (obj: PublicKey): Uint8ArrayList => {
return encodeMessage(obj, PublicKey.codec())
}
export const decode = (buf: Uint8Array): PublicKey => {
export const decode = (buf: Uint8Array | Uint8ArrayList): PublicKey => {
return decodeMessage(buf, PublicKey.codec())
}
}

9
test/configuration/utils.ts
View File

@ -10,6 +10,7 @@ import type { Libp2pInit, Libp2pOptions } from '../../src/index.js'
import type { PeerId } from '@libp2p/interface-peer-id'
import * as cborg from 'cborg'
import { peerIdFromString } from '@libp2p/peer-id'
import { Uint8ArrayList } from 'uint8arraylist'
const relayAddr = MULTIADDRS_WEBSOCKETS[0]
@ -32,16 +33,16 @@ class MockPubSub extends PubSubBaseProtocol {
return cborg.decode(bytes)
}
encodeRpc (rpc: PubSubRPC): Uint8Array {
return cborg.encode(rpc)
encodeRpc (rpc: PubSubRPC): Uint8ArrayList {
return new Uint8ArrayList(cborg.encode(rpc))
}
decodeMessage (bytes: Uint8Array): PubSubRPCMessage {
return cborg.decode(bytes)
}
encodeMessage (rpc: PubSubRPCMessage): Uint8Array {
return cborg.encode(rpc)
encodeMessage (rpc: PubSubRPCMessage): Uint8ArrayList {
return new Uint8ArrayList(cborg.encode(rpc))
}
async publishMessage (from: PeerId, message: Message): Promise<PublishResult> {

2
test/relay/auto-relay.node.ts
View File

@ -16,7 +16,7 @@ import type Sinon from 'sinon'
import { createRelayOptions, createNodeOptions } from './utils.js'
import { protocols } from '@multiformats/multiaddr'
async function usingAsRelay (node: Libp2pNode, relay: Libp2pNode, opts?: PWaitForOptions) {
async function usingAsRelay (node: Libp2pNode, relay: Libp2pNode, opts?: PWaitForOptions<boolean>) {
// Wait for peer to be used as a relay
await pWaitFor(() => {
for (const addr of node.getMultiaddrs()) {