From 2f598eba09cff4301474af08196158065e3602d8 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Thu, 25 Nov 2021 16:32:19 +0000 Subject: [PATCH] feat: update dht (#1009) Changes dht creation to use factory function and updates docs BREAKING CHANGE: libp2p-kad-dht has a new event-based API which is exposed as `_dht` --- .github/workflows/main.yml | 3 +- doc/CONFIGURATION.md | 74 ++++++++++++------------- examples/delegated-routing/package.json | 2 +- examples/transports/README.md | 2 +- package.json | 5 +- src/content-routing/index.js | 44 ++++++++++++--- src/dht/dht-content-routing.js | 44 +++++++++++++++ src/dht/dht-peer-routing.js | 51 +++++++++++++++++ src/errors.js | 4 +- src/index.js | 11 +--- src/nat-manager.js | 2 +- src/peer-routing.js | 3 +- test/ts-use/package.json | 2 +- 13 files changed, 181 insertions(+), 66 deletions(-) create mode 100644 src/dht/dht-content-routing.js create mode 100644 src/dht/dht-peer-routing.js diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 69cd85eb..8c401645 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -17,7 +17,6 @@ jobs: node-version: 14 - run: npm install - run: npx aegir lint - - uses: gozala/typescript-error-reporter-action@v1.0.8 - run: npx aegir build - run: npx aegir dep-check - uses: ipfs/aegir/actions/bundle-size@v32.1.0 @@ -67,4 +66,4 @@ jobs: steps: - uses: actions/checkout@v2 - run: npm install - - run: cd node_modules/interop-libp2p && yarn && LIBP2P_JS=${GITHUB_WORKSPACE}/src/index.js npx aegir test -t node --bail -- --exit + - run: npm run test:interop -- --bail -- --exit diff --git a/doc/CONFIGURATION.md b/doc/CONFIGURATION.md index b9b24fd0..7d4523f7 100644 --- a/doc/CONFIGURATION.md +++ b/doc/CONFIGURATION.md @@ -1,37 +1,37 @@ -# +# -- [Configuration](#configuration) - - [Overview](#overview) - - [Modules](#modules) - - [Transport](#transport) - - [Stream Multiplexing](#stream-multiplexing) - - [Connection Encryption](#connection-encryption) - - [Peer Discovery](#peer-discovery) - - [Content Routing](#content-routing) - - [Peer Routing](#peer-routing) - - [DHT](#dht) - - [Pubsub](#pubsub) - - [Customizing libp2p](#customizing-libp2p) - - [Examples](#examples) - - [Basic setup](#basic-setup) - - [Customizing Peer Discovery](#customizing-peer-discovery) - - [Setup webrtc transport and discovery](#setup-webrtc-transport-and-discovery) - - [Customizing Pubsub](#customizing-pubsub) - - [Customizing DHT](#customizing-dht) - - [Setup with Content and Peer Routing](#setup-with-content-and-peer-routing) - - [Setup with Relay](#setup-with-relay) - - [Setup with Auto Relay](#setup-with-auto-relay) - - [Setup with Keychain](#setup-with-keychain) - - [Configuring Dialing](#configuring-dialing) - - [Configuring Connection Manager](#configuring-connection-manager) - - [Configuring Transport Manager](#configuring-transport-manager) - - [Configuring Metrics](#configuring-metrics) - - [Configuring PeerStore](#configuring-peerstore) - - [Customizing Transports](#customizing-transports) - - [Configuring the NAT Manager](#configuring-the-nat-manager) - - [Browser support](#browser-support) - - [UPnP and NAT-PMP](#upnp-and-nat-pmp) - - [Configuration examples](#configuration-examples) +- [Overview](#overview) +- [Modules](#modules) + - [Transport](#transport) + - [Stream Multiplexing](#stream-multiplexing) + - [Connection Encryption](#connection-encryption) + - [Peer Discovery](#peer-discovery) + - [Content Routing](#content-routing) + - [Peer Routing](#peer-routing) + - [DHT](#dht) + - [Pubsub](#pubsub) +- [Customizing libp2p](#customizing-libp2p) + - [Examples](#examples) + - [Basic setup](#basic-setup) + - [Customizing Peer Discovery](#customizing-peer-discovery) + - [Setup webrtc transport and discovery](#setup-webrtc-transport-and-discovery) + - [Customizing Pubsub](#customizing-pubsub) + - [Customizing DHT](#customizing-dht) + - [Setup with Content and Peer Routing](#setup-with-content-and-peer-routing) + - [Setup with Relay](#setup-with-relay) + - [Setup with Auto Relay](#setup-with-auto-relay) + - [Setup with Keychain](#setup-with-keychain) + - [Configuring Dialing](#configuring-dialing) + - [Configuring Connection Manager](#configuring-connection-manager) + - [Configuring Transport Manager](#configuring-transport-manager) + - [Configuring Metrics](#configuring-metrics) + - [Configuring PeerStore](#configuring-peerstore) + - [Customizing Transports](#customizing-transports) + - [Configuring the NAT Manager](#configuring-the-nat-manager) + - [Browser support](#browser-support) + - [UPnP and NAT-PMP](#upnp-and-nat-pmp) + - [Configuring protocol name](#configuring-protocol-name) +- [Configuration examples](#configuration-examples) ## Overview @@ -374,11 +374,7 @@ const node = await Libp2p.create({ dht: { // The DHT options (and defaults) can be found in its documentation kBucketSize: 20, enabled: true, // This flag is required for DHT to run (disabled by default) - randomWalk: { - enabled: true, // Allows to disable discovery (enabled by default) - interval: 300e3, - timeout: 10e3 - } + clientMode: false // Whether to run the WAN DHT in client or server mode (default: client mode) } } }) @@ -788,7 +784,7 @@ By default under nodejs libp2p will attempt to use [UPnP](https://en.wikipedia.o #### Configuring protocol name -Changing the protocol name prefix can isolate default public network (IPFS) for custom purposes. +Changing the protocol name prefix can isolate default public network (IPFS) for custom purposes. ```js const node = await Libp2p.create({ diff --git a/examples/delegated-routing/package.json b/examples/delegated-routing/package.json index 164e0d3c..76a06b46 100644 --- a/examples/delegated-routing/package.json +++ b/examples/delegated-routing/package.json @@ -7,7 +7,7 @@ "libp2p": "github:libp2p/js-libp2p#master", "libp2p-delegated-content-routing": "~0.2.2", "libp2p-delegated-peer-routing": "~0.2.2", - "libp2p-kad-dht": "~0.14.12", + "libp2p-kad-dht": "^0.26.5", "libp2p-mplex": "~0.8.5", "libp2p-secio": "~0.11.1", "libp2p-webrtc-star": "~0.15.8", diff --git a/examples/transports/README.md b/examples/transports/README.md index 8c9d23b9..18f5975b 100644 --- a/examples/transports/README.md +++ b/examples/transports/README.md @@ -91,7 +91,7 @@ const concat = require('it-concat') const MPLEX = require('libp2p-mplex') ``` -We are going to reuse the `createNode` function from step 1, but this time add a stream multiplexer from `libp2p-mplex`. +We are going to reuse the `createNode` function from step 1, but this time add a stream multiplexer from `libp2p-mplex`. ```js const createNode = async () => { const node = await Libp2p.create({ diff --git a/package.json b/package.json index 919d16a8..7febe185 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ "test:node": "aegir test -t node -f \"./test/**/*.{node,spec}.js\"", "test:browser": "aegir test -t browser", "test:examples": "cd examples && npm run test:all", + "test:interop": "LIBP2P_JS=$PWD npx aegir test -t node -f ./node_modules/libp2p-interop/test/*", "prepare": "aegir build --no-bundle", "release": "aegir release -t node -t browser", "release-minor": "aegir release --type minor -t node -t browser", @@ -142,7 +143,6 @@ "buffer": "^6.0.3", "datastore-core": "^6.0.7", "delay": "^5.0.0", - "interop-libp2p": "^0.4.0", "into-stream": "^7.0.0", "ipfs-http-client": "^52.0.2", "it-concat": "^2.0.0", @@ -155,7 +155,8 @@ "libp2p-floodsub": "^0.27.0", "libp2p-gossipsub": "^0.11.0", "libp2p-interfaces-compliance-tests": "^1.0.0", - "libp2p-kad-dht": "^0.24.2", + "libp2p-interop": "^0.5.0", + "libp2p-kad-dht": "^0.26.5", "libp2p-mdns": "^0.17.0", "libp2p-mplex": "^0.10.1", "libp2p-tcp": "^0.17.0", diff --git a/src/content-routing/index.js b/src/content-routing/index.js index 7fc4b4fb..924b9873 100644 --- a/src/content-routing/index.js +++ b/src/content-routing/index.js @@ -8,9 +8,10 @@ const { requirePeers, maybeLimitSource } = require('./utils') - +const drain = require('it-drain') const merge = require('it-merge') const { pipe } = require('it-pipe') +const { DHTContentRouting } = require('../dht/dht-content-routing') /** * @typedef {import('peer-id')} PeerId @@ -38,7 +39,7 @@ class ContentRouting { // If we have the dht, add it to the available content routers if (this.dht && libp2p._config.dht.enabled) { - this.routers.push(this.dht) + this.routers.push(new DHTContentRouting(this.dht)) } } @@ -91,12 +92,12 @@ class ContentRouting { * @param {number} [options.minPeers] - minimum number of peers required to successfully put * @returns {Promise} */ - put (key, value, options) { + async put (key, value, options) { if (!this.libp2p.isStarted() || !this.dht.isStarted) { throw errCode(new Error(messages.NOT_STARTED_YET), codes.DHT_NOT_STARTED) } - return this.dht.put(key, value, options) + await drain(this.dht.put(key, value, options)) } /** @@ -108,12 +109,18 @@ class ContentRouting { * @param {number} [options.timeout] - optional timeout (default: 60000) * @returns {Promise} */ - get (key, options) { + async get (key, options) { if (!this.libp2p.isStarted() || !this.dht.isStarted) { throw errCode(new Error(messages.NOT_STARTED_YET), codes.DHT_NOT_STARTED) } - return this.dht.get(key, options) + for await (const event of this.dht.get(key, options)) { + if (event.name === 'VALUE') { + return { from: event.peerId, val: event.value } + } + } + + throw errCode(new Error(messages.NOT_FOUND), codes.ERR_NOT_FOUND) } /** @@ -123,14 +130,33 @@ class ContentRouting { * @param {number} nVals * @param {Object} [options] - get options * @param {number} [options.timeout] - optional timeout (default: 60000) - * @returns {Promise} */ - async getMany (key, nVals, options) { // eslint-disable-line require-await + async * getMany (key, nVals, options) { // eslint-disable-line require-await if (!this.libp2p.isStarted() || !this.dht.isStarted) { throw errCode(new Error(messages.NOT_STARTED_YET), codes.DHT_NOT_STARTED) } - return this.dht.getMany(key, nVals, options) + if (!nVals) { + return + } + + let gotValues = 0 + + for await (const event of this.dht.get(key, options)) { + if (event.name === 'VALUE') { + yield { from: event.peerId, val: event.value } + + gotValues++ + + if (gotValues === nVals) { + break + } + } + } + + if (gotValues === 0) { + throw errCode(new Error(messages.NOT_FOUND), codes.ERR_NOT_FOUND) + } } } diff --git a/src/dht/dht-content-routing.js b/src/dht/dht-content-routing.js new file mode 100644 index 00000000..ead668f3 --- /dev/null +++ b/src/dht/dht-content-routing.js @@ -0,0 +1,44 @@ +'use strict' + +const drain = require('it-drain') + +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('libp2p-interfaces/src/content-routing/types').ContentRouting} ContentRoutingModule + * @typedef {import('multiformats/cid').CID} CID + */ + +/** + * Wrapper class to convert events into returned values + * + * @implements {ContentRoutingModule} + */ +class DHTContentRouting { + /** + * @param {import('libp2p-kad-dht').DHT} dht + */ + constructor (dht) { + this._dht = dht + } + + /** + * @param {CID} cid + */ + async provide (cid) { + await drain(this._dht.provide(cid)) + } + + /** + * @param {CID} cid + * @param {*} options + */ + async * findProviders (cid, options) { + for await (const event of this._dht.findProviders(cid, options)) { + if (event.name === 'PROVIDER') { + yield * event.providers + } + } + } +} + +module.exports = { DHTContentRouting } diff --git a/src/dht/dht-peer-routing.js b/src/dht/dht-peer-routing.js new file mode 100644 index 00000000..762abc80 --- /dev/null +++ b/src/dht/dht-peer-routing.js @@ -0,0 +1,51 @@ +'use strict' + +const errCode = require('err-code') +const { messages, codes } = require('../errors') + +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('libp2p-interfaces/src/peer-routing/types').PeerRouting} PeerRoutingModule + */ + +/** + * Wrapper class to convert events into returned values + * + * @implements {PeerRoutingModule} + */ +class DHTPeerRouting { + /** + * @param {import('libp2p-kad-dht').DHT} dht + */ + constructor (dht) { + this._dht = dht + } + + /** + * @param {PeerId} peerId + * @param {any} options + */ + async findPeer (peerId, options = {}) { + for await (const event of this._dht.findPeer(peerId, options)) { + if (event.name === 'FINAL_PEER') { + return event.peer + } + } + + throw errCode(new Error(messages.NOT_FOUND), codes.ERR_NOT_FOUND) + } + + /** + * @param {Uint8Array} key + * @param {any} options + */ + async * getClosestPeers (key, options = {}) { + for await (const event of this._dht.getClosestPeers(key, options)) { + if (event.name === 'PEER_RESPONSE') { + yield * event.closer + } + } + } +} + +module.exports = { DHTPeerRouting } diff --git a/src/errors.js b/src/errors.js index 5b4d070f..efaed29c 100644 --- a/src/errors.js +++ b/src/errors.js @@ -3,7 +3,8 @@ exports.messages = { NOT_STARTED_YET: 'The libp2p node is not started yet', DHT_DISABLED: 'DHT is not available', - CONN_ENCRYPTION_REQUIRED: 'At least one connection encryption module is required' + CONN_ENCRYPTION_REQUIRED: 'At least one connection encryption module is required', + NOT_FOUND: 'Not found' } exports.codes = { @@ -29,6 +30,7 @@ exports.codes = { ERR_INVALID_PARAMETERS: 'ERR_INVALID_PARAMETERS', ERR_INVALID_PEER: 'ERR_INVALID_PEER', ERR_MUXER_UNAVAILABLE: 'ERR_MUXER_UNAVAILABLE', + ERR_NOT_FOUND: 'ERR_NOT_FOUND', ERR_TIMEOUT: 'ERR_TIMEOUT', ERR_TRANSPORT_UNAVAILABLE: 'ERR_TRANSPORT_UNAVAILABLE', ERR_TRANSPORT_DIAL_FAILED: 'ERR_TRANSPORT_DIAL_FAILED', diff --git a/src/index.js b/src/index.js index b06393bd..63edbf69 100644 --- a/src/index.js +++ b/src/index.js @@ -301,14 +301,9 @@ class Libp2p extends EventEmitter { // dht provided components (peerRouting, contentRouting, dht) if (this._modules.dht) { const DHT = this._modules.dht - // @ts-ignore Object is not constructable - this._dht = new DHT({ + // @ts-ignore TODO: types need fixing - DHT is an `object` which has no `create` method + this._dht = DHT.create({ libp2p: this, - dialer: this.dialer, - peerId: this.peerId, - peerStore: this.peerStore, - registrar: this.registrar, - datastore: this.datastore, ...this._config.dht }) } @@ -624,7 +619,7 @@ class Libp2p extends EventEmitter { // DHT subsystem if (this._config.dht.enabled) { - this._dht && this._dht.start() + this._dht && await this._dht.start() // TODO: this should be modified once random-walk is used as // the other discovery modules diff --git a/src/nat-manager.js b/src/nat-manager.js index 247401c7..4b0b60dd 100644 --- a/src/nat-manager.js +++ b/src/nat-manager.js @@ -114,7 +114,7 @@ class NatManager { const client = this._getClient() const publicIp = this._externalIp || await client.externalIp() - // @ts-ignore isPrivate has no call signatures + // @ts-expect-error types are wrong if (isPrivateIp(publicIp)) { throw new Error(`${publicIp} is private - please set config.nat.externalIp to an externally routable IP or ensure you are not behind a double NAT`) } diff --git a/src/peer-routing.js b/src/peer-routing.js index bc22f8e4..f37ce663 100644 --- a/src/peer-routing.js +++ b/src/peer-routing.js @@ -21,6 +21,7 @@ const { clearDelayedInterval // @ts-ignore module with no types } = require('set-delayed-interval') +const { DHTPeerRouting } = require('./dht/dht-peer-routing') /** * @typedef {import('peer-id')} PeerId @@ -51,7 +52,7 @@ class PeerRouting { // If we have the dht, add it to the available peer routers if (libp2p._dht && libp2p._config.dht.enabled) { - this._routers.push(libp2p._dht) + this._routers.push(new DHTPeerRouting(libp2p._dht)) } this._refreshManagerOptions = libp2p._options.peerRouting.refreshManager diff --git a/test/ts-use/package.json b/test/ts-use/package.json index 6445e42f..c056526a 100644 --- a/test/ts-use/package.json +++ b/test/ts-use/package.json @@ -10,7 +10,7 @@ "libp2p-delegated-peer-routing": "^0.10.0", "libp2p-gossipsub": "^0.9.0", "libp2p-interfaces": "^1.0.1", - "libp2p-kad-dht": "^0.23.1", + "libp2p-kad-dht": "^0.26.5", "libp2p-mplex": "^0.10.4", "@chainsafe/libp2p-noise": "^4.1.0", "libp2p-record": "^0.10.4",