fluencelabs/js-libp2p
1
0
Fork 0
mirror of https://github.com/fluencelabs/js-libp2p synced 2025-06-24 14:31:35 +00:00
Code Issues Projects Releases Wiki Activity

feat: async peerstore backed by datastores (#1058)

Browse Source 
We have a peerstore that keeps all data for all observed peers in memory with no eviction.

This is fine when you don't discover many peers but when using the DHT you encounter a significant number of peers so our peer storage grows and grows over time.

We have a persistent peer store, but it just periodically writes peers into the datastore to be read at startup, still keeping them in memory.

It also means a restart doesn't give you any temporary reprieve from the memory leak as the previously observed peer data is read into memory at startup.

This change refactors the peerstore to use a datastore by default, reading and writing peer info as it arrives.  It can be configured with a MemoryDatastore if desired.

It was necessary to change the peerstore and *book interfaces to be asynchronous since the datastore api is asynchronous.

BREAKING CHANGE: `libp2p.handle`, `libp2p.registrar.register` and the peerstore methods have become async
This commit is contained in:
Alex Potsides
2022-01-20 12:03:35 +00:00
committed by GitHub
parent 0a4dc54d08
commit 978eb3676f
94 changed files with 3263 additions and 4039 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
examples/connection-encryption/1.js
View File

@ -30,7 +30,7 @@ const createNode = async () => {
createNode()
])
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
node2.handle('/a-protocol', ({ stream }) => {
pipe(

20
examples/delegated-routing/package.json
View File

@ -3,16 +3,16 @@
"version": "0.1.0",
"private": true,
"dependencies": {
"ipfs": "~0.34.4",
"libp2p": "github:libp2p/js-libp2p#master",
"libp2p-delegated-content-routing": "~0.2.2",
"libp2p-delegated-peer-routing": "~0.2.2",
"libp2p-kad-dht": "^0.26.5",
"libp2p-mplex": "~0.8.5",
"libp2p-secio": "~0.11.1",
"libp2p-webrtc-star": "~0.15.8",
"libp2p-websocket-star": "~0.10.2",
"libp2p-websockets": "~0.12.2",
"@chainsafe/libp2p-noise": "^5.0.2",
"ipfs-core": "^0.13.0",
"libp2p": "../../",
"libp2p-delegated-content-routing": "^0.11.0",
"libp2p-delegated-peer-routing": "^0.11.1",
"libp2p-kad-dht": "^0.28.6",
"libp2p-mplex": "^0.10.4",
"libp2p-webrtc-star": "^0.25.0",
"libp2p-websocket-star": "^0.10.2",
"libp2p-websockets": "^0.16.2",
"react": "^16.8.6",
"react-dom": "^16.8.6",
"react-scripts": "2.1.8"

4
examples/delegated-routing/src/App.js
View File

@ -2,7 +2,7 @@
'use strict'
import React from 'react'
import Ipfs from 'ipfs'
import Ipfs from 'ipfs-core'
import libp2pBundle from './libp2p-bundle'
const Component = React.Component
@ -70,7 +70,7 @@ class App extends Component {
}
componentDidMount () {
window.ipfs = this.ipfs = new Ipfs({
window.ipfs = this.ipfs = Ipfs.create({
config: {
Addresses: {
Swarm: []

4
examples/delegated-routing/src/libp2p-bundle.js
View File

@ -6,7 +6,7 @@ const Websockets = require('libp2p-websockets')
const WebSocketStar = require('libp2p-websocket-star')
const WebRTCStar = require('libp2p-webrtc-star')
const MPLEX = require('libp2p-mplex')
const SECIO = require('libp2p-secio')
const { NOISE } = require('@chainsafe/libp2p-noise')
const KadDHT = require('libp2p-kad-dht')
const DelegatedPeerRouter = require('libp2p-delegated-peer-routing')
const DelegatedContentRouter = require('libp2p-delegated-content-routing')
@ -48,7 +48,7 @@ export default function Libp2pBundle ({peerInfo, peerBook}) {
MPLEX
],
connEncryption: [
SECIO
NOISE
],
dht: KadDHT
},

2
examples/discovery-mechanisms/3.js
View File

@ -5,7 +5,7 @@ const Libp2p = require('../../')
const TCP = require('libp2p-tcp')
const Mplex = require('libp2p-mplex')
const { NOISE } = require('@chainsafe/libp2p-noise')
const Gossipsub = require('libp2p-gossipsub')
const Gossipsub = require('@achingbrain/libp2p-gossipsub')
const Bootstrap = require('libp2p-bootstrap')
const PubsubPeerDiscovery = require('libp2p-pubsub-peer-discovery')

4
examples/discovery-mechanisms/bootstrapers.js vendored
View File

@ -1,13 +1,13 @@
'use strict'
// Find this list at: https://github.com/ipfs/js-ipfs/blob/master/packages/ipfs-core/src/runtime/config-nodejs.js
// Find this list at: https://github.com/ipfs/js-ipfs/blob/master/packages/ipfs-core-config/src/config.js
const bootstrapers = [
'/ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmZa1sAxajnQjVM8WjWXoMbmPd7NsWhfKsPkErzpm9wGkp',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt'
'/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt',
]
module.exports = bootstrapers

26
examples/discovery-mechanisms/test-1.js
View File

@ -2,17 +2,9 @@
const path = require('path')
const execa = require('execa')
const pWaitFor = require('p-wait-for')
const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
const bootstrapers = require('./bootstrapers')
const discoveredCopy = 'Discovered:'
const connectedCopy = 'Connection established to:'
async function test () {
const discoveredNodes = []
const connectedNodes = []
process.stdout.write('1.js\n')
const proc = execa('node', [path.join(__dirname, '1.js')], {
@ -20,23 +12,17 @@ async function test () {
all: true
})
let output = ''
proc.all.on('data', async (data) => {
process.stdout.write(data)
const line = uint8ArrayToString(data)
output += uint8ArrayToString(data)
// Discovered or Connected
if (line.includes(discoveredCopy)) {
const id = line.trim().split(discoveredCopy)[1]
discoveredNodes.push(id)
} else if (line.includes(connectedCopy)) {
const id = line.trim().split(connectedCopy)[1]
connectedNodes.push(id)
// Discovered and connected
if (output.includes('Connection established to:')) {
proc.kill()
}
})
await pWaitFor(() => discoveredNodes.length === bootstrapers.length && connectedNodes.length === bootstrapers.length)
proc.kill()
}
module.exports = test

1
examples/libp2p-in-the-browser/.babelrc
View File

@ -1,4 +1,3 @@
{
"presets": ["@babel/preset-env"],
"plugins": ["syntax-async-functions","transform-regenerator"]
}

7
examples/libp2p-in-the-browser/package.json
View File

@ -14,12 +14,11 @@
"author": "",
"license": "ISC",
"dependencies": {
"@babel/preset-env": "^7.13.0",
"@chainsafe/libp2p-noise": "^5.0.2",
"libp2p": "../../",
"libp2p-bootstrap": "^0.13.0",
"libp2p-bootstrap": "^0.14.0",
"libp2p-mplex": "^0.10.4",
"@chainsafe/libp2p-noise": "^4.1.0",
"libp2p-webrtc-star": "^0.23.0",
"libp2p-webrtc-star": "^0.25.0",
"libp2p-websockets": "^0.16.1"
},
"devDependencies": {

2
examples/package.json
View File

@ -8,12 +8,12 @@
},
"license": "MIT",
"dependencies": {
"@achingbrain/libp2p-gossipsub": "^0.12.2",
"execa": "^2.1.0",
"fs-extra": "^8.1.0",
"libp2p": "../src",
"libp2p-pubsub-peer-discovery": "^4.0.0",
"libp2p-relay-server": "^0.3.0",
"libp2p-gossipsub": "^0.11.0",
"p-defer": "^3.0.0",
"uint8arrays": "^3.0.0",
"which": "^2.0.1"

4
examples/peer-and-content-routing/1.js
View File

@ -38,8 +38,8 @@ const createNode = async () => {
createNode()
])
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await Promise.all([
node1.dial(node2.peerId),

4
examples/peer-and-content-routing/2.js
View File

@ -40,8 +40,8 @@ const createNode = async () => {
createNode()
])
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await Promise.all([
node1.dial(node2.peerId),

4
examples/peer-and-content-routing/README.md
View File

@ -43,8 +43,8 @@ const node1 = nodes[0]
const node2 = nodes[1]
const node3 = nodes[2]
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await Promise.all([
node1.dial(node2.peerId),

19
examples/peer-and-content-routing/test-1.js
View File

@ -2,35 +2,28 @@
const path = require('path')
const execa = require('execa')
const pWaitFor = require('p-wait-for')
const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
async function test() {
process.stdout.write('1.js\n')
const addrs = []
let foundIt = false
const proc = execa('node', [path.join(__dirname, '1.js')], {
cwd: path.resolve(__dirname),
all: true
})
let output = ''
proc.all.on('data', async (data) => {
process.stdout.write(data)
const line = uint8ArrayToString(data)
output += uint8ArrayToString(data)
// Discovered peer
if (!foundIt && line.includes('Found it, multiaddrs are:')) {
foundIt = true
// Discovered peers
if (output.includes('Found it, multiaddrs are:')) {
proc.kill()
}
addrs.push(line)
})
await pWaitFor(() => addrs.length === 2)
proc.kill()
}
module.exports = test

22
examples/peer-and-content-routing/test-2.js
View File

@ -2,39 +2,27 @@
const path = require('path')
const execa = require('execa')
const pDefer = require('p-defer')
const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
const providedCopy = 'is providing'
const foundCopy = 'Found provider:'
async function test() {
process.stdout.write('2.js\n')
const providedDefer = pDefer()
const foundDefer = pDefer()
const proc = execa('node', [path.join(__dirname, '2.js')], {
cwd: path.resolve(__dirname),
all: true
})
let output = ''
proc.all.on('data', async (data) => {
process.stdout.write(data)
const line = uint8ArrayToString(data)
output += uint8ArrayToString(data)
if (line.includes(providedCopy)) {
providedDefer.resolve()
} else if (line.includes(foundCopy)) {
foundDefer.resolve()
if (output.includes('Found provider:')) {
proc.kill()
}
})
await Promise.all([
providedDefer.promise,
foundDefer.promise
])
proc.kill()
}
module.exports = test

2
examples/pnet/index.js
View File

@ -29,7 +29,7 @@ generate(otherSwarmKey)
console.log('nodes started...')
// Add node 2 data to node1's PeerStore
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.dial(node2.peerId)
node2.handle('/private', ({ stream }) => {

2
examples/protocol-and-stream-muxing/1.js
View File

@ -31,7 +31,7 @@ const createNode = async () => {
])
// Add node's 2 data to the PeerStore
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
// exact matching
node2.handle('/your-protocol', ({ stream }) => {

2
examples/protocol-and-stream-muxing/2.js
View File

@ -31,7 +31,7 @@ const createNode = async () => {
])
// Add node's 2 data to the PeerStore
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
node2.handle(['/a', '/b'], ({ protocol, stream }) => {
pipe(

4
examples/protocol-and-stream-muxing/3.js
View File

@ -32,8 +32,8 @@ const createNode = async () => {
])
// Add node's 2 data to the PeerStore
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
node1.handle('/node-1', ({ stream }) => {
pipe(
stream,

2
examples/protocol-and-stream-muxing/README.md
View File

@ -20,7 +20,7 @@ const node1 = nodes[0]
const node2 = nodes[1]
// Add node's 2 data to the PeerStore
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
// Here we are telling libp2p that if someone dials this node to talk with the `/your-protocol`
// multicodec, the protocol identifier, please call this handler and give it the stream

4
examples/pubsub/1.js
View File

@ -5,7 +5,7 @@ const Libp2p = require('../../')
const TCP = require('libp2p-tcp')
const Mplex = require('libp2p-mplex')
const { NOISE } = require('@chainsafe/libp2p-noise')
const Gossipsub = require('libp2p-gossipsub')
const Gossipsub = require('@achingbrain/libp2p-gossipsub')
const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
@ -35,7 +35,7 @@ const createNode = async () => {
])
// Add node's 2 data to the PeerStore
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.dial(node2.peerId)
node1.pubsub.on(topic, (msg) => {

2
examples/pubsub/README.md
View File

@ -49,7 +49,7 @@ const node1 = nodes[0]
const node2 = nodes[1]
// Add node's 2 data to the PeerStore
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.dial(node2.peerId)
node1.pubsub.on(topic, (msg) => {

6
examples/pubsub/message-filtering/1.js
View File

@ -5,7 +5,7 @@ const Libp2p = require('../../../')
const TCP = require('libp2p-tcp')
const Mplex = require('libp2p-mplex')
const { NOISE } = require('@chainsafe/libp2p-noise')
const Gossipsub = require('libp2p-gossipsub')
const Gossipsub = require('@achingbrain/libp2p-gossipsub')
const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
const { toString: uint8ArrayToString } = require('uint8arrays/to-string')
@ -36,10 +36,10 @@ const createNode = async () => {
])
// node1 conect to node2 and node2 conect to node3
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.dial(node2.peerId)
node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await node2.dial(node3.peerId)
//subscribe

4
examples/pubsub/message-filtering/README.md
View File

@ -32,10 +32,10 @@ const [node1, node2, node3] = await Promise.all([
createNode(),
])
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.dial(node2.peerId)
node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await node2.dial(node3.peerId)
```

2
examples/transports/2.js
View File

@ -49,7 +49,7 @@ function printAddrs (node, number) {
console.log(result.toString())
})
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
const { stream } = await node1.dialProtocol(node2.peerId, '/print')
await pipe(

6
examples/transports/3.js
View File

@ -60,9 +60,9 @@ function print ({ stream }) {
node2.handle('/print', print)
node3.handle('/print', print)
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
node3.peerStore.addressBook.set(node1.peerId, node1.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await node3.peerStore.addressBook.set(node1.peerId, node1.multiaddrs)
// node 1 (TCP) dials to node 2 (TCP+WebSockets)
const { stream } = await node1.dialProtocol(node2.peerId, '/print')

8
examples/transports/README.md
View File

@ -140,7 +140,7 @@ Then add,
console.log(result.toString())
})
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
const { stream } = await node1.dialProtocol(node2.peerId, '/print')
await pipe(
@ -224,9 +224,9 @@ node1.handle('/print', print)
node2.handle('/print', print)
node3.handle('/print', print)
node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
node3.peerStore.addressBook.set(node1.peerId, node1.multiaddrs)
await node1.peerStore.addressBook.set(node2.peerId, node2.multiaddrs)
await node2.peerStore.addressBook.set(node3.peerId, node3.multiaddrs)
await node3.peerStore.addressBook.set(node1.peerId, node1.multiaddrs)
// node 1 (TCP) dials to node 2 (TCP+WebSockets)
const { stream } = await node1.dialProtocol(node2.peerId, '/print')

2
examples/utils.js
View File

@ -30,7 +30,7 @@ async function waitForOutput (expectedOutput, command, args = [], opts = {}) {
const proc = execa(command, args, opts)
let output = ''
let time = 120000
let time = 600000
let timeout = setTimeout(() => {
throw new Error(`Did not see "${expectedOutput}" in output from "${[command].concat(args).join(' ')}" after ${time/1000}s`)

1
examples/webrtc-direct/listener.js
View File

@ -40,5 +40,4 @@ const PeerId = require('peer-id')
console.log('Listening on:')
node.multiaddrs.forEach((ma) => console.log(`${ma.toString()}/p2p/${node.peerId.toB58String()}`))
})()

4
examples/webrtc-direct/package.json
View File

@ -20,10 +20,10 @@
"util": "^0.12.3"
},
"dependencies": {
"@chainsafe/libp2p-noise": "^5.0.2",
"libp2p": "../../",
"libp2p-bootstrap": "^0.13.0",
"libp2p-bootstrap": "^0.14.0",
"libp2p-mplex": "^0.10.4",
"@chainsafe/libp2p-noise": "^4.1.0",
"libp2p-webrtc-direct": "^0.7.0",
"peer-id": "^0.16.0"
},