fluencelabs/js-libp2p
1
0
Fork 0
mirror of https://github.com/fluencelabs/js-libp2p synced 2025-07-24 13:01:58 +00:00
Code Issues Projects Releases Wiki Activity

feat: async peerstore backed by datastores (#1058)

Browse Source 
We have a peerstore that keeps all data for all observed peers in memory with no eviction.

This is fine when you don't discover many peers but when using the DHT you encounter a significant number of peers so our peer storage grows and grows over time.

We have a persistent peer store, but it just periodically writes peers into the datastore to be read at startup, still keeping them in memory.

It also means a restart doesn't give you any temporary reprieve from the memory leak as the previously observed peer data is read into memory at startup.

This change refactors the peerstore to use a datastore by default, reading and writing peer info as it arrives.  It can be configured with a MemoryDatastore if desired.

It was necessary to change the peerstore and *book interfaces to be asynchronous since the datastore api is asynchronous.

BREAKING CHANGE: `libp2p.handle`, `libp2p.registrar.register` and the peerstore methods have become async
This commit is contained in:
Alex Potsides
2022-01-20 12:03:35 +00:00
committed by GitHub
parent 0a4dc54d08
commit 978eb3676f
94 changed files with 3263 additions and 4039 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
test/relay/auto-relay.node.js
View File

@@ -82,7 +82,7 @@ describe('auto-relay', () => {
const originalMultiaddrsLength = relayLibp2p.multiaddrs.length
// Discover relay
libp2p.peerStore.addressBook.add(relayLibp2p.peerId, relayLibp2p.multiaddrs)
await libp2p.peerStore.addressBook.add(relayLibp2p.peerId, relayLibp2p.multiaddrs)
await libp2p.dial(relayLibp2p.peerId)
// Wait for peer added as listen relay
@@ -94,7 +94,7 @@ describe('auto-relay', () => {
expect(libp2p.multiaddrs[originalMultiaddrsLength].getPeerId()).to.eql(relayLibp2p.peerId.toB58String())
// Peer has relay multicodec
const knownProtocols = libp2p.peerStore.protoBook.get(relayLibp2p.peerId)
const knownProtocols = await libp2p.peerStore.protoBook.get(relayLibp2p.peerId)
expect(knownProtocols).to.include(relayMulticodec)
})
})
@@ -165,7 +165,7 @@ describe('auto-relay', () => {
sinon.spy(autoRelay1, '_addListenRelay')
// Discover relay
relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
const originalMultiaddrs1Length = relayLibp2p1.multiaddrs.length
const originalMultiaddrs2Length = relayLibp2p2.multiaddrs.length
@@ -184,7 +184,7 @@ describe('auto-relay', () => {
expect(relayLibp2p1.multiaddrs[originalMultiaddrs1Length].getPeerId()).to.eql(relayLibp2p2.peerId.toB58String())
// Peer has relay multicodec
const knownProtocols = relayLibp2p1.peerStore.protoBook.get(relayLibp2p2.peerId)
const knownProtocols = await relayLibp2p1.peerStore.protoBook.get(relayLibp2p2.peerId)
expect(knownProtocols).to.include(relayMulticodec)
})
@@ -193,7 +193,7 @@ describe('auto-relay', () => {
const originalMultiaddrs2Length = relayLibp2p2.multiaddrs.length
// Discover relay
relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.dial(relayLibp2p2.peerId)
@@ -206,7 +206,7 @@ describe('auto-relay', () => {
// Dial from the other through a relay
const relayedMultiaddr2 = new Multiaddr(`${relayLibp2p1.multiaddrs[0]}/p2p/${relayLibp2p1.peerId.toB58String()}/p2p-circuit`)
libp2p.peerStore.addressBook.add(relayLibp2p2.peerId, [relayedMultiaddr2])
await libp2p.peerStore.addressBook.add(relayLibp2p2.peerId, [relayedMultiaddr2])
await libp2p.dial(relayLibp2p2.peerId)
})
@@ -220,7 +220,7 @@ describe('auto-relay', () => {
sinon.spy(autoRelay1._listenRelays, 'add')
// Discover one relay and connect
relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.dial(relayLibp2p2.peerId)
expect(relayLibp2p1.connectionManager.size).to.eql(1)
@@ -237,11 +237,11 @@ describe('auto-relay', () => {
expect(relayLibp2p1.multiaddrs[originalMultiaddrs1Length].getPeerId()).to.eql(relayLibp2p2.peerId.toB58String())
// Relay2 has relay multicodec
const knownProtocols2 = relayLibp2p1.peerStore.protoBook.get(relayLibp2p2.peerId)
const knownProtocols2 = await relayLibp2p1.peerStore.protoBook.get(relayLibp2p2.peerId)
expect(knownProtocols2).to.include(relayMulticodec)
// Discover an extra relay and connect
relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p1.dial(relayLibp2p3.peerId)
// Wait to guarantee the dialed peer is not added as a listen relay
@@ -253,7 +253,7 @@ describe('auto-relay', () => {
expect(relayLibp2p1.connectionManager.size).to.eql(2)
// Relay2 has relay multicodec
const knownProtocols3 = relayLibp2p1.peerStore.protoBook.get(relayLibp2p3.peerId)
const knownProtocols3 = await relayLibp2p1.peerStore.protoBook.get(relayLibp2p3.peerId)
expect(knownProtocols3).to.include(relayMulticodec)
})
@@ -264,7 +264,7 @@ describe('auto-relay', () => {
sinon.spy(relayLibp2p1.identifyService, 'pushToPeerStore')
// Discover one relay and connect
relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.dial(relayLibp2p2.peerId)
// Wait for listenning on the relay
@@ -283,7 +283,7 @@ describe('auto-relay', () => {
expect(autoRelay1._listenRelays.size).to.equal(0)
// Identify push for removing listen relay multiaddr
expect(relayLibp2p1.identifyService.pushToPeerStore.callCount).to.equal(2)
await pWaitFor(() => relayLibp2p1.identifyService.pushToPeerStore.callCount === 2)
})
it('should try to listen on other connected peers relayed address if one used relay disconnects', async () => {
@@ -294,11 +294,11 @@ describe('auto-relay', () => {
sinon.spy(relayLibp2p1.transportManager, 'listen')
// Discover one relay and connect
relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.dial(relayLibp2p2.peerId)
// Discover an extra relay and connect
relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p1.dial(relayLibp2p3.peerId)
// Wait for both peer to be attempted to added as listen relay
@@ -337,11 +337,11 @@ describe('auto-relay', () => {
sinon.spy(relayLibp2p1.transportManager, 'listen')
// Discover one relay and connect
relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.dial(relayLibp2p2.peerId)
// Discover an extra relay and connect to gather its Hop support
relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p1.dial(relayLibp2p3.peerId)
// Wait for both peer to be attempted to added as listen relay
@@ -382,11 +382,11 @@ describe('auto-relay', () => {
sinon.spy(relayLibp2p1.transportManager, 'listen')
// Discover one relay and connect
relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, relayLibp2p2.multiaddrs)
await relayLibp2p1.dial(relayLibp2p2.peerId)
// Discover an extra relay and connect to gather its Hop support
relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p1.dial(relayLibp2p3.peerId)
// Wait for both peer to be attempted to added as listen relay
@@ -479,7 +479,7 @@ describe('auto-relay', () => {
sinon.spy(autoRelay2, '_addListenRelay')
// Relay 1 discovers Relay 3 and connect
relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p1.dial(relayLibp2p3.peerId)
// Wait for peer added as listen relay
@@ -487,7 +487,7 @@ describe('auto-relay', () => {
expect(autoRelay1._listenRelays.size).to.equal(1)
// Relay 2 discovers Relay 3 and connect
relayLibp2p2.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p2.peerStore.addressBook.add(relayLibp2p3.peerId, relayLibp2p3.multiaddrs)
await relayLibp2p2.dial(relayLibp2p3.peerId)
// Wait for peer added as listen relay
@@ -496,7 +496,7 @@ describe('auto-relay', () => {
// Relay 1 discovers Relay 2 relayed multiaddr via Relay 3
const ma2RelayedBy3 = relayLibp2p2.multiaddrs[relayLibp2p2.multiaddrs.length - 1]
relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, [ma2RelayedBy3])
await relayLibp2p1.peerStore.addressBook.add(relayLibp2p2.peerId, [ma2RelayedBy3])
await relayLibp2p1.dial(relayLibp2p2.peerId)
// Peer not added as listen relay

8
test/relay/relay.node.js
View File

@@ -8,7 +8,6 @@ const { Multiaddr } = require('multiaddr')
const { collect } = require('streaming-iterables')
const pipe = require('it-pipe')
const AggregateError = require('aggregate-error')
const PeerId = require('peer-id')
const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
const { createPeerId } = require('../utils/creators/peer')
@@ -46,14 +45,13 @@ describe('Dialing (via relay, TCP)', () => {
return Promise.all([srcLibp2p, relayLibp2p, dstLibp2p].map(libp2p => libp2p.start()))
})
afterEach(() => {
afterEach(async () => {
// Stop each node
return Promise.all([srcLibp2p, relayLibp2p, dstLibp2p].map(async libp2p => {
await libp2p.stop()
// Clear the peer stores
for (const peerIdStr of libp2p.peerStore.peers.keys()) {
const peerId = PeerId.createFromB58String(peerIdStr)
libp2p.peerStore.delete(peerId)
for await (const peer of libp2p.peerStore.getPeers()) {
libp2p.peerStore.delete(peer.id)
}
}))
})