fluencelabs/js-libp2p
1
0
Fork 0
mirror of https://github.com/fluencelabs/js-libp2p synced 2025-06-25 06:51:32 +00:00
Code Issues Projects Releases Wiki Activity

feat: async peerstore backed by datastores (#1058)

Browse Source 
We have a peerstore that keeps all data for all observed peers in memory with no eviction.

This is fine when you don't discover many peers but when using the DHT you encounter a significant number of peers so our peer storage grows and grows over time.

We have a persistent peer store, but it just periodically writes peers into the datastore to be read at startup, still keeping them in memory.

It also means a restart doesn't give you any temporary reprieve from the memory leak as the previously observed peer data is read into memory at startup.

This change refactors the peerstore to use a datastore by default, reading and writing peer info as it arrives.  It can be configured with a MemoryDatastore if desired.

It was necessary to change the peerstore and *book interfaces to be asynchronous since the datastore api is asynchronous.

BREAKING CHANGE: `libp2p.handle`, `libp2p.registrar.register` and the peerstore methods have become async
This commit is contained in:
Alex Potsides
2022-01-20 12:03:35 +00:00
committed by GitHub
parent 0a4dc54d08
commit 978eb3676f
94 changed files with 3263 additions and 4039 deletions
Download Patch File Download Diff File Expand all files Collapse all files

200
test/peer-store/metadata-book.spec.js
View File

@ -3,7 +3,7 @@
const { expect } = require('aegir/utils/chai')
const { fromString: uint8ArrayFromString } = require('uint8arrays/from-string')
const { MemoryDatastore } = require('datastore-core/memory')
const pDefer = require('p-defer')
const PeerStore = require('../../src/peer-store')
@ -12,7 +12,14 @@ const {
codes: { ERR_INVALID_PARAMETERS }
} = require('../../src/errors')
/**
* @typedef {import('../../src/peer-store/types').PeerStore} PeerStore
* @typedef {import('../../src/peer-store/types').MetadataBook} MetadataBook
* @typedef {import('peer-id')} PeerId
*/
describe('metadataBook', () => {
/** @type {PeerId} */
let peerId
before(async () => {
@ -20,10 +27,16 @@ describe('metadataBook', () => {
})
describe('metadataBook.set', () => {
let peerStore, mb
/** @type {PeerStore} */
let peerStore
/** @type {MetadataBook} */
let mb
beforeEach(() => {
peerStore = new PeerStore({ peerId })
peerStore = new PeerStore({
peerId,
datastore: new MemoryDatastore()
})
mb = peerStore.metadataBook
})
@ -31,9 +44,9 @@ describe('metadataBook', () => {
peerStore.removeAllListeners()
})
it('throws invalid parameters error if invalid PeerId is provided', () => {
it('throws invalid parameters error if invalid PeerId is provided', async () => {
try {
mb.set('invalid peerId')
await mb.set('invalid peerId')
} catch (/** @type {any} */ err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
@ -41,9 +54,9 @@ describe('metadataBook', () => {
throw new Error('invalid peerId should throw error')
})
it('throws invalid parameters error if no key provided', () => {
it('throws invalid parameters error if no metadata provided', async () => {
try {
mb.set(peerId)
await mb.set(peerId)
} catch (/** @type {any} */ err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
@ -51,9 +64,9 @@ describe('metadataBook', () => {
throw new Error('no key provided should throw error')
})
it('throws invalid parameters error if no value provided', () => {
it('throws invalid parameters error if no value provided', async () => {
try {
mb.set(peerId, 'location')
await mb.setValue(peerId, 'location')
} catch (/** @type {any} */ err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
@ -61,9 +74,9 @@ describe('metadataBook', () => {
throw new Error('no value provided should throw error')
})
it('throws invalid parameters error if value is not a buffer', () => {
it('throws invalid parameters error if value is not a buffer', async () => {
try {
mb.set(peerId, 'location', 'mars')
await mb.setValue(peerId, 'location', 'mars')
} catch (/** @type {any} */ err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
@ -71,30 +84,30 @@ describe('metadataBook', () => {
throw new Error('invalid value provided should throw error')
})
it('stores the content and emit change event', () => {
it('stores the content and emit change event', async () => {
const defer = pDefer()
const metadataKey = 'location'
const metadataValue = uint8ArrayFromString('mars')
peerStore.once('change:metadata', ({ peerId, metadata }) => {
expect(peerId).to.exist()
expect(metadata).to.equal(metadataKey)
expect(metadata.get(metadataKey)).to.equalBytes(metadataValue)
defer.resolve()
})
mb.set(peerId, metadataKey, metadataValue)
await mb.setValue(peerId, metadataKey, metadataValue)
const value = mb.getValue(peerId, metadataKey)
const value = await mb.getValue(peerId, metadataKey)
expect(value).to.equalBytes(metadataValue)
const peerMetadata = mb.get(peerId)
const peerMetadata = await mb.get(peerId)
expect(peerMetadata).to.exist()
expect(peerMetadata.get(metadataKey)).to.equalBytes(metadataValue)
return defer.promise
})
it('emits on set if not storing the exact same content', () => {
it('emits on set if not storing the exact same content', async () => {
const defer = pDefer()
const metadataKey = 'location'
const metadataValue1 = uint8ArrayFromString('mars')
@ -109,22 +122,22 @@ describe('metadataBook', () => {
})
// set 1
mb.set(peerId, metadataKey, metadataValue1)
await mb.setValue(peerId, metadataKey, metadataValue1)
// set 2 (same content)
mb.set(peerId, metadataKey, metadataValue2)
await mb.setValue(peerId, metadataKey, metadataValue2)
const value = mb.getValue(peerId, metadataKey)
const value = await mb.getValue(peerId, metadataKey)
expect(value).to.equalBytes(metadataValue2)
const peerMetadata = mb.get(peerId)
const peerMetadata = await mb.get(peerId)
expect(peerMetadata).to.exist()
expect(peerMetadata.get(metadataKey)).to.equalBytes(metadataValue2)
return defer.promise
})
it('does not emit on set if it is storing the exact same content', () => {
it('does not emit on set if it is storing the exact same content', async () => {
const defer = pDefer()
const metadataKey = 'location'
const metadataValue = uint8ArrayFromString('mars')
@ -138,10 +151,10 @@ describe('metadataBook', () => {
})
// set 1
mb.set(peerId, metadataKey, metadataValue)
await mb.setValue(peerId, metadataKey, metadataValue)
// set 2 (same content)
mb.set(peerId, metadataKey, metadataValue)
await mb.setValue(peerId, metadataKey, metadataValue)
// Wait 50ms for incorrect second event
setTimeout(() => {
@ -153,16 +166,22 @@ describe('metadataBook', () => {
})
describe('metadataBook.get', () => {
let peerStore, mb
/** @type {PeerStore} */
let peerStore
/** @type {MetadataBook} */
let mb
beforeEach(() => {
peerStore = new PeerStore({ peerId })
peerStore = new PeerStore({
peerId,
datastore: new MemoryDatastore()
})
mb = peerStore.metadataBook
})
it('throws invalid parameters error if invalid PeerId is provided', () => {
it('throws invalid parameters error if invalid PeerId is provided', async () => {
try {
mb.get('invalid peerId')
await mb.get('invalid peerId')
} catch (/** @type {any} */ err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
@ -170,35 +189,43 @@ describe('metadataBook', () => {
throw new Error('invalid peerId should throw error')
})
it('returns undefined if no metadata is known for the provided peer', () => {
const metadata = mb.get(peerId)
it('returns empty if no metadata is known for the provided peer', async () => {
const metadata = await mb.get(peerId)
expect(metadata).to.not.exist()
expect(metadata).to.be.empty()
})
it('returns the metadata stored', () => {
it('returns the metadata stored', async () => {
const metadataKey = 'location'
const metadataValue = uint8ArrayFromString('mars')
const metadata = new Map()
metadata.set(metadataKey, metadataValue)
mb.set(peerId, metadataKey, metadataValue)
await mb.set(peerId, metadata)
const peerMetadata = mb.get(peerId)
const peerMetadata = await mb.get(peerId)
expect(peerMetadata).to.exist()
expect(peerMetadata.get(metadataKey)).to.equalBytes(metadataValue)
})
})
describe('metadataBook.getValue', () => {
let peerStore, mb
/** @type {PeerStore} */
let peerStore
/** @type {MetadataBook} */
let mb
beforeEach(() => {
peerStore = new PeerStore({ peerId })
peerStore = new PeerStore({
peerId,
datastore: new MemoryDatastore()
})
mb = peerStore.metadataBook
})
it('throws invalid parameters error if invalid PeerId is provided', () => {
it('throws invalid parameters error if invalid PeerId is provided', async () => {
try {
mb.getValue('invalid peerId')
await mb.getValue('invalid peerId')
} catch (/** @type {any} */ err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
@ -206,48 +233,53 @@ describe('metadataBook', () => {
throw new Error('invalid peerId should throw error')
})
it('returns undefined if no metadata is known for the provided peer', () => {
it('returns undefined if no metadata is known for the provided peer', async () => {
const metadataKey = 'location'
const metadata = mb.getValue(peerId, metadataKey)
const metadata = await mb.getValue(peerId, metadataKey)
expect(metadata).to.not.exist()
})
it('returns the metadata value stored for the given key', () => {
it('returns the metadata value stored for the given key', async () => {
const metadataKey = 'location'
const metadataValue = uint8ArrayFromString('mars')
mb.set(peerId, metadataKey, metadataValue)
await mb.setValue(peerId, metadataKey, metadataValue)
const value = mb.getValue(peerId, metadataKey)
const value = await mb.getValue(peerId, metadataKey)
expect(value).to.exist()
expect(value).to.equalBytes(metadataValue)
})
it('returns undefined if no metadata is known for the provided peer and key', () => {
it('returns undefined if no metadata is known for the provided peer and key', async () => {
const metadataKey = 'location'
const metadataBadKey = 'nickname'
const metadataValue = uint8ArrayFromString('mars')
mb.set(peerId, metadataKey, metadataValue)
const metadata = mb.getValue(peerId, metadataBadKey)
await mb.setValue(peerId, metadataKey, metadataValue)
const metadata = await mb.getValue(peerId, metadataBadKey)
expect(metadata).to.not.exist()
})
})
describe('metadataBook.delete', () => {
let peerStore, mb
/** @type {PeerStore} */
let peerStore
/** @type {MetadataBook} */
let mb
beforeEach(() => {
peerStore = new PeerStore({ peerId })
peerStore = new PeerStore({
peerId,
datastore: new MemoryDatastore()
})
mb = peerStore.metadataBook
})
it('throwns invalid parameters error if invalid PeerId is provided', () => {
it('throws invalid parameters error if invalid PeerId is provided', async () => {
try {
mb.delete('invalid peerId')
await mb.delete('invalid peerId')
} catch (/** @type {any} */ err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
@ -255,16 +287,14 @@ describe('metadataBook', () => {
throw new Error('invalid peerId should throw error')
})
it('returns false if no records exist for the peer and no event is emitted', () => {
it('should not emit event if no records exist for the peer', async () => {
const defer = pDefer()
peerStore.on('change:metadata', () => {
defer.reject()
})
const deleted = mb.delete(peerId)
expect(deleted).to.equal(false)
await mb.delete(peerId)
// Wait 50ms for incorrect invalid event
setTimeout(() => {
@ -274,37 +304,41 @@ describe('metadataBook', () => {
return defer.promise
})
it('returns true if the record exists and an event is emitted', () => {
it('should emit an event if the record exists for the peer', async () => {
const defer = pDefer()
const metadataKey = 'location'
const metadataValue = uint8ArrayFromString('mars')
mb.set(peerId, metadataKey, metadataValue)
await mb.setValue(peerId, metadataKey, metadataValue)
// Listen after set
peerStore.on('change:metadata', () => {
defer.resolve()
})
const deleted = mb.delete(peerId)
expect(deleted).to.equal(true)
await mb.delete(peerId)
return defer.promise
})
})
describe('metadataBook.deleteValue', () => {
let peerStore, mb
/** @type {PeerStore} */
let peerStore
/** @type {MetadataBook} */
let mb
beforeEach(() => {
peerStore = new PeerStore({ peerId })
peerStore = new PeerStore({
peerId,
datastore: new MemoryDatastore()
})
mb = peerStore.metadataBook
})
it('throws invalid parameters error if invalid PeerId is provided', () => {
it('throws invalid parameters error if invalid PeerId is provided', async () => {
try {
mb.deleteValue('invalid peerId')
await mb.deleteValue('invalid peerId')
} catch (/** @type {any} */ err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
@ -312,7 +346,7 @@ describe('metadataBook', () => {
throw new Error('invalid peerId should throw error')
})
it('returns false if no records exist for the peer and no event is emitted', () => {
it('should not emit event if no records exist for the peer', async () => {
const defer = pDefer()
const metadataKey = 'location'
@ -320,9 +354,7 @@ describe('metadataBook', () => {
defer.reject()
})
const deleted = mb.deleteValue(peerId, metadataKey)
expect(deleted).to.equal(false)
await mb.deleteValue(peerId, metadataKey)
// Wait 50ms for incorrect invalid event
setTimeout(() => {
@ -332,45 +364,19 @@ describe('metadataBook', () => {
return defer.promise
})
it('returns true if the record exists and an event is emitted', () => {
it('should emit event if a record exists for the peer', async () => {
const defer = pDefer()
const metadataKey = 'location'
const metadataValue = uint8ArrayFromString('mars')
mb.set(peerId, metadataKey, metadataValue)
await mb.setValue(peerId, metadataKey, metadataValue)
// Listen after set
peerStore.on('change:metadata', () => {
defer.resolve()
})
const deleted = mb.deleteValue(peerId, metadataKey)
expect(deleted).to.equal(true)
return defer.promise
})
it('returns false if there is a record for the peer but not the given metadata key', () => {
const defer = pDefer()
const metadataKey = 'location'
const metadataBadKey = 'nickname'
const metadataValue = uint8ArrayFromString('mars')
mb.set(peerId, metadataKey, metadataValue)
peerStore.on('change:metadata', () => {
defer.reject()
})
const deleted = mb.deleteValue(peerId, metadataBadKey)
expect(deleted).to.equal(false)
// Wait 50ms for incorrect invalid event
setTimeout(() => {
defer.resolve()
}, 50)
await mb.deleteValue(peerId, metadataKey)
return defer.promise
})