feat: peerStore persistence

This commit is contained in:
Vasco Santos 2020-04-23 14:08:34 +02:00 committed by Jacob Heun
parent 43630f1e0b
commit 5123a8357b
13 changed files with 493 additions and 63 deletions

View File

@ -50,6 +50,7 @@
"err-code": "^2.0.0",
"events": "^3.1.0",
"hashlru": "^2.3.0",
"interface-datastore": "^0.8.3",
"ipfs-utils": "^2.2.0",
"it-all": "^1.0.1",
"it-buffer": "^0.1.2",

View File

@ -20,6 +20,9 @@ const DefaultConfig = {
metrics: {
enabled: false
},
peerStore: {
persistence: true
},
config: {
dht: {
enabled: false,

View File

@ -6,6 +6,7 @@ const globalThis = require('ipfs-utils/src/globalthis')
const log = debug('libp2p')
log.error = debug('libp2p:error')
const { MemoryDatastore } = require('interface-datastore')
const PeerId = require('peer-id')
const peerRouting = require('./peer-routing')
@ -43,9 +44,12 @@ class Libp2p extends EventEmitter {
// and add default values where appropriate
this._options = validateConfig(_options)
this.datastore = this._options.datastore
this.peerId = this._options.peerId
this.peerStore = new PeerStore()
this.datastore = this._options.datastore || new MemoryDatastore()
this.peerStore = new PeerStore({
datastore: this.datastore,
...this._options.peerStore
})
// Addresses {listen, announce, noAnnounce}
this.addresses = this._options.addresses
@ -393,6 +397,9 @@ class Libp2p extends EventEmitter {
// Listen on the provided transports
await this.transportManager.listen()
// Start PeerStore
await this.peerStore.load()
if (this._config.pubsub.enabled) {
this.pubsub && this.pubsub.start()
}

View File

@ -83,7 +83,51 @@ Access to its underlying books:
- `change:multiaadrs` - emitted when a known peer has a different set of multiaddrs.
- `change:protocols` - emitted when a known peer supports a different set of protocols.
## Data Persistence
The data stored in the PeerStore will be persisted by default. Keeping a record of the peers already discovered by the peer, as well as their known data aims to improve the efficiency of peers joining the network after being offline.
---
TODO: Discuss if we should make it persisted by default now. Taking into consideration that we will use a MemoryDatastore by default, unless the user configures a datastore to use, it will be worthless. It might make sense to make it disabled by default until we work on improving configuration and provide good defauls for each environment.
---
The libp2p node will need to receive a [datastore](https://github.com/ipfs/interface-datastore), in order to store this data in a persistent way. Otherwise, it will be stored on a [memory datastore](https://github.com/ipfs/interface-datastore/blob/master/src/memory.js).
A [datastore](https://github.com/ipfs/interface-datastore) stores its data in a key-value fashion. As a result, we need coherent keys so that we do not overwrite data.
Taking into account that a datastore allows queries using a key prefix, we can find all the information if we define a consistent namespace that allow us to find the content without having any information. The namespaces were defined as follows:
**AddressBook**
All the knownw peer addresses are stored with a key pattern as follows:
`/peers/addrs/<b32 peer id no padding>`
**ProtoBook**
All the knownw peer protocols are stored with a key pattern as follows:
`/peers/protos/<b32 peer id no padding>`
**KeyBook**
_NOT_YET_IMPLEMENTED_
All public and private keys are stored under the following pattern:
` /peers/keys/<b32 peer id no padding>/{pub, priv}`
**MetadataBook**
_NOT_YET_IMPLEMENTED_
Metadata is stored under the following key pattern:
`/peers/metadata/<b32 peer id no padding>/<key>`
## Future Considerations
- If multiaddr TTLs are added, the PeerStore may schedule jobs to delete all addresses that exceed the TTL to prevent AddressBook bloating
- Further API methods will probably need to be added in the context of multiaddr validity and confidence.
- When improving libp2p configuration for specific runtimes, we should take into account the PeerStore recommended datastore.
- When improving libp2p configuration, we should think about a possible way of allowing the configuration of Bootstrap to be influenced by the persisted peers, as a way to decrease the load on Bootstrap nodes.

View File

@ -9,14 +9,17 @@ const multiaddr = require('multiaddr')
const PeerId = require('peer-id')
const Book = require('./book')
const Protobuf = require('./pb/address-book.proto')
const {
ERR_INVALID_PARAMETERS
codes: { ERR_INVALID_PARAMETERS }
} = require('../errors')
/**
* The AddressBook is responsible for keeping the known multiaddrs
* of a peer.
* This data will be persisted in the PeerStore datastore as follows:
* /peers/addrs/<b32 peer id no padding>
*/
class AddressBook extends Book {
/**
@ -35,7 +38,20 @@ class AddressBook extends Book {
* "peer" - emitted when a peer is discovered by the node.
* "change:multiaddrs" - emitted when the known multiaddrs of a peer change.
*/
super(peerStore, 'change:multiaddrs', 'multiaddrs')
super({
peerStore,
eventName: 'change:multiaddrs',
eventProperty: 'multiaddrs',
protoBuf: Protobuf,
dsPrefix: '/peers/addrs/',
eventTransformer: (data) => data.map((address) => address.multiaddr),
dsSetTransformer: (data) => ({
addrs: data.map((address) => address.multiaddr.buffer)
}),
dsGetTransformer: (data) => data.addrs.map((a) => ({
multiaddr: multiaddr(a)
}))
})
/**
* Map known peers to their known Addresses.
@ -78,8 +94,7 @@ class AddressBook extends Book {
}
}
this.data.set(id, addresses)
this._setPeerId(peerId)
this._setData(peerId, addresses)
log(`stored provided multiaddrs for ${id}`)
// Notify the existance of a new peer
@ -87,11 +102,6 @@ class AddressBook extends Book {
this._ps.emit('peer', peerId)
}
this._ps.emit('change:multiaddrs', {
peerId,
multiaddrs: addresses.map((mi) => mi.multiaddr)
})
return this
}
@ -127,16 +137,9 @@ class AddressBook extends Book {
return this
}
this._setPeerId(peerId)
this.data.set(id, addresses)
this._setData(peerId, addresses)
log(`added provided multiaddrs for ${id}`)
this._ps.emit('change:multiaddrs', {
peerId,
multiaddrs: addresses.map((mi) => mi.multiaddr)
})
// Notify the existance of a new peer
if (!rec) {
this._ps.emit('peer', peerId)
@ -147,6 +150,7 @@ class AddressBook extends Book {
/**
* Transforms received multiaddrs into Address.
* @private
* @param {Array<Multiaddr>} multiaddrs
* @returns {Array<Address>}
*/

View File

@ -1,20 +1,57 @@
'use strict'
const errcode = require('err-code')
const debug = require('debug')
const log = debug('libp2p:peer-store:book')
log.error = debug('libp2p:peer-store:book:error')
const { Key } = require('interface-datastore')
const PeerId = require('peer-id')
const {
ERR_INVALID_PARAMETERS
codes: { ERR_INVALID_PARAMETERS }
} = require('../errors')
const passthrough = data => data
/**
* The Book is the skeleton for the PeerStore books.
* It handles the PeerStore persistence and events.
*/
class Book {
constructor (peerStore, eventName, eventProperty) {
/**
* @constructor
* @param {Object} properties
* @param {PeerStore} properties.peerStore PeerStore instance.
* @param {string} properties.eventName Name of the event to emit by the PeerStore.
* @param {string} properties.eventProperty Name of the property to emit by the PeerStore.
* @param {Object} properties.protoBuf Suffix of the Datastore Key
* @param {String} properties.dsPrefix Prefix of the Datastore Key
* @param {String} [properties.dsSuffix] Suffix of the Datastore Key
* @param {function} [properties.eventTransformer] Transformer function of the provided data for being emitted.
* @param {function} [properties.dsSetTransformer] Transformer function of the provided data for being persisted.
* @param {function} [properties.dsGetTransformer] Transformer function of the persisted data to be loaded.
*/
constructor ({
peerStore,
eventName,
eventProperty,
protoBuf,
dsPrefix,
dsSuffix = '',
eventTransformer = passthrough,
dsSetTransformer = passthrough,
dsGetTransformer = passthrough
}) {
this._ps = peerStore
this.eventName = eventName
this.eventProperty = eventProperty
this.protoBuf = protoBuf
this.dsPrefix = dsPrefix
this.dsSuffix = dsSuffix
this.eventTransformer = eventTransformer
this.dsSetTransformer = dsSetTransformer
this.dsGetTransformer = dsGetTransformer
/**
* Map known peers to their data.
@ -23,6 +60,91 @@ class Book {
this.data = new Map()
}
/**
* Load data from peerStore datastore into the books datastructures.
* This will not persist the replicated data nor emit modify events.
* @private
* @return {Promise<void>}
*/
async _loadData () {
if (!this._ps._datastore || !this._ps._enabledPersistance) {
return
}
const persistenceQuery = {
prefix: this.dsPrefix
}
for await (const { key, value } of this._ps._datastore.query(persistenceQuery)) {
try {
// PeerId to add to the book
const b32key = key.toString()
.replace(this.dsPrefix, '') // remove prefix from key
.replace(this.dsSuffix, '') // remove suffix from key
const peerId = PeerId.createFromCID(b32key)
// Data in the format to add to the book
const data = this.dsGetTransformer(this.protoBuf.decode(value))
// Add the book without persist the replicated data and emit modify
this._setData(peerId, data, {
persist: false,
emit: false
})
} catch (err) {
log.error(err)
}
}
}
/**
* Set data into the datastructure, persistence and emit it using the provided transformers.
* @private
* @param {PeerId} peerId peerId of the data to store
* @param {Array<*>} data data to store.
* @param {Object} [options] storing options.
* @param {boolean} [options.persist = true] persist the provided data.
* @param {boolean} [options.emit = true] emit the provided data.
* @return {Promise<void>}
*/
async _setData (peerId, data, { persist = true, emit = true } = {}) {
const b58key = peerId.toB58String()
// Store data in memory
this.data.set(b58key, data)
this._setPeerId(peerId)
// Emit event
emit && this._ps.emit(this.eventName, {
peerId,
[this.eventProperty]: this.eventTransformer(data)
})
// Add to Persistence datastore
persist && await this._persistData(peerId, data)
}
/**
* Persist data on the datastore
* @private
* @param {PeerId} peerId peerId of the data to persist
* @param {Array<*>} data data to persist
* @return {Promise<void>}
*/
async _persistData (peerId, data) {
if (!this._ps._datastore || !this._ps._enabledPersistance) {
return
}
const b32key = peerId.toString()
const k = `${this.dsPrefix}${b32key}${this.dsSuffix}`
try {
const value = this.protoBuf.encode(this.dsSetTransformer(data))
await this._ps._datastore.put(new Key(k), value)
} catch (err) {
log.error(err)
}
}
/**
* Set known data of a provided peer.
* @param {PeerId} peerId
@ -75,9 +197,17 @@ class Book {
[this.eventProperty]: []
})
// Update Persistence datastore
this._persistData(peerId, [])
return true
}
/**
* Set PeerId into peerStore datastructure.
* @private
* @param {PeerId} peerId
*/
_setPeerId (peerId) {
if (!this._ps.peerIds.get(peerId)) {
this._ps.peerIds.set(peerId.toB58String(), peerId)

View File

@ -30,11 +30,22 @@ class PeerStore extends EventEmitter {
* @property {Array<string>} protocols peer's supported protocols.
*/
constructor () {
/**
* @constructor
* @param {Object} properties
* @param {Datastore} [properties.datastore] Datastore to persist data.
* @param {boolean} [properties.persistance = true] Persist peerstore data.
*/
constructor ({ datastore, persistance = true } = {}) {
super()
/**
* AddressBook containing a map of peerIdStr to Address
* Backend datastore used to persist data.
*/
this._datastore = datastore
/**
* AddressBook containing a map of peerIdStr to Address.
*/
this.addressBook = new AddressBook(this)
@ -49,6 +60,18 @@ class PeerStore extends EventEmitter {
* @type {Map<string, Array<PeerId>}
*/
this.peerIds = new Map()
this._enabledPersistance = persistance
}
/**
* Load data from the datastore to populate the PeerStore.
*/
async load () {
if (this._enabledPersistance) {
await this.addressBook._loadData()
await this.protoBook._loadData()
}
}
/**

View File

@ -0,0 +1,12 @@
'use strict'
const protons = require('protons')
/* eslint-disable no-tabs */
const message = `
message Addresses {
repeated bytes addrs = 1;
}
`
module.exports = protons(message).Addresses

View File

@ -0,0 +1,12 @@
'use strict'
const protons = require('protons')
/* eslint-disable no-tabs */
const message = `
message Protocols {
repeated string protocols = 1;
}
`
module.exports = protons(message).Protocols

View File

@ -8,15 +8,17 @@ log.error = debug('libp2p:peer-store:proto-book:error')
const PeerId = require('peer-id')
const Book = require('./book')
const Protobuf = require('./pb/proto-book.proto')
const {
ERR_INVALID_PARAMETERS
codes: { ERR_INVALID_PARAMETERS }
} = require('../errors')
/**
* The ProtoBook is responsible for keeping the known supported
* protocols of a peer.
* @fires ProtoBook#change:protocols
* This data will be persisted in the PeerStore datastore as follows:
* /peers/protos/<b32 peer id no padding>
*/
class ProtoBook extends Book {
/**
@ -28,7 +30,18 @@ class ProtoBook extends Book {
* PeerStore Event emitter, used by the ProtoBook to emit:
* "change:protocols" - emitted when the known protocols of a peer change.
*/
super(peerStore, 'change:protocols', 'protocols')
super({
peerStore,
eventName: 'change:protocols',
eventProperty: 'protocols',
protoBuf: Protobuf,
dsPrefix: '/peers/protos/',
eventTransformer: (data) => Array.from(data),
dsSetTransformer: (data) => ({
protocols: Array.from(data)
}),
dsGetTransformer: (data) => new Set(data.protocols)
})
/**
* Map known peers to their known protocols.
@ -69,15 +82,9 @@ class ProtoBook extends Book {
return this
}
this.data.set(id, newSet)
this._setPeerId(peerId)
this._setData(peerId, newSet)
log(`stored provided protocols for ${id}`)
this._ps.emit('change:protocols', {
peerId,
protocols
})
return this
}
@ -110,17 +117,9 @@ class ProtoBook extends Book {
return this
}
protocols = [...newSet]
this.data.set(id, newSet)
this._setPeerId(peerId)
this._setData(peerId, newSet)
log(`added provided protocols for ${id}`)
this._ps.emit('change:protocols', {
peerId,
protocols
})
return this
}
}

View File

@ -12,7 +12,7 @@ const PeerStore = require('../../src/peer-store')
const peerUtils = require('../utils/creators/peer')
const {
ERR_INVALID_PARAMETERS
codes: { ERR_INVALID_PARAMETERS }
} = require('../../src/errors')
const addr1 = multiaddr('/ip4/127.0.0.1/tcp/8000')
@ -41,21 +41,33 @@ describe('addressBook', () => {
})
it('throwns invalid parameters error if invalid PeerId is provided', () => {
expect(() => {
try {
ab.set('invalid peerId')
}).to.throw(ERR_INVALID_PARAMETERS)
} catch (err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
}
throw new Error('invalid peerId should throw error')
})
it('throwns invalid parameters error if no addresses provided', () => {
expect(() => {
try {
ab.set(peerId)
}).to.throw(ERR_INVALID_PARAMETERS)
} catch (err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
}
throw new Error('no addresses should throw error')
})
it('throwns invalid parameters error if invalid multiaddrs are provided', () => {
expect(() => {
ab.set(peerId, 'invalid multiaddr')
}).to.throw(ERR_INVALID_PARAMETERS)
try {
ab.set(peerId, ['invalid multiaddr'])
} catch (err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
}
throw new Error('invalid multiaddrs should throw error')
})
it('replaces the stored content by default and emit change event', () => {
@ -143,21 +155,33 @@ describe('addressBook', () => {
})
it('throwns invalid parameters error if invalid PeerId is provided', () => {
expect(() => {
try {
ab.add('invalid peerId')
}).to.throw(ERR_INVALID_PARAMETERS)
} catch (err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
}
throw new Error('invalid peerId should throw error')
})
it('throwns invalid parameters error if no addresses provided', () => {
expect(() => {
try {
ab.add(peerId)
}).to.throw(ERR_INVALID_PARAMETERS)
} catch (err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
}
throw new Error('no addresses provided should throw error')
})
it('throwns invalid parameters error if invalid multiaddrs are provided', () => {
expect(() => {
ab.add(peerId, 'invalid multiaddr')
}).to.throw(ERR_INVALID_PARAMETERS)
try {
ab.add(peerId, ['invalid multiaddr'])
} catch (err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
}
throw new Error('invalid multiaddr should throw error')
})
it('adds the new content and emits change event', () => {
@ -255,9 +279,13 @@ describe('addressBook', () => {
})
it('throwns invalid parameters error if invalid PeerId is provided', () => {
expect(() => {
try {
ab.get('invalid peerId')
}).to.throw(ERR_INVALID_PARAMETERS)
} catch (err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
}
throw new Error('invalid peerId should throw error')
})
it('returns undefined if no multiaddrs are known for the provided peer', () => {
@ -286,9 +314,13 @@ describe('addressBook', () => {
})
it('throwns invalid parameters error if invalid PeerId is provided', () => {
expect(() => {
try {
ab.getMultiaddrsForPeer('invalid peerId')
}).to.throw(ERR_INVALID_PARAMETERS)
} catch (err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
}
throw new Error('invalid peerId should throw error')
})
it('returns undefined if no multiaddrs are known for the provided peer', () => {
@ -318,9 +350,13 @@ describe('addressBook', () => {
})
it('throwns invalid parameters error if invalid PeerId is provided', () => {
expect(() => {
try {
ab.delete('invalid peerId')
}).to.throw(ERR_INVALID_PARAMETERS)
} catch (err) {
expect(err.code).to.equal(ERR_INVALID_PARAMETERS)
return
}
throw new Error('invalid peerId should throw error')
})
it('returns false if no records exist for the peer and no event is emitted', () => {

View File

@ -4,9 +4,11 @@
const chai = require('chai')
chai.use(require('dirty-chai'))
const { expect } = chai
const sinon = require('sinon')
const PeerStore = require('../../src/peer-store')
const multiaddr = require('multiaddr')
const { MemoryDatastore } = require('interface-datastore')
const peerUtils = require('../utils/creators/peer')
@ -150,3 +152,157 @@ describe('peer-store', () => {
})
})
})
describe('libp2p.peerStore', () => {
let libp2p
let memoryDatastore
beforeEach(async () => {
memoryDatastore = new MemoryDatastore()
;[libp2p] = await peerUtils.createPeer({
started: false,
config: {
datastore: memoryDatastore,
peerStore: {
persistance: true
}
}
})
})
it('should try to load content from an empty datastore', async () => {
const spyPeerStore = sinon.spy(libp2p.peerStore, 'load')
const spyDs = sinon.spy(memoryDatastore, 'query')
await libp2p.start()
expect(spyPeerStore).to.have.property('callCount', 1)
// Should be called for AddressBook and ProtoBook
expect(spyDs).to.have.property('callCount', 2)
// No data to populate
expect(libp2p.peerStore.peers.size).to.eq(0)
})
it('should store peerStore content on datastore', async () => {
const [peer] = await peerUtils.createPeerId({ number: 2 })
const multiaddrs = [multiaddr('/ip4/156.10.1.22/tcp/1000')]
const protocols = ['/ping/1.0.0']
const spyDs = sinon.spy(memoryDatastore, 'put')
await libp2p.start()
// AddressBook
await libp2p.peerStore.addressBook.set(peer, multiaddrs)
expect(spyDs).to.have.property('callCount', 1)
// ProtoBook
await libp2p.peerStore.protoBook.set(peer, protocols)
expect(spyDs).to.have.property('callCount', 2)
// Should have two peer records stored in the datastore
const queryParams = {
prefix: '/peers/'
}
let count = 0
for await (const _ of memoryDatastore.query(queryParams)) { // eslint-disable-line
count++
}
expect(count).to.equal(2)
})
it('should load content to the peerStore when restart but not put in datastore again', async () => {
const spyDs = sinon.spy(memoryDatastore, 'put')
const peers = await peerUtils.createPeerId({ number: 2 })
const multiaddrs = [
multiaddr('/ip4/156.10.1.22/tcp/1000'),
multiaddr('/ip4/156.10.1.23/tcp/1000')
]
const protocols = ['/ping/1.0.0']
await libp2p.start()
// AddressBook
await libp2p.peerStore.addressBook.set(peers[0], [multiaddrs[0]])
await libp2p.peerStore.addressBook.set(peers[1], [multiaddrs[1]])
// ProtoBook
await libp2p.peerStore.protoBook.set(peers[0], protocols)
await libp2p.peerStore.protoBook.set(peers[1], protocols)
expect(spyDs).to.have.property('callCount', 4)
expect(libp2p.peerStore.peers.size).to.equal(2)
await libp2p.stop()
// Load on restart
const spyAb = sinon.spy(libp2p.peerStore.addressBook, '_loadData')
const spyPb = sinon.spy(libp2p.peerStore.protoBook, '_loadData')
await libp2p.start()
expect(spyAb).to.have.property('callCount', 1)
expect(spyPb).to.have.property('callCount', 1)
expect(spyDs).to.have.property('callCount', 4)
expect(libp2p.peerStore.peers.size).to.equal(2)
})
it('should load content to the peerStore when a new node is started with the same datastore', async () => {
const peers = await peerUtils.createPeerId({ number: 2 })
const multiaddrs = [
multiaddr('/ip4/156.10.1.22/tcp/1000'),
multiaddr('/ip4/156.10.1.23/tcp/1000')
]
const protocols = ['/ping/1.0.0']
await libp2p.start()
// AddressBook
await libp2p.peerStore.addressBook.set(peers[0], [multiaddrs[0]])
await libp2p.peerStore.addressBook.set(peers[1], [multiaddrs[1]])
// ProtoBook
await libp2p.peerStore.protoBook.set(peers[0], protocols)
await libp2p.peerStore.protoBook.set(peers[1], protocols)
expect(libp2p.peerStore.peers.size).to.equal(2)
await libp2p.stop()
// Use a new node with the previously populated datastore
const [newNode] = await peerUtils.createPeer({
started: false,
config: {
datastore: memoryDatastore,
peerStore: {
persistance: true
}
}
})
expect(newNode.peerStore.peers.size).to.equal(0)
const spyAb = sinon.spy(newNode.peerStore.addressBook, '_loadData')
const spyPb = sinon.spy(newNode.peerStore.protoBook, '_loadData')
await newNode.start()
expect(spyAb).to.have.property('callCount', 1)
expect(spyPb).to.have.property('callCount', 1)
expect(newNode.peerStore.peers.size).to.equal(2)
// Validate data
const peer0 = newNode.peerStore.get(peers[0])
expect(peer0.id.toB58String()).to.eql(peers[0].toB58String())
expect(peer0.protocols).to.have.members(protocols)
expect(peer0.addresses.map((a) => a.multiaddr.toString())).to.have.members([multiaddrs[0].toString()])
const peer1 = newNode.peerStore.get(peers[1])
expect(peer1.id.toB58String()).to.eql(peers[1].toB58String())
expect(peer1.protocols).to.have.members(protocols)
expect(peer1.addresses.map((a) => a.multiaddr.toString())).to.have.members([multiaddrs[1].toString()])
await newNode.stop()
})
})

View File

@ -28,6 +28,9 @@ async function createPeer ({ number = 1, fixture = true, started = true, populat
const peers = await pTimes(number, (i) => Libp2p.create({
peerId: peerIds[i],
addresses,
peerStore: {
persistence: false
},
...defaultOptions,
...config
}))