feat: enable and test Circuit Relay

* feat: new super simplified API

* feat: append peer id to multiaddr if not there

* [WIP] Awesome DHT (#86)

* feat: integrate dht

* better interfaces

* docs: add documentation for peerRouting, contentRouting, dht

* fix: take in passed datastore

* fix: update usage of _getPeerInfo

* fix: getPeerInfo

* docs: update docs

* moar

feat: correctly handle p2p-circuit addrs when creating a peer info object

refactor: rework config options

* feat: adding circuit relaying

* feat: rework circuit relay for protobufs

* feat: circuit loading and tests

* fix: clean up _getPeerInfo to work with /p2p-circuit

* wip: tests cleaup

* test: clean up

* wip

* fix: bringing back test reworks and new aegir

* test: group tests

* test: clean up

* test: adjust test

* fix: use getPeerId to determine if the ipfs fragment is missing

* feat: adding circuit relaying

* feat: circuit loading and tests

* test: clean up

* wip

* feat: upgrade to latest aegir

* fix: removing unused tests

* feat: cleanup tests

* fix: create node defautl options

* chore: upgrade swarm to latest version

* fix: updated aegir and adjust timeouts

* feat: more timeouts

* chore: updating deps

* fix: circle ci builds

* test: timeouts

.aegir.js

@@ -1,6 +1,5 @@
 'use strict'
 
-const gulp = require('gulp')
 const Node = require('./test/nodejs-bundle/nodejs-bundle.js')
 const PeerInfo = require('peer-info')
 const PeerId = require('peer-id')
@@ -12,7 +11,7 @@ let server
 let node
 const rawPeer = require('./test/browser-bundle/peer.json')
 
-gulp.task('libnode:start', (done) => {
+const before = (done) => {
   let count = 0
   const ready = () => ++count === 2 ? done() : null
 
@@ -36,20 +35,21 @@ gulp.task('libnode:start', (done) => {
     node.handle('/echo/1.0.0', (protocol, conn) => pull(conn, conn))
     node.start(() => ready())
   })
-})
+}
 
-gulp.task('libnode:stop', (done) => {
+const after = (done) => {
   setTimeout(() => node.stop((err) => {
     if (err) {
       return done(err)
     }
     server.stop(done)
   }), 2000)
-})
+}
 
-gulp.task('test:browser:before', ['libnode:start'])
+module.exports = {
-gulp.task('test:node:before', ['libnode:start'])
+  hooks: {
-gulp.task('test:browser:after', ['libnode:stop'])
+    pre: before,
-gulp.task('test:node:after', ['libnode:stop'])
+    post: after
+  }
+}
 
-require('aegir/gulp')(gulp)

circle.yml

@@ -6,9 +6,13 @@ dependencies:
   pre:
     - google-chrome --version
     - curl -L -o google-chrome.deb https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+    - for v in $(curl http://archive.ubuntu.com/ubuntu/pool/main/n/nss/ | grep "href=" | grep "libnss3.*deb\"" -o | grep -o "libnss3.*deb" | grep "3.28" | grep "14.04"); do curl -L -o $v http://archive.ubuntu.com/ubuntu/pool/main/n/nss/$v; done && rm libnss3-tools*_i386.deb libnss3-dev*_i386.deb
     - sudo dpkg -i google-chrome.deb || true
+    - sudo dpkg -i libnss3*.deb || true
     - sudo apt-get update
+    - sudo apt-get install -f || true
+    - sudo dpkg -i libnss3*.deb
     - sudo apt-get install -f
     - sudo apt-get install --only-upgrade lsb-base
     - sudo dpkg -i google-chrome.deb
-    - google-chrome --version
+    - google-chrome --version

examples/protocol-and-stream-muxing/3.js

@@ -74,10 +74,10 @@ parallel([
     if (err) { throw err }
     console.log('Addresses by which both peers are connected')
     node1.peerBook
-         .getAllArray()
+      .getAllArray()
-         .forEach((peer) => console.log('node 1 to node 2:', peer.isConnected().toString()))
+      .forEach((peer) => console.log('node 1 to node 2:', peer.isConnected().toString()))
     node2.peerBook
-         .getAllArray()
+      .getAllArray()
-         .forEach((peer) => console.log('node 2 to node 1:', peer.isConnected().toString()))
+      .forEach((peer) => console.log('node 2 to node 1:', peer.isConnected().toString()))
   })
 })

package.json

@@ -4,16 +4,16 @@
   "description": "JavaScript base class for libp2p bundles",
   "main": "src/index.js",
   "scripts": {
-    "test": "gulp test",
+    "lint": "aegir lint",
-    "test:node": "gulp test:node",
+    "build": "aegir build",
-    "test:browser": "gulp test:browser --dom",
+    "test": "aegir test --target node --target browser --no-parallel",
-    "release": "gulp release --dom",
+    "test:node": "aegir test --target node --no-parallel",
-    "release-minor": "gulp release --type minor --dom",
+    "test:browser": "aegir test --target browser --no-parallel",
-    "release-major": "gulp release --type major --dom",
+    "release": "aegir test release --target node --target browser --no-parallel",
-    "build": "gulp build",
+    "release-minor": "aegir release --type minor --target node --target browser --no-parallel",
-    "lint": "aegir-lint",
+    "release-major": "aegir release --type major --target node --target browser --no-parallel",
-    "coverage": "aegir-coverage",
+    "coverage": "aegir coverage",
-    "coverage-publish": "aegir-coverage publish"
+    "coverage-publish": "aegir coverage --provider coveralls"
   },
   "repository": {
     "type": "git",
@@ -39,39 +39,40 @@
   "dependencies": {
     "async": "^2.5.0",
     "libp2p-ping": "~0.6.0",
-    "libp2p-swarm": "~0.32.4",
+    "libp2p-swarm": "~0.33.0",
-    "mafmt": "^3.0.1",
+    "mafmt": "^3.0.2",
-    "multiaddr": "^3.0.1",
+    "multiaddr": "^3.0.0",
-    "peer-book": "~0.5.1",
+    "peer-book": "~0.5.0",
-    "peer-id": "~0.10.1",
+    "peer-id": "~0.10.0",
     "peer-info": "~0.11.0"
   },
   "devDependencies": {
-    "aegir": "^11.0.2",
+    "aegir": "^12.1.0",
     "chai": "^4.1.2",
-    "dirty-chai": "^2.0.1",
     "cids": "~0.5.1",
-    "libp2p-kad-dht": "~0.5.1",
+    "dirty-chai": "^2.0.1",
-    "libp2p-mdns": "~0.9.1",
+    "electron-webrtc": "^0.3.0",
+    "libp2p-circuit": "^0.1.2",
+    "libp2p-kad-dht": "~0.5.0",
+    "libp2p-mdns": "~0.9.0",
     "libp2p-multiplex": "~0.5.0",
-    "libp2p-railing": "~0.7.1",
+    "libp2p-railing": "~0.7.0",
-    "libp2p-secio": "~0.8.1",
+    "libp2p-secio": "~0.8.0",
     "libp2p-spdy": "~0.11.0",
-    "libp2p-tcp": "~0.11.0",
+    "libp2p-tcp": "^0.11.0",
-    "libp2p-webrtc-star": "~0.13.2",
+    "libp2p-webrtc-star": "^0.13.2",
-    "libp2p-websockets": "~0.10.1",
+    "libp2p-websockets": "^0.10.4",
     "lodash.times": "^4.3.2",
     "pre-commit": "^1.2.2",
     "pull-goodbye": "0.0.2",
     "pull-serializer": "^0.3.2",
-    "pull-stream": "^3.6.1",
+    "pull-stream": "^3.6.0",
     "safe-buffer": "^5.1.1",
-    "electron-webrtc": "^0.3.0",
+    "sinon": "^2.3.6",
     "wrtc": "0.0.62"
   },
   "contributors": [
     "Chris Bratlien <chrisbratlien@gmail.com>",
-    "Daijiro Wachi <daijiro.wachi@gmail.com>",
     "David Dias <daviddias.p@gmail.com>",
     "Elven <mon.samuel@qq.com>",
     "Friedel Ziegelmayer <dignifiedquire@gmail.com>",

src/index.js

@@ -12,7 +12,6 @@ const Swarm = require('libp2p-swarm')
 const PeerId = require('peer-id')
 const PeerInfo = require('peer-info')
 const PeerBook = require('peer-book')
-const mafmt = require('mafmt')
 const multiaddr = require('multiaddr')
 
 exports = module.exports
@@ -43,6 +42,9 @@ class Node extends EventEmitter {
       // If muxer exists, we can use Identify
       this.swarm.connection.reuse()
 
+      // If muxer exists, we can use Relay for listening/dialing
+      this.swarm.connection.enableCircuitRelay(_options.relay)
+
       // Received incommind dial and muxer upgrade happened,
       // reuse this muxed connection
       this.swarm.on('peer-mux-established', (peerInfo) => {
@@ -155,7 +157,7 @@ class Node extends EventEmitter {
     const maOld = []
     const maNew = []
     this.peerInfo.multiaddrs.forEach((ma) => {
-      if (!mafmt.IPFS.matches(ma)) {
+      if (!ma.getPeerId()) {
         maOld.push(ma)
         maNew.push(ma.encapsulate('/ipfs/' + this.peerInfo.id.toB58String()))
       }

test/node.js

@@ -8,3 +8,4 @@ require('./nodejs-bundle/stream-muxing')
 require('./nodejs-bundle/discovery')
 require('./nodejs-bundle/peer-routing')
 require('./nodejs-bundle/content-routing')
+require('./nodejs-bundle/circuit')

test/nodejs-bundle/circuit.js

@@ -0,0 +1,223 @@
+/* eslint-env mocha */
+'use strict'
+
+const pull = require('pull-stream')
+const waterfall = require('async/waterfall')
+const series = require('async/series')
+const parallel = require('async/parallel')
+const utils = require('./utils')
+const Circuit = require('libp2p-circuit')
+const multiaddr = require('multiaddr')
+
+const chai = require('chai')
+chai.use(require('dirty-chai'))
+
+const expect = chai.expect
+const sinon = require('sinon')
+
+describe(`circuit`, function () {
+  let handlerSpies = []
+  let relayNode1
+  let relayNode2
+  let nodeWS1
+  let nodeWS2
+  let nodeTCP1
+  let nodeTCP2
+
+  function setupNode (addrs, options, cb) {
+    if (typeof options === 'function') {
+      cb = options
+      options = {}
+    }
+
+    options = options || {}
+
+    return utils.createNode(addrs, options, (err, node) => {
+      expect(err).to.not.exist()
+
+      node.handle('/echo/1.0.0', utils.echo)
+      node.start((err) => {
+        expect(err).to.not.exist()
+
+        handlerSpies.push(sinon.spy(node.swarm.transports[Circuit.tag].listeners[0].hopHandler, 'handle'))
+        cb(node)
+      })
+    })
+  }
+
+  before(function (done) {
+    this.timeout(20000)
+
+    waterfall([
+      // set up passive relay
+      (cb) => setupNode([
+        `/ip4/0.0.0.0/tcp/0/ws`,
+        `/ip4/0.0.0.0/tcp/0`
+      ], {
+        relay: {
+          enabled: true,
+          hop: {
+            enabled: true,
+            active: false // passive relay
+          }
+        }
+      }, (node) => {
+        relayNode1 = node
+        cb()
+      }),
+      // setup active relay
+      (cb) => setupNode([
+        `/ip4/0.0.0.0/tcp/0/ws`,
+        `/ip4/0.0.0.0/tcp/0`
+      ], {
+        relay: {
+          enabled: true,
+          hop: {
+            enabled: true,
+            active: false // passive relay
+          }
+        }
+      }, (node) => {
+        relayNode2 = node
+        cb()
+      }),
+      // setup node with WS
+      (cb) => setupNode([
+        `/ip4/0.0.0.0/tcp/0/ws`
+      ], {
+        relay: {
+          enabled: true
+        }
+      }, (node) => {
+        nodeWS1 = node
+        cb()
+      }),
+      // setup node with WS
+      (cb) => setupNode([
+        `/ip4/0.0.0.0/tcp/0/ws`
+      ], {
+        relay: {
+          enabled: true
+        }
+      }, (node) => {
+        nodeWS2 = node
+        cb()
+      }),
+      // set up node with TCP and listening on relay1
+      (cb) => setupNode([
+        `/ip4/0.0.0.0/tcp/0`,
+        `/ipfs/${relayNode1.peerInfo.id.toB58String()}/p2p-circuit`
+      ], {
+        relay: {
+          enabled: true
+        }
+      }, (node) => {
+        nodeTCP1 = node
+        cb()
+      }),
+      // set up node with TCP and listening on relay2 over TCP transport
+      (cb) => setupNode([
+        `/ip4/0.0.0.0/tcp/0`,
+        `/ip4/0.0.0.0/tcp/0/ipfs/${relayNode2.peerInfo.id.toB58String()}/p2p-circuit`
+      ], {
+        relay: {
+          enabled: true
+        }
+      }, (node) => {
+        nodeTCP2 = node
+        cb()
+      })
+    ], (err) => {
+      expect(err).to.not.exist()
+
+      series([
+        (cb) => nodeWS1.dial(relayNode1.peerInfo, cb),
+        (cb) => nodeWS1.dial(relayNode2.peerInfo, cb),
+        (cb) => nodeTCP1.dial(relayNode1.peerInfo, cb),
+        (cb) => nodeTCP2.dial(relayNode2.peerInfo, cb)
+      ], done)
+    })
+  })
+
+  after((done) => {
+    parallel([
+      (cb) => relayNode1.stop(cb),
+      (cb) => relayNode2.stop(cb),
+      (cb) => nodeWS1.stop(cb),
+      (cb) => nodeWS2.stop(cb),
+      (cb) => nodeTCP1.stop(cb),
+      (cb) => nodeTCP2.stop(cb)
+    ], done)
+  })
+
+  describe(`any relay`, function () {
+    this.timeout(20000)
+    it('should dial from WS1 to TCP1 over any R', function (done) {
+      nodeWS1.dial(nodeTCP1.peerInfo, '/echo/1.0.0', (err, conn) => {
+        expect(err).to.not.exist()
+        expect(conn).to.exist()
+
+        pull(
+          pull.values(['hello']),
+          conn,
+          pull.collect((e, result) => {
+            expect(e).to.not.exist()
+            expect(result[0].toString()).to.equal('hello')
+            done()
+          })
+        )
+      })
+    })
+
+    it(`should not dial - no R from WS2 to TCP1`, function (done) {
+      nodeWS2.dial(nodeTCP2.peerInfo, '/echo/1.0.0', (err, conn) => {
+        expect(err).to.exist()
+        expect(conn).to.not.exist()
+        done()
+      })
+    })
+  })
+
+  describe(`explicit relay`, function () {
+    this.timeout(20000)
+    it('should dial from WS1 to TCP1 over R1', function (done) {
+      nodeWS1.dial(nodeTCP1.peerInfo, '/echo/1.0.0', (err, conn) => {
+        expect(err).to.not.exist()
+        expect(conn).to.exist()
+
+        pull(
+          pull.values(['hello']),
+          conn,
+          pull.collect((e, result) => {
+            expect(e).to.not.exist()
+            expect(result[0].toString()).to.equal('hello')
+
+            const addr = multiaddr(handlerSpies[0].args[2][0].dstPeer.addrs[0]).toString()
+            expect(addr).to.equal(`/ipfs/${nodeTCP1.peerInfo.id.toB58String()}`)
+            done()
+          })
+        )
+      })
+    })
+
+    it(`should dial from WS1 to TCP2 over R2`, function (done) {
+      nodeWS1.dial(nodeTCP2.peerInfo, '/echo/1.0.0', (err, conn) => {
+        expect(err).to.not.exist()
+        expect(conn).to.exist()
+
+        pull(
+          pull.values(['hello']),
+          conn,
+          pull.collect((e, result) => {
+            expect(e).to.not.exist()
+            expect(result[0].toString()).to.equal('hello')
+
+            const addr = multiaddr(handlerSpies[1].args[2][0].dstPeer.addrs[0]).toString()
+            expect(addr).to.equal(`/ipfs/${nodeTCP2.peerInfo.id.toB58String()}`)
+            done()
+          })
+        )
+      })
+    })
+  })
+})

test/nodejs-bundle/content-routing.js

@@ -19,7 +19,8 @@ describe('.contentRouting', () => {
   let nodeD
   let nodeE
 
-  before((done) => {
+  before(function (done) {
+    this.timeout(5000)
     const tasks = _times(5, () => (cb) => {
       createNode('/ip4/0.0.0.0/tcp/0', {
         mdns: false,

test/nodejs-bundle/discovery.js

@@ -60,7 +60,8 @@ describe('discovery', () => {
   describe('MulticastDNS', () => {
     setup({ mdns: true })
 
-    it('find a peer', (done) => {
+    it('find a peer', function (done) {
+      this.timeout(15000)
       nodeA.once('peer:discovery', (peerInfo) => {
         expect(nodeB.peerInfo.id.toB58String())
           .to.eql(peerInfo.id.toB58String())
@@ -73,7 +74,8 @@ describe('discovery', () => {
   describe.skip('WebRTCStar', () => {
     setup({ webRTCStar: true })
 
-    it('find a peer', (done) => {
+    it('find a peer', function (done) {
+      this.timeout(15000)
       nodeA.once('peer:discovery', (peerInfo) => {
         expect(nodeB.peerInfo.id.toB58String())
           .to.eql(peerInfo.id.toB58String())
@@ -88,7 +90,8 @@ describe('discovery', () => {
       mdns: true
     })
 
-    it('find a peer', (done) => {
+    it('find a peer', function (done) {
+      this.timeout(15000)
       nodeA.once('peer:discovery', (peerInfo) => {
         expect(nodeB.peerInfo.id.toB58String())
           .to.eql(peerInfo.id.toB58String())

test/nodejs-bundle/peer-routing.js

@@ -18,7 +18,8 @@ describe('.peerRouting', () => {
   let nodeD
   let nodeE
 
-  before((done) => {
+  before(function (done) {
+    this.timeout(5000)
     const tasks = _times(5, () => (cb) => {
       createNode('/ip4/0.0.0.0/tcp/0', {
         mdns: false,

test/nodejs-bundle/stream-muxing.js

@@ -16,11 +16,11 @@ function test (nodeA, nodeB, callback) {
     expect(err).to.not.exist()
 
     pull(
-      pull.values([new Buffer('hey')]),
+      pull.values([Buffer.from('hey')]),
       conn,
       pull.collect((err, data) => {
         expect(err).to.not.exist()
-        expect(data).to.be.eql([new Buffer('hey')])
+        expect(data).to.be.eql([Buffer.from('hey')])
         callback()
       })
     )
@@ -34,8 +34,10 @@ function teardown (nodeA, nodeB, callback) {
   ], callback)
 }
 
-describe('stream muxing', (done) => {
+describe('stream muxing', () => {
-  it('spdy only', (done) => {
+  it('spdy only', function (done) {
+    this.timeout(5000)
+
     let nodeA
     let nodeB
 
@@ -99,7 +101,9 @@ describe('stream muxing', (done) => {
     ], done)
   })
 
-  it('spdy + multiplex', (done) => {
+  it('spdy + multiplex', function (done) {
+    this.timeout(5000)
+
     let nodeA
     let nodeB
 
@@ -131,7 +135,9 @@ describe('stream muxing', (done) => {
     ], done)
   })
 
-  it('spdy + multiplex switched order', (done) => {
+  it('spdy + multiplex switched order', function (done) {
+    this.timeout(5000)
+
     let nodeA
     let nodeB
 
@@ -163,7 +169,9 @@ describe('stream muxing', (done) => {
     ], done)
   })
 
-  it('one without the other fails to establish a muxedConn', (done) => {
+  it('one without the other fails to establish a muxedConn', function (done) {
+    this.timeout(5000)
+
     let nodeA
     let nodeB
 

test/nodejs-bundle/tcp+websockets+webrtc-star.js

@@ -20,7 +20,8 @@ describe('TCP + WebSockets + WebRTCStar', () => {
 
   let ss
 
-  before((done) => {
+  before(function (done) {
+    this.timeout(5000)
     parallel([
       (cb) => {
         signalling.start({ port: 24642 }, (err, server) => {
@@ -194,7 +195,8 @@ describe('TCP + WebSockets + WebRTCStar', () => {
     })
   })
 
-  it('nodeAll.dial nodeWStar using PeerInfo', (done) => {
+  it('nodeAll.dial nodeWStar using PeerInfo', function (done) {
+    this.timeout(10000)
     nodeAll.dial(nodeWStar.peerInfo, (err) => {
       expect(err).to.not.exist()
 

test/nodejs-bundle/tcp.js

@@ -70,11 +70,11 @@ describe('TCP only', () => {
       expect(err).to.not.exist()
 
       pull(
-        pull.values([new Buffer('hey')]),
+        pull.values([Buffer.from('hey')]),
         conn,
         pull.collect((err, data) => {
           expect(err).to.not.exist()
-          expect(data).to.be.eql([new Buffer('hey')])
+          expect(data).to.be.eql([Buffer.from('hey')])
           done()
         })
       )
@@ -130,11 +130,11 @@ describe('TCP only', () => {
           }
         ], () => {
           pull(
-            pull.values([new Buffer('hey')]),
+            pull.values([Buffer.from('hey')]),
             conn,
             pull.collect((err, data) => {
               expect(err).to.not.exist()
-              expect(data).to.be.eql([new Buffer('hey')])
+              expect(data).to.be.eql([Buffer.from('hey')])
               done()
             })
           )
@@ -193,11 +193,11 @@ describe('TCP only', () => {
           }
         ], () => {
           pull(
-            pull.values([new Buffer('hey')]),
+            pull.values([Buffer.from('hey')]),
             conn,
             pull.collect((err, data) => {
               expect(err).to.not.exist()
-              expect(data).to.be.eql([new Buffer('hey')])
+              expect(data).to.be.eql([Buffer.from('hey')])
               done()
             })
           )

test/nodejs-bundle/turbolence.js

@@ -57,11 +57,11 @@ describe('Turbolence tests', () => {
       expect(Object.keys(peers)).to.have.length(1)
 
       pull(
-        pull.values([new Buffer('hey')]),
+        pull.values([Buffer.from('hey')]),
         conn,
         pull.collect((err, data) => {
           expect(err).to.not.exist()
-          expect(data).to.eql([new Buffer('hey')])
+          expect(data).to.eql([Buffer.from('hey')])
           done()
         })
       )

test/nodejs-bundle/utils.js

@@ -15,6 +15,8 @@ function createNode (multiaddrs, options, callback) {
     options = {}
   }
 
+  options = options || {}
+
   if (!Array.isArray(multiaddrs)) {
     multiaddrs = [multiaddrs]
   }