refactor: add core modules to libp2p (#400)

* refactor: add js-libp2p-connection-manager to repo

Co-authored-by: David Dias <daviddias.p@gmail.com>
Co-authored-by: Jacob Heun <jacobheun@gmail.com>
Co-authored-by: Pedro Teixeira <i@pgte.me>
Co-authored-by: Vasco Santos <vasco.santos@ua.pt>

* test(conn-mgr): only run in node

* refactor: add js-libp2p-identify to repo

Co-authored-by: David Dias <daviddias.p@gmail.com>
Co-authored-by: Friedel Ziegelmayer <dignifiedquire@gmail.com>
Co-authored-by: Hugo Dias <hugomrdias@gmail.com>
Co-authored-by: Jacob Heun <jacobheun@gmail.com>
Co-authored-by: Maciej Krüger <mkg20001@gmail.com>
Co-authored-by: Richard Littauer <richard.littauer@gmail.com>
Co-authored-by: Vasco Santos <vasco.santos@moxy.studio>
Co-authored-by: Yusef Napora <yusef@protocol.ai>
Co-authored-by: ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ <victorbjelkholm@gmail.com>

* refactor: add libp2p-pnet to repo

Co-authored-by: Jacob Heun <jacobheun@gmail.com>
Co-authored-by: Vasco Santos <vasco.santos@moxy.studio>

* refactor: add libp2p-ping to repo

Co-authored-by: David Dias <daviddias.p@gmail.com>
Co-authored-by: Francisco Baio Dias <xicombd@gmail.com>
Co-authored-by: Friedel Ziegelmayer <dignifiedquire@gmail.com>
Co-authored-by: Hugo Dias <mail@hugodias.me>
Co-authored-by: Jacob Heun <jacobheun@gmail.com>
Co-authored-by: João Antunes <j.goncalo.antunes@gmail.com>
Co-authored-by: Richard Littauer <richard.littauer@gmail.com>
Co-authored-by: Vasco Santos <vasco.santos@moxy.studio>
Co-authored-by: Vasco Santos <vasco.santos@ua.pt>
Co-authored-by: ᴠɪᴄᴛᴏʀ ʙᴊᴇʟᴋʜᴏʟᴍ <victorbjelkholm@gmail.com>

* refactor: add libp2p-circuit to repo

Co-authored-by: David Dias <daviddias.p@gmail.com>
Co-authored-by: Dmitriy Ryajov <dryajov@gmail.com>
Co-authored-by: Friedel Ziegelmayer <dignifiedquire@gmail.com>
Co-authored-by: Hugo Dias <mail@hugodias.me>
Co-authored-by: Jacob Heun <jacobheun@gmail.com>
Co-authored-by: Maciej Krüger <mkg20001@gmail.com>
Co-authored-by: Oli Evans <oli@tableflip.io>
Co-authored-by: Pedro Teixeira <i@pgte.me>
Co-authored-by: Vasco Santos <vasco.santos@ua.pt>
Co-authored-by: Victor Bjelkholm <victorbjelkholm@gmail.com>
Co-authored-by: Yusef Napora <yusef@napora.org>
Co-authored-by: dirkmc <dirk@mccormick.cx>

* test(switch): avoid using instanceof

* chore(switch): update bignumber dep

* refactor(circuit): clean up tests

* refactor(switch): consolidate get peer utils

* test(identify): do deep checks of addresses

* test(identify): bump timeout for identify test

* test(switch): tidy up limit dialer test

* refactor(switch): remove redundant circuit tests

* chore: add coverage script

* refactor(circuit): consolidate get peer info

* docs: reference original repositories in each sub readme

* docs: fix comment

* refactor: clean up sub package.json files and readmes

examples/pnet-ipfs/.gitignore

@@ -0,0 +1 @@
+tmp/

examples/pnet-ipfs/README.md

@@ -0,0 +1,29 @@
+# Private Networking with IPFS
+This example shows how to set up a private network of IPFS nodes.
+
+## Setup
+Install dependencies:
+
+```
+npm install
+```
+
+## Run
+Running the example will cause two nodes with the same swarm key to be started and exchange basic information.
+
+```
+node index.js
+```
+
+### Using different keys
+This example includes `TASK` comments that can be used to try the example with different swarm keys. This will
+allow you to see how nodes will fail to connect if they are on different private networks and try to connect to
+one another.
+
+To change the swarm key of one of the nodes, look through `index.js` for comments starting with `TASK` to indicate
+where lines are that pertain to changing the swarm key of node 2.
+
+### Exploring the repos
+Once you've run the example you can take a look at the repos in the `./tmp` directory to see how they differ, including
+the swarm keys. You should see a `swarm.key` file in each of the repos and when the nodes are on the same private network
+this contents of the `swarm.key` files should be the same.

examples/pnet-ipfs/index.js

@@ -0,0 +1,145 @@
+/* eslint no-console: ["off"] */
+'use strict'
+
+const IPFS = require('ipfs')
+const assert = require('assert').strict
+const writeKey = require('libp2p-pnet').generate
+const path = require('path')
+const fs = require('fs')
+const privateLibp2pBundle = require('./libp2p-bundle')
+const { mkdirp } = require('./utils')
+
+// Create two separate repo paths so we can run two nodes and check their output
+const repo1 = path.resolve('./tmp', 'repo1', '.ipfs')
+const repo2 = path.resolve('./tmp', 'repo2', '.ipfs')
+mkdirp(repo1)
+mkdirp(repo2)
+
+// Create a buffer and write the swarm key to it
+const swarmKey = Buffer.alloc(95)
+writeKey(swarmKey)
+
+// This key is for the `TASK` mentioned in the writeFileSync calls below
+const otherSwarmKey = Buffer.alloc(95)
+writeKey(otherSwarmKey)
+
+// Add the swarm key to both repos
+const swarmKey1Path = path.resolve(repo1, 'swarm.key')
+const swarmKey2Path = path.resolve(repo2, 'swarm.key')
+fs.writeFileSync(swarmKey1Path, swarmKey)
+// TASK: switch the commented out line below so we're using a different key, to see the nodes fail to connect
+fs.writeFileSync(swarmKey2Path, swarmKey)
+// fs.writeFileSync(swarmKey2Path, otherSwarmKey)
+
+// Create the first ipfs node
+const node1 = new IPFS({
+  repo: repo1,
+  libp2p: privateLibp2pBundle(swarmKey1Path),
+  config: {
+    Addresses: {
+      // Set the swarm address so we dont get port collision on the nodes
+      Swarm: ['/ip4/0.0.0.0/tcp/9101']
+    }
+  }
+})
+
+// Create the second ipfs node
+const node2 = new IPFS({
+  repo: repo2,
+  libp2p: privateLibp2pBundle(swarmKey2Path),
+  config: {
+    Addresses: {
+      // Set the swarm address so we dont get port collision on the nodes
+      Swarm: ['/ip4/0.0.0.0/tcp/9102']
+    }
+  }
+})
+
+console.log('auto starting the nodes...')
+
+// `nodesStarted` keeps track of how many of our nodes have started
+let nodesStarted = 0
+/**
+ * Calls `connectAndTalk` when both nodes have started
+ * @returns {void}
+ */
+const didStartHandler = () => {
+  if (++nodesStarted === 2) {
+    // If both nodes are up, start talking
+    connectAndTalk()
+  }
+}
+
+/**
+ * Exits the process when all started nodes have stopped
+ * @returns {void}
+ */
+const didStopHandler = () => {
+  if (--nodesStarted < 1) {
+    console.log('all nodes stopped, exiting.')
+    process.exit(0)
+  }
+}
+
+/**
+ * Stops the running nodes
+ * @param {Error} err An optional error to log to the console
+ * @returns {void}
+ */
+const doStop = (err) => {
+  if (err) {
+    console.error(err)
+  }
+
+  console.log('Shutting down...')
+  node1.stop()
+  node2.stop()
+}
+
+/**
+ * Connects the IPFS nodes and transfers data between them
+ * @returns {void}
+ */
+const connectAndTalk = async () => {
+  console.log('connecting the nodes...')
+  const node2Id = await node2.id()
+  const dataToAdd = Buffer.from('Hello, private friend!')
+
+  // Connect the nodes
+  // This will error when different private keys are used
+  try {
+    await node1.swarm.connect(node2Id.addresses[0])
+  } catch (err) {
+    return doStop(err)
+  }
+  console.log('the nodes are connected, let\'s add some data')
+
+  // Add some data to node 1
+  let addedCID
+  try {
+    addedCID = await node1.files.add(dataToAdd)
+  } catch (err) {
+    return doStop(err)
+  }
+  console.log(`added ${addedCID[0].path} to the node1`)
+
+  // Retrieve the data from node 2
+  let cattedData
+  try {
+    cattedData = await node2.files.cat(addedCID[0].path)
+  } catch (err) {
+    return doStop(err)
+  }
+  assert.deepEqual(cattedData.toString(), dataToAdd.toString(), 'Should have equal data')
+  console.log(`successfully retrieved "${dataToAdd.toString()}" from node2`)
+
+  doStop()
+}
+
+// Wait for the nodes to boot
+node1.once('start', didStartHandler)
+node2.once('start', didStartHandler)
+
+// Listen for the nodes stopping so we can cleanup
+node1.once('stop', didStopHandler)
+node2.once('stop', didStopHandler)

examples/pnet-ipfs/libp2p-bundle.js

@@ -0,0 +1,60 @@
+'use strict'
+
+const Libp2p = require('libp2p')
+const TCP = require('libp2p-tcp')
+const MPLEX = require('libp2p-mplex')
+const SECIO = require('libp2p-secio')
+const fs = require('fs')
+const Protector = require('libp2p-pnet')
+
+/**
+ * Options for the libp2p bundle
+ * @typedef {Object} libp2pBundle~options
+ * @property {PeerInfo} peerInfo - The PeerInfo of the IPFS node
+ * @property {PeerBook} peerBook - The PeerBook of the IPFS node
+ * @property {Object} config - The config of the IPFS node
+ * @property {Object} options - The options given to the IPFS node
+ */
+
+/**
+ * privateLibp2pBundle returns a libp2p bundle function that will use the swarm
+ * key at the given `swarmKeyPath` to create the Protector
+ *
+ * @param {string} swarmKeyPath The path to our swarm key
+ * @returns {libp2pBundle} Returns a libp2pBundle function for use in IPFS creation
+ */
+const privateLibp2pBundle = (swarmKeyPath) => {
+  /**
+   * This is the bundle we will use to create our fully customized libp2p bundle.
+   *
+   * @param {libp2pBundle~options} opts The options to use when generating the libp2p node
+   * @returns {Libp2p} Our new libp2p node
+   */
+  const libp2pBundle = (opts) => {
+    // Set convenience variables to clearly showcase some of the useful things that are available
+    const peerInfo = opts.peerInfo
+    const peerBook = opts.peerBook
+
+    // Build and return our libp2p node
+    return new Libp2p({
+      peerInfo,
+      peerBook,
+      modules: {
+        transport: [TCP], // We're only using the TCP transport for this example
+        streamMuxer: [MPLEX], // We're only using mplex muxing
+        // Let's make sure to use identifying crypto in our pnet since the protector doesn't
+        // care about node identity, and only the presence of private keys
+        connEncryption: [SECIO],
+        // Leave peer discovery empty, we don't want to find peers. We could omit the property, but it's
+        // being left in for explicit readability.
+        // We should explicitly dial pnet peers, or use a custom discovery service for finding nodes in our pnet
+        peerDiscovery: [],
+        connProtector: new Protector(fs.readFileSync(swarmKeyPath))
+      }
+    })
+  }
+
+  return libp2pBundle
+}
+
+module.exports = privateLibp2pBundle

examples/pnet-ipfs/package.json

@@ -0,0 +1,21 @@
+{
+  "name": "pnet-ipfs-example",
+  "version": "1.0.0",
+  "description": "An example of private networking with IPFS",
+  "main": "index.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1",
+    "start": "node index.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "dependencies": {
+    "ipfs": "~0.32.3",
+    "libp2p": "~0.23.1",
+    "libp2p-mplex": "~0.8.2",
+    "libp2p-pnet": "../../",
+    "libp2p-secio": "~0.10.0",
+    "libp2p-tcp": "~0.13.0"
+  }
+}

examples/pnet-ipfs/utils.js

@@ -0,0 +1,28 @@
+'use strict'
+const fs = require('fs')
+const path = require('path')
+
+/**
+ * mkdirp recursively creates needed folders for the given dir path
+ * @param {string} dir
+ * @returns {string} The path that was created
+ */
+module.exports.mkdirp = (dir) => {
+  return path
+    .resolve(dir)
+    .split(path.sep)
+    .reduce((acc, cur) => {
+      const currentPath = path.normalize(acc + path.sep + cur)
+
+      try {
+        fs.statSync(currentPath)
+      } catch (e) {
+        if (e.code === 'ENOENT') {
+          fs.mkdirSync(currentPath)
+        } else {
+          throw e
+        }
+      }
+      return currentPath
+    }, '')
+}