diff --git a/TransportIPFS.js b/TransportIPFS.js index d2327c9..215dcb1 100644 --- a/TransportIPFS.js +++ b/TransportIPFS.js @@ -236,7 +236,7 @@ class TransportIPFS extends Transport { const ipfspath = TransportIPFS.ipfsFrom(url) // Need because dag.get has different requirement than file.cat try { - const res = await utils.p_timeout(this.ipfs.dag.get(cid), timeoutMS); // Will reject and throw TimeoutError if times out + const res = await utils.p_timeout(this.ipfs.dag.get(cid), timeoutMS, "Timed out IPFS fetch of "+TransportIPFS._stringFrom(cid)); // Will reject and throw TimeoutError if times out // noinspection Annotator if (res.remainderPath.length) { // noinspection ExceptionCaughtLocallyJS @@ -268,9 +268,10 @@ class TransportIPFS extends Transport { } catch (err) { // TimeoutError or could be some other error from IPFS etc console.log("Caught misc error in TransportIPFS.p_rawfetch trying IPFS", err.message); try { + let ipfsurl = TransportIPFS.ipfsGatewayFrom(url) return await utils.p_timeout( - httptools.p_GET(TransportIPFS.ipfsGatewayFrom(url)), // Returns a buffer - timeoutMS) + httptools.p_GET(ipfsurl), // Returns a buffer + timeoutMS, "Timed out IPFS fetch of "+ipfsurl) } catch (err) { console.log("Caught misc error in TransportIPFS.p_rawfetch trying gateway", err.message); throw err; diff --git a/dist/dweb-transports-bundle.js b/dist/dweb-transports-bundle.js index a9cc4c7..931c619 100644 --- a/dist/dweb-transports-bundle.js +++ b/dist/dweb-transports-bundle.js @@ -722,7 +722,7 @@ eval("const Transport = __webpack_require__(/*! ./Transport */ \"./Transport.js\ /*! no static exports found */ /***/ (function(module, exports, __webpack_require__) { -eval("/* WEBPACK VAR INJECTION */(function(Buffer) {/*\nThis is a shim to the IPFS library, (Lists are handled in YJS or OrbitDB)\nSee https://github.com/ipfs/js-ipfs but note its often out of date relative to the generic API doc.\n*/\n\nconst httptools = __webpack_require__(/*! ./httptools */ \"./httptools.js\"); // Expose some of the httptools so that IPFS can use it as a backup\n\n// IPFS components\nconst IPFS = __webpack_require__(/*! ipfs */ \"./node_modules/ipfs/src/core/index.js\");\nconst CID = __webpack_require__(/*! cids */ \"./node_modules/cids/src/index.js\");\n//Removed next two as not needed if use \"Kludge\" flagged below.\n//const dagPB = require('ipld-dag-pb');\n//const DAGNode = dagPB.DAGNode; // So can check its type\nconst unixFs = __webpack_require__(/*! ipfs-unixfs */ \"./node_modules/ipfs-unixfs/src/index.js\");\n\n// Library packages other than IPFS\nconst Url = __webpack_require__(/*! url */ \"../../../../usr/local/lib/node_modules/webpack/node_modules/url/url.js\");\nconst stream = __webpack_require__(/*! readable-stream */ \"./node_modules/readable-stream/readable-browser.js\"); // Needed for the pullthrough - this is NOT Ipfs streams\n// Alternative to through - as used in WebTorrent\n\n// Utility packages (ours) And one-liners\n//No longer reqd: const promisify = require('promisify-es6');\n//const makepromises = require('./utils/makepromises'); // Replaced by direct call to promisify\n\n// Other Dweb modules\nconst errors = __webpack_require__(/*! ./Errors */ \"./Errors.js\"); // Standard Dweb Errors\nconst Transport = __webpack_require__(/*! ./Transport.js */ \"./Transport.js\"); // Base class for TransportXyz\nconst Transports = __webpack_require__(/*! ./Transports */ \"./Transports.js\"); // Manage all Transports that are loaded\nconst utils = __webpack_require__(/*! ./utils */ \"./utils.js\"); // Utility functions\n\nconst defaultoptions = {\n ipfs: {\n repo: '/tmp/dweb_ipfsv2700', //TODO-IPFS think through where, esp for browser\n //init: false,\n //start: false,\n //TODO-IPFS-Q how is this decentralized - can it run offline? Does it depend on star-signal.cloud.ipfs.team\n config: {\n // Addresses: { Swarm: [ '/dns4/star-signal.cloud.ipfs.team/wss/p2p-webrtc-star']}, // For Y - same as defaults\n // Addresses: { Swarm: [ ] }, // Disable WebRTC to test browser crash, note disables Y so doesnt work.\n Addresses: {Swarm: ['/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star']}, // from https://github.com/ipfs/js-ipfs#faq 2017-12-05 as alternative to webrtc works sort-of\n //Bootstrap: ['/dns4/gateway.dweb.me/tcp/443/ws/ipfs/QmPNgKEjC7wkpu3aHUzKKhZmbEfiGzL5TP1L8zZoHJyXZW'], // Supposedly connects to Dweb IPFS instance, but doesnt work (nor does \".../wss/...\")\n },\n //init: true, // Comment out for Y\n EXPERIMENTAL: {\n pubsub: true\n }\n }\n};\n\nclass TransportIPFS extends Transport {\n /*\n IPFS specific transport\n\n Fields:\n ipfs: object returned when starting IPFS\n yarray: object returned when starting yarray\n */\n\n constructor(options, verbose) {\n super(options, verbose);\n this.ipfs = undefined; // Undefined till start IPFS\n this.options = options; // Dictionary of options { ipfs: {...}, \"yarrays\", yarray: {...} }\n this.name = \"IPFS\"; // For console log etc\n this.supportURLs = ['ipfs'];\n this.supportFunctions = ['fetch', 'store']; // Does not support reverse, createReadStream fails on files uploaded with urlstore TODO reenable when Kyle fixes urlstore\n this.status = Transport.STATUS_LOADED;\n }\n\n/*\n _makepromises() {\n //Utility function to promisify Block\n //Replaced promisified utility since only two to promisify\n //this.promisified = {ipfs:{}};\n //makepromises(this.ipfs, this.promisified.ipfs, [ { block: [\"put\", \"get\"] }]); // Has to be after this.ipfs defined\n this.promisified = { ipfs: { block: {\n put: promisify(this.ipfs.block.put),\n get: promisify(this.ipfs.block.get)\n }}}\n }\n*/\n p_ipfsstart(verbose) {\n /*\n Just start IPFS - not Y (note used with \"yarrays\" and will be used for non-IPFS list management)\n Note - can't figure out how to use async with this, as we resolve the promise based on the event callback\n */\n const self = this;\n return new Promise((resolve, reject) => {\n this.ipfs = new IPFS(this.options.ipfs);\n this.ipfs.on('ready', () => {\n //this._makepromises();\n resolve();\n });\n this.ipfs.on('error', (err) => reject(err));\n })\n .then(() => self.ipfs.version())\n .then((version) => console.log('IPFS READY',version))\n .catch((err) => {\n console.log(\"Error caught in p_ipfsstart\");\n throw(err);\n });\n }\n\n static setup0(options, verbose) {\n /*\n First part of setup, create obj, add to Transports but dont attempt to connect, typically called instead of p_setup if want to parallelize connections.\n */\n const combinedoptions = Transport.mergeoptions(defaultoptions, options);\n if (verbose) console.log(\"IPFS loading options %o\", combinedoptions);\n const t = new TransportIPFS(combinedoptions, verbose); // Note doesnt start IPFS\n Transports.addtransport(t);\n return t;\n }\n\n async p_setup1(verbose, cb) {\n try {\n if (verbose) console.log(\"IPFS starting and connecting\");\n this.status = Transport.STATUS_STARTING; // Should display, but probably not refreshed in most case\n if (cb) cb(this);\n await this.p_ipfsstart(verbose); // Throws Error(\"websocket error\") and possibly others.\n this.status = await this.p_status(verbose);\n } catch(err) {\n console.error(\"IPFS failed to connect\",err);\n this.status = Transport.STATUS_FAILED;\n }\n if (cb) cb(this);\n return this;\n }\n\n async p_status(verbose) {\n /*\n Return a numeric code for the status of a transport.\n */\n this.status = (await this.ipfs.isOnline()) ? Transport.STATUS_CONNECTED : Transport.STATUS_FAILED;\n return super.p_status(verbose);\n }\n\n // Everything else - unless documented here - should be opaque to the actual structure of a CID\n // or a url. This code may change as its not clear (from IPFS docs) if this is the right mapping.\n static urlFrom(unknown) {\n /*\n Convert a CID into a standardised URL e.g. ipfs:/ipfs/abc123\n */\n if (unknown instanceof CID)\n return \"ipfs:/ipfs/\"+unknown.toBaseEncodedString();\n if (typeof unknown === \"object\" && unknown.hash) // e.g. from files.add\n return \"ipfs:/ipfs/\"+unknown.hash;\n if (typeof unknown === \"string\") // Not used currently\n return \"ipfs:/ipfs/\"+unknown;\n throw new errors.CodingError(\"TransportIPFS.urlFrom: Cant convert to url from\",unknown);\n }\n\n static cidFrom(url) {\n /*\n Convert a URL e.g. ipfs:/ipfs/abc123 into a CID structure suitable for retrieval\n url: String of form \"ipfs://ipfs/\" or parsed URL or CID\n returns: CID\n throws: TransportError if cant convert\n */\n if (url instanceof CID) return url;\n if (typeof(url) === \"string\") url = Url.parse(url);\n if (url && url[\"pathname\"]) { // On browser \"instanceof Url\" isn't valid)\n const patharr = url.pathname.split('/');\n if ((![\"ipfs:\",\"dweb:\"].includes(url.protocol)) || (patharr[1] !== 'ipfs') || (patharr.length < 3))\n throw new errors.TransportError(\"TransportIPFS.cidFrom bad format for url should be dweb: or ipfs:/ipfs/...: \" + url.href);\n if (patharr.length > 3)\n throw new errors.TransportError(\"TransportIPFS.cidFrom not supporting paths in url yet, should be dweb: or ipfs:/ipfs/...: \" + url.href);\n return new CID(patharr[2]);\n } else {\n throw new errors.CodingError(\"TransportIPFS.cidFrom: Cant convert url\",url);\n }\n }\n\n static _stringFrom(url) {\n // Tool for ipfsFrom and ipfsGatewayFrom\n if (url instanceof CID)\n return \"/ipfs/\"+url.toBaseEncodedString();\n if (typeof url === 'object' && url.path) { // It better be URL which unfortunately is hard to test\n return url.path;\n }\n }\n static ipfsFrom(url) {\n /*\n Convert to a ipfspath i.e. /ipfs/Qm....\n Required because of strange differences in APIs between files.cat and dag.get see https://github.com/ipfs/js-ipfs/issues/1229\n */\n url = this._stringFrom(url); // Convert CID or Url to a string hopefully containing /ipfs/\n if (url.indexOf('/ipfs/') > -1) {\n return url.slice(url.indexOf('/ipfs/'));\n }\n throw new errors.CodingError(`TransportIPFS.ipfsFrom: Cant convert url ${url} into a path starting /ipfs/`);\n }\n\n static ipfsGatewayFrom(url) {\n /*\n url: CID, Url, or a string\n returns: https://ipfs.io/ipfs/\n */\n url = this._stringFrom(url); // Convert CID or Url to a string hopefully containing /ipfs/\n if (url.indexOf('/ipfs/') > -1) {\n return \"https://ipfs.io\" + url.slice(url.indexOf('/ipfs/'));\n }\n throw new errors.CodingError(`TransportIPFS.ipfsGatewayFrom: Cant convert url ${url} into a path starting /ipfs/`);\n }\n\n static multihashFrom(url) {\n if (url instanceof CID)\n return cid.toBaseEncodedString();\n if (typeof url === 'object' && url.path)\n url = url.path; // /ipfs/Q...\n if (typeof(url) === \"string\") {\n const idx = url.indexOf(\"/ipfs/\");\n if (idx > -1) {\n return url.slice(idx+6);\n }\n }\n throw new errors.CodingError(`Cant turn ${url} into a multihash`);\n }\n\n async p_rawfetch(url, {verbose=false, timeoutMS=60000, relay=false}={}) {\n /*\n Fetch some bytes based on a url of the form ipfs:/ipfs/Qm..... or ipfs:/ipfs/z.... .\n No assumption is made about the data in terms of size or structure, nor can we know whether it was created with dag.put or ipfs add or http /api/v0/add/\n\n Where required by the underlying transport it should retrieve a number if its \"blocks\" and concatenate them.\n Returns a new Promise that resolves currently to a string.\n There may also be need for a streaming version of this call, at this point undefined since we havent (currently) got a use case..\n\n :param string url: URL of object being retrieved {ipfs|dweb}:/ipfs/ or /\n :param boolean verbose: true for debugging output\n :resolve buffer: Return the object being fetched. (may in the future return a stream and buffer externally)\n :throws: TransportError if url invalid - note this happens immediately, not as a catch in the promise\n */\n if (verbose) console.log(\"IPFS p_rawfetch\", utils.stringfrom(url));\n if (!url) throw new errors.CodingError(\"TransportIPFS.p_rawfetch: requires url\");\n const cid = TransportIPFS.cidFrom(url); // Throws TransportError if url bad\n const ipfspath = TransportIPFS.ipfsFrom(url) // Need because dag.get has different requirement than file.cat\n\n try {\n const res = await utils.p_timeout(this.ipfs.dag.get(cid), timeoutMS); // Will reject and throw TimeoutError if times out\n // noinspection Annotator\n if (res.remainderPath.length)\n { // noinspection ExceptionCaughtLocallyJS\n throw new errors.TransportError(\"Not yet supporting paths in p_rawfetch\");\n } //TODO-PATH\n let buff;\n //if (res.value instanceof DAGNode) { // Its file or something added with the HTTP API for example, TODO not yet handling multiple files\n if (res.value.constructor.name === \"DAGNode\") { // Kludge to replace above, as its not matching the type against the \"require\" above.\n if (verbose) console.log(\"IPFS p_rawfetch looks like its a file\", url);\n //console.log(\"Case a or b\" - we can tell the difference by looking at (res.value._links.length > 0) but dont need to\n // as since we dont know if we are on node or browser best way is to try the files.cat and if it fails try the block to get an approximate file);\n // Works on Node, but fails on Chrome, cant figure out how to get data from the DAGNode otherwise (its the wrong size)\n buff = await this.ipfs.files.cat(ipfspath); //See js-ipfs v0.27 version and https://github.com/ipfs/js-ipfs/issues/1229 and https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#cat\n\n /* Was needed on v0.26, not on v0.27\n if (buff.length === 0) { // Hit the Chrome bug\n // This will get a file padded with ~14 bytes - 4 at front, 4 at end and cant find the other 6 !\n // but it seems to work for PDFs which is what I'm testing on.\n if (verbose) console.log(\"Kludge alert - files.cat fails in Chrome, trying block.get\");\n let blk = await this.promisified.ipfs.block.get(cid);\n buff = blk.data;\n }\n END of v0.26 version */\n } else { //c: not a file\n buff = res.value;\n }\n if (verbose) console.log(`IPFS fetched ${buff.length} from ${ipfspath}`);\n return buff;\n } catch (err) { // TimeoutError or could be some other error from IPFS etc\n console.log(\"Caught misc error in TransportIPFS.p_rawfetch trying IPFS\", err.message);\n try {\n return await utils.p_timeout(\n httptools.p_GET(TransportIPFS.ipfsGatewayFrom(url)), // Returns a buffer\n timeoutMS)\n } catch (err) {\n console.log(\"Caught misc error in TransportIPFS.p_rawfetch trying gateway\", err.message);\n throw err;\n }\n }\n }\n\n async p_rawstore(data, {verbose}) {\n /*\n Store a blob of data onto the decentralised transport.\n Returns a promise that resolves to the url of the data\n\n :param string|Buffer data: Data to store - no assumptions made to size or content\n :param boolean verbose: true for debugging output\n :resolve string: url of data stored\n */\n console.assert(data, \"TransportIPFS.p_rawstore: requires data\");\n const buf = (data instanceof Buffer) ? data : new Buffer(data);\n //return this.promisified.ipfs.block.put(buf).then((block) => block.cid)\n //https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md#dagput\n //let res = await this.ipfs.dag.put(buf,{ format: 'dag-cbor', hashAlg: 'sha2-256' });\n const res = (await this.ipfs.files.add(buf,{ \"cid-version\": 1, hashAlg: 'sha2-256'}))[0];\n return TransportIPFS.urlFrom(res);\n }\n\n // Based on https://github.com/ipfs/js-ipfs/pull/1231/files\n\n async p_offsetStream(stream, links, startByte, endByte) {\n let streamPosition = 0\n try {\n for (let l in links) {\n const link = links[l];\n if (!stream.writable) { return } // The stream has been closed\n // DAGNode Links report unixfs object data sizes 14 bytes larger due to the protobuf wrapper\n const bytesInLinkedObjectData = link.size - 14\n if (startByte > (streamPosition + bytesInLinkedObjectData)) {\n // Start byte is after this block so skip it\n streamPosition += bytesInLinkedObjectData;\n } else if (endByte && endByte < streamPosition) { // TODO-STREAM this is copied from https://github.com/ipfs/js-ipfs/pull/1231/files but I think it should be endByte <= since endByte is first byte DONT want\n // End byte was before this block so skip it\n streamPosition += bytesInLinkedObjectData;\n } else {\n let lmh = link.multihash;\n let data;\n await this.ipfs.object.data(lmh)\n .then ((d) => unixFs.unmarshal(d).data)\n .then ((d) => data = d )\n .catch((err) => {console.log(\"XXX@289 err=\",err);});\n if (!stream.writable) { return; } // The stream was closed while we were getting data\n const length = data.length;\n if (startByte > streamPosition && startByte < (streamPosition + length)) {\n // If the startByte is in the current block, skip to the startByte\n data = data.slice(startByte - streamPosition);\n }\n console.log(`Writing ${data.length} to stream`)\n stream.write(data);\n streamPosition += length;\n }\n }\n } catch(err) {\n console.log(err.message);\n }\n }\n async p_f_createReadStream(url, {verbose=false}={}) { // Asynchronously return a function that can be used in createReadStream\n verbose = true;\n if (verbose) console.log(\"p_f_createReadStream\",url);\n const mh = TransportIPFS.multihashFrom(url);\n const links = await this.ipfs.object.links(mh)\n let throughstream; //Holds pointer to stream between calls.\n const self = this;\n function crs(opts) { // This is a synchronous function\n // Return a readable stream that provides the bytes between offsets \"start\" and \"end\" inclusive\n console.log(\"opts=\",JSON.stringify(opts));\n /* Can replace rest of crs with this when https://github.com/ipfs/js-ipfs/pull/1231/files lands (hopefully v0.28.3)\n return self.ipfs.catReadableStream(mh, opts ? opts.start : 0, opts && opts.end) ? opts.end+1 : undefined)\n */\n if (!opts) return throughstream; //TODO-STREAM unclear why called without opts - take this out when figured out\n if (throughstream && throughstream.destroy) throughstream.destroy();\n throughstream = new stream.PassThrough();\n\n self.p_offsetStream( // Ignore promise returned, this will right to the stream asynchronously\n throughstream,\n links, // Uses the array of links created above in this function\n opts ? opts.start : 0,\n (opts && opts.end) ? opts.end : undefined);\n return throughstream;\n }\n return crs;\n }\n\n static async p_test(opts, verbose) {\n if (verbose) {console.log(\"TransportIPFS.test\")}\n try {\n const transport = await this.p_setup(opts, verbose); // Assumes IPFS already setup\n if (verbose) console.log(transport.name,\"setup\");\n const res = await transport.p_status(verbose);\n console.assert(res === Transport.STATUS_CONNECTED)\n\n let urlqbf;\n const qbf = \"The quick brown fox\";\n const qbf_url = \"ipfs:/ipfs/zdpuAscRnisRkYnEyJAp1LydQ3po25rCEDPPEDMymYRfN1yPK\"; // Expected url\n const testurl = \"1114\"; // Just a predictable number can work with\n const url = await transport.p_rawstore(qbf, {verbose});\n if (verbose) console.log(\"rawstore returned\", url);\n const newcid = TransportIPFS.cidFrom(url); // Its a CID which has a buffer in it\n console.assert(url === qbf_url, \"url should match url from rawstore\");\n const cidmultihash = url.split('/')[2]; // Store cid from first block in form of multihash\n const newurl = TransportIPFS.urlFrom(newcid);\n console.assert(url === newurl, \"Should round trip\");\n urlqbf = url;\n const data = await transport.p_rawfetch(urlqbf, {verbose});\n console.assert(data.toString() === qbf, \"Should fetch block stored above\");\n //console.log(\"TransportIPFS test complete\");\n return transport\n } catch(err) {\n console.log(\"Exception thrown in TransportIPFS.test:\", err.message);\n throw err;\n }\n }\n\n}\nTransports._transportclasses[\"IPFS\"] = TransportIPFS;\nexports = module.exports = TransportIPFS;\n\n/* WEBPACK VAR INJECTION */}.call(this, __webpack_require__(/*! ./../../../../usr/local/lib/node_modules/webpack/node_modules/buffer/index.js */ \"../../../../usr/local/lib/node_modules/webpack/node_modules/buffer/index.js\").Buffer))\n\n//# sourceURL=webpack:///./TransportIPFS.js?"); +eval("/* WEBPACK VAR INJECTION */(function(Buffer) {/*\nThis is a shim to the IPFS library, (Lists are handled in YJS or OrbitDB)\nSee https://github.com/ipfs/js-ipfs but note its often out of date relative to the generic API doc.\n*/\n\nconst httptools = __webpack_require__(/*! ./httptools */ \"./httptools.js\"); // Expose some of the httptools so that IPFS can use it as a backup\n\n// IPFS components\nconst IPFS = __webpack_require__(/*! ipfs */ \"./node_modules/ipfs/src/core/index.js\");\nconst CID = __webpack_require__(/*! cids */ \"./node_modules/cids/src/index.js\");\n//Removed next two as not needed if use \"Kludge\" flagged below.\n//const dagPB = require('ipld-dag-pb');\n//const DAGNode = dagPB.DAGNode; // So can check its type\nconst unixFs = __webpack_require__(/*! ipfs-unixfs */ \"./node_modules/ipfs-unixfs/src/index.js\");\n\n// Library packages other than IPFS\nconst Url = __webpack_require__(/*! url */ \"../../../../usr/local/lib/node_modules/webpack/node_modules/url/url.js\");\nconst stream = __webpack_require__(/*! readable-stream */ \"./node_modules/readable-stream/readable-browser.js\"); // Needed for the pullthrough - this is NOT Ipfs streams\n// Alternative to through - as used in WebTorrent\n\n// Utility packages (ours) And one-liners\n//No longer reqd: const promisify = require('promisify-es6');\n//const makepromises = require('./utils/makepromises'); // Replaced by direct call to promisify\n\n// Other Dweb modules\nconst errors = __webpack_require__(/*! ./Errors */ \"./Errors.js\"); // Standard Dweb Errors\nconst Transport = __webpack_require__(/*! ./Transport.js */ \"./Transport.js\"); // Base class for TransportXyz\nconst Transports = __webpack_require__(/*! ./Transports */ \"./Transports.js\"); // Manage all Transports that are loaded\nconst utils = __webpack_require__(/*! ./utils */ \"./utils.js\"); // Utility functions\n\nconst defaultoptions = {\n ipfs: {\n repo: '/tmp/dweb_ipfsv2700', //TODO-IPFS think through where, esp for browser\n //init: false,\n //start: false,\n //TODO-IPFS-Q how is this decentralized - can it run offline? Does it depend on star-signal.cloud.ipfs.team\n config: {\n // Addresses: { Swarm: [ '/dns4/star-signal.cloud.ipfs.team/wss/p2p-webrtc-star']}, // For Y - same as defaults\n // Addresses: { Swarm: [ ] }, // Disable WebRTC to test browser crash, note disables Y so doesnt work.\n Addresses: {Swarm: ['/dns4/ws-star.discovery.libp2p.io/tcp/443/wss/p2p-websocket-star']}, // from https://github.com/ipfs/js-ipfs#faq 2017-12-05 as alternative to webrtc works sort-of\n //Bootstrap: ['/dns4/gateway.dweb.me/tcp/443/ws/ipfs/QmPNgKEjC7wkpu3aHUzKKhZmbEfiGzL5TP1L8zZoHJyXZW'], // Supposedly connects to Dweb IPFS instance, but doesnt work (nor does \".../wss/...\")\n },\n //init: true, // Comment out for Y\n EXPERIMENTAL: {\n pubsub: true\n }\n }\n};\n\nclass TransportIPFS extends Transport {\n /*\n IPFS specific transport\n\n Fields:\n ipfs: object returned when starting IPFS\n yarray: object returned when starting yarray\n */\n\n constructor(options, verbose) {\n super(options, verbose);\n this.ipfs = undefined; // Undefined till start IPFS\n this.options = options; // Dictionary of options { ipfs: {...}, \"yarrays\", yarray: {...} }\n this.name = \"IPFS\"; // For console log etc\n this.supportURLs = ['ipfs'];\n this.supportFunctions = ['fetch', 'store']; // Does not support reverse, createReadStream fails on files uploaded with urlstore TODO reenable when Kyle fixes urlstore\n this.status = Transport.STATUS_LOADED;\n }\n\n/*\n _makepromises() {\n //Utility function to promisify Block\n //Replaced promisified utility since only two to promisify\n //this.promisified = {ipfs:{}};\n //makepromises(this.ipfs, this.promisified.ipfs, [ { block: [\"put\", \"get\"] }]); // Has to be after this.ipfs defined\n this.promisified = { ipfs: { block: {\n put: promisify(this.ipfs.block.put),\n get: promisify(this.ipfs.block.get)\n }}}\n }\n*/\n p_ipfsstart(verbose) {\n /*\n Just start IPFS - not Y (note used with \"yarrays\" and will be used for non-IPFS list management)\n Note - can't figure out how to use async with this, as we resolve the promise based on the event callback\n */\n const self = this;\n return new Promise((resolve, reject) => {\n this.ipfs = new IPFS(this.options.ipfs);\n this.ipfs.on('ready', () => {\n //this._makepromises();\n resolve();\n });\n this.ipfs.on('error', (err) => reject(err));\n })\n .then(() => self.ipfs.version())\n .then((version) => console.log('IPFS READY',version))\n .catch((err) => {\n console.log(\"Error caught in p_ipfsstart\");\n throw(err);\n });\n }\n\n static setup0(options, verbose) {\n /*\n First part of setup, create obj, add to Transports but dont attempt to connect, typically called instead of p_setup if want to parallelize connections.\n */\n const combinedoptions = Transport.mergeoptions(defaultoptions, options);\n if (verbose) console.log(\"IPFS loading options %o\", combinedoptions);\n const t = new TransportIPFS(combinedoptions, verbose); // Note doesnt start IPFS\n Transports.addtransport(t);\n return t;\n }\n\n async p_setup1(verbose, cb) {\n try {\n if (verbose) console.log(\"IPFS starting and connecting\");\n this.status = Transport.STATUS_STARTING; // Should display, but probably not refreshed in most case\n if (cb) cb(this);\n await this.p_ipfsstart(verbose); // Throws Error(\"websocket error\") and possibly others.\n this.status = await this.p_status(verbose);\n } catch(err) {\n console.error(\"IPFS failed to connect\",err);\n this.status = Transport.STATUS_FAILED;\n }\n if (cb) cb(this);\n return this;\n }\n\n async p_status(verbose) {\n /*\n Return a numeric code for the status of a transport.\n */\n this.status = (await this.ipfs.isOnline()) ? Transport.STATUS_CONNECTED : Transport.STATUS_FAILED;\n return super.p_status(verbose);\n }\n\n // Everything else - unless documented here - should be opaque to the actual structure of a CID\n // or a url. This code may change as its not clear (from IPFS docs) if this is the right mapping.\n static urlFrom(unknown) {\n /*\n Convert a CID into a standardised URL e.g. ipfs:/ipfs/abc123\n */\n if (unknown instanceof CID)\n return \"ipfs:/ipfs/\"+unknown.toBaseEncodedString();\n if (typeof unknown === \"object\" && unknown.hash) // e.g. from files.add\n return \"ipfs:/ipfs/\"+unknown.hash;\n if (typeof unknown === \"string\") // Not used currently\n return \"ipfs:/ipfs/\"+unknown;\n throw new errors.CodingError(\"TransportIPFS.urlFrom: Cant convert to url from\",unknown);\n }\n\n static cidFrom(url) {\n /*\n Convert a URL e.g. ipfs:/ipfs/abc123 into a CID structure suitable for retrieval\n url: String of form \"ipfs://ipfs/\" or parsed URL or CID\n returns: CID\n throws: TransportError if cant convert\n */\n if (url instanceof CID) return url;\n if (typeof(url) === \"string\") url = Url.parse(url);\n if (url && url[\"pathname\"]) { // On browser \"instanceof Url\" isn't valid)\n const patharr = url.pathname.split('/');\n if ((![\"ipfs:\",\"dweb:\"].includes(url.protocol)) || (patharr[1] !== 'ipfs') || (patharr.length < 3))\n throw new errors.TransportError(\"TransportIPFS.cidFrom bad format for url should be dweb: or ipfs:/ipfs/...: \" + url.href);\n if (patharr.length > 3)\n throw new errors.TransportError(\"TransportIPFS.cidFrom not supporting paths in url yet, should be dweb: or ipfs:/ipfs/...: \" + url.href);\n return new CID(patharr[2]);\n } else {\n throw new errors.CodingError(\"TransportIPFS.cidFrom: Cant convert url\",url);\n }\n }\n\n static _stringFrom(url) {\n // Tool for ipfsFrom and ipfsGatewayFrom\n if (url instanceof CID)\n return \"/ipfs/\"+url.toBaseEncodedString();\n if (typeof url === 'object' && url.path) { // It better be URL which unfortunately is hard to test\n return url.path;\n }\n }\n static ipfsFrom(url) {\n /*\n Convert to a ipfspath i.e. /ipfs/Qm....\n Required because of strange differences in APIs between files.cat and dag.get see https://github.com/ipfs/js-ipfs/issues/1229\n */\n url = this._stringFrom(url); // Convert CID or Url to a string hopefully containing /ipfs/\n if (url.indexOf('/ipfs/') > -1) {\n return url.slice(url.indexOf('/ipfs/'));\n }\n throw new errors.CodingError(`TransportIPFS.ipfsFrom: Cant convert url ${url} into a path starting /ipfs/`);\n }\n\n static ipfsGatewayFrom(url) {\n /*\n url: CID, Url, or a string\n returns: https://ipfs.io/ipfs/\n */\n url = this._stringFrom(url); // Convert CID or Url to a string hopefully containing /ipfs/\n if (url.indexOf('/ipfs/') > -1) {\n return \"https://ipfs.io\" + url.slice(url.indexOf('/ipfs/'));\n }\n throw new errors.CodingError(`TransportIPFS.ipfsGatewayFrom: Cant convert url ${url} into a path starting /ipfs/`);\n }\n\n static multihashFrom(url) {\n if (url instanceof CID)\n return cid.toBaseEncodedString();\n if (typeof url === 'object' && url.path)\n url = url.path; // /ipfs/Q...\n if (typeof(url) === \"string\") {\n const idx = url.indexOf(\"/ipfs/\");\n if (idx > -1) {\n return url.slice(idx+6);\n }\n }\n throw new errors.CodingError(`Cant turn ${url} into a multihash`);\n }\n\n async p_rawfetch(url, {verbose=false, timeoutMS=60000, relay=false}={}) {\n /*\n Fetch some bytes based on a url of the form ipfs:/ipfs/Qm..... or ipfs:/ipfs/z.... .\n No assumption is made about the data in terms of size or structure, nor can we know whether it was created with dag.put or ipfs add or http /api/v0/add/\n\n Where required by the underlying transport it should retrieve a number if its \"blocks\" and concatenate them.\n Returns a new Promise that resolves currently to a string.\n There may also be need for a streaming version of this call, at this point undefined since we havent (currently) got a use case..\n\n :param string url: URL of object being retrieved {ipfs|dweb}:/ipfs/ or /\n :param boolean verbose: true for debugging output\n :resolve buffer: Return the object being fetched. (may in the future return a stream and buffer externally)\n :throws: TransportError if url invalid - note this happens immediately, not as a catch in the promise\n */\n if (verbose) console.log(\"IPFS p_rawfetch\", utils.stringfrom(url));\n if (!url) throw new errors.CodingError(\"TransportIPFS.p_rawfetch: requires url\");\n const cid = TransportIPFS.cidFrom(url); // Throws TransportError if url bad\n const ipfspath = TransportIPFS.ipfsFrom(url) // Need because dag.get has different requirement than file.cat\n\n try {\n const res = await utils.p_timeout(this.ipfs.dag.get(cid), timeoutMS, \"Timed out IPFS fetch of \"+TransportIPFS._stringFrom(cid)); // Will reject and throw TimeoutError if times out\n // noinspection Annotator\n if (res.remainderPath.length)\n { // noinspection ExceptionCaughtLocallyJS\n throw new errors.TransportError(\"Not yet supporting paths in p_rawfetch\");\n } //TODO-PATH\n let buff;\n //if (res.value instanceof DAGNode) { // Its file or something added with the HTTP API for example, TODO not yet handling multiple files\n if (res.value.constructor.name === \"DAGNode\") { // Kludge to replace above, as its not matching the type against the \"require\" above.\n if (verbose) console.log(\"IPFS p_rawfetch looks like its a file\", url);\n //console.log(\"Case a or b\" - we can tell the difference by looking at (res.value._links.length > 0) but dont need to\n // as since we dont know if we are on node or browser best way is to try the files.cat and if it fails try the block to get an approximate file);\n // Works on Node, but fails on Chrome, cant figure out how to get data from the DAGNode otherwise (its the wrong size)\n buff = await this.ipfs.files.cat(ipfspath); //See js-ipfs v0.27 version and https://github.com/ipfs/js-ipfs/issues/1229 and https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#cat\n\n /* Was needed on v0.26, not on v0.27\n if (buff.length === 0) { // Hit the Chrome bug\n // This will get a file padded with ~14 bytes - 4 at front, 4 at end and cant find the other 6 !\n // but it seems to work for PDFs which is what I'm testing on.\n if (verbose) console.log(\"Kludge alert - files.cat fails in Chrome, trying block.get\");\n let blk = await this.promisified.ipfs.block.get(cid);\n buff = blk.data;\n }\n END of v0.26 version */\n } else { //c: not a file\n buff = res.value;\n }\n if (verbose) console.log(`IPFS fetched ${buff.length} from ${ipfspath}`);\n return buff;\n } catch (err) { // TimeoutError or could be some other error from IPFS etc\n console.log(\"Caught misc error in TransportIPFS.p_rawfetch trying IPFS\", err.message);\n try {\n let ipfsurl = TransportIPFS.ipfsGatewayFrom(url)\n return await utils.p_timeout(\n httptools.p_GET(ipfsurl), // Returns a buffer\n timeoutMS, \"Timed out IPFS fetch of \"+ipfsurl)\n } catch (err) {\n console.log(\"Caught misc error in TransportIPFS.p_rawfetch trying gateway\", err.message);\n throw err;\n }\n }\n }\n\n async p_rawstore(data, {verbose}) {\n /*\n Store a blob of data onto the decentralised transport.\n Returns a promise that resolves to the url of the data\n\n :param string|Buffer data: Data to store - no assumptions made to size or content\n :param boolean verbose: true for debugging output\n :resolve string: url of data stored\n */\n console.assert(data, \"TransportIPFS.p_rawstore: requires data\");\n const buf = (data instanceof Buffer) ? data : new Buffer(data);\n //return this.promisified.ipfs.block.put(buf).then((block) => block.cid)\n //https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md#dagput\n //let res = await this.ipfs.dag.put(buf,{ format: 'dag-cbor', hashAlg: 'sha2-256' });\n const res = (await this.ipfs.files.add(buf,{ \"cid-version\": 1, hashAlg: 'sha2-256'}))[0];\n return TransportIPFS.urlFrom(res);\n }\n\n // Based on https://github.com/ipfs/js-ipfs/pull/1231/files\n\n async p_offsetStream(stream, links, startByte, endByte) {\n let streamPosition = 0\n try {\n for (let l in links) {\n const link = links[l];\n if (!stream.writable) { return } // The stream has been closed\n // DAGNode Links report unixfs object data sizes 14 bytes larger due to the protobuf wrapper\n const bytesInLinkedObjectData = link.size - 14\n if (startByte > (streamPosition + bytesInLinkedObjectData)) {\n // Start byte is after this block so skip it\n streamPosition += bytesInLinkedObjectData;\n } else if (endByte && endByte < streamPosition) { // TODO-STREAM this is copied from https://github.com/ipfs/js-ipfs/pull/1231/files but I think it should be endByte <= since endByte is first byte DONT want\n // End byte was before this block so skip it\n streamPosition += bytesInLinkedObjectData;\n } else {\n let lmh = link.multihash;\n let data;\n await this.ipfs.object.data(lmh)\n .then ((d) => unixFs.unmarshal(d).data)\n .then ((d) => data = d )\n .catch((err) => {console.log(\"XXX@289 err=\",err);});\n if (!stream.writable) { return; } // The stream was closed while we were getting data\n const length = data.length;\n if (startByte > streamPosition && startByte < (streamPosition + length)) {\n // If the startByte is in the current block, skip to the startByte\n data = data.slice(startByte - streamPosition);\n }\n console.log(`Writing ${data.length} to stream`)\n stream.write(data);\n streamPosition += length;\n }\n }\n } catch(err) {\n console.log(err.message);\n }\n }\n async p_f_createReadStream(url, {verbose=false}={}) { // Asynchronously return a function that can be used in createReadStream\n verbose = true;\n if (verbose) console.log(\"p_f_createReadStream\",url);\n const mh = TransportIPFS.multihashFrom(url);\n const links = await this.ipfs.object.links(mh)\n let throughstream; //Holds pointer to stream between calls.\n const self = this;\n function crs(opts) { // This is a synchronous function\n // Return a readable stream that provides the bytes between offsets \"start\" and \"end\" inclusive\n console.log(\"opts=\",JSON.stringify(opts));\n /* Can replace rest of crs with this when https://github.com/ipfs/js-ipfs/pull/1231/files lands (hopefully v0.28.3)\n return self.ipfs.catReadableStream(mh, opts ? opts.start : 0, opts && opts.end) ? opts.end+1 : undefined)\n */\n if (!opts) return throughstream; //TODO-STREAM unclear why called without opts - take this out when figured out\n if (throughstream && throughstream.destroy) throughstream.destroy();\n throughstream = new stream.PassThrough();\n\n self.p_offsetStream( // Ignore promise returned, this will right to the stream asynchronously\n throughstream,\n links, // Uses the array of links created above in this function\n opts ? opts.start : 0,\n (opts && opts.end) ? opts.end : undefined);\n return throughstream;\n }\n return crs;\n }\n\n static async p_test(opts, verbose) {\n if (verbose) {console.log(\"TransportIPFS.test\")}\n try {\n const transport = await this.p_setup(opts, verbose); // Assumes IPFS already setup\n if (verbose) console.log(transport.name,\"setup\");\n const res = await transport.p_status(verbose);\n console.assert(res === Transport.STATUS_CONNECTED)\n\n let urlqbf;\n const qbf = \"The quick brown fox\";\n const qbf_url = \"ipfs:/ipfs/zdpuAscRnisRkYnEyJAp1LydQ3po25rCEDPPEDMymYRfN1yPK\"; // Expected url\n const testurl = \"1114\"; // Just a predictable number can work with\n const url = await transport.p_rawstore(qbf, {verbose});\n if (verbose) console.log(\"rawstore returned\", url);\n const newcid = TransportIPFS.cidFrom(url); // Its a CID which has a buffer in it\n console.assert(url === qbf_url, \"url should match url from rawstore\");\n const cidmultihash = url.split('/')[2]; // Store cid from first block in form of multihash\n const newurl = TransportIPFS.urlFrom(newcid);\n console.assert(url === newurl, \"Should round trip\");\n urlqbf = url;\n const data = await transport.p_rawfetch(urlqbf, {verbose});\n console.assert(data.toString() === qbf, \"Should fetch block stored above\");\n //console.log(\"TransportIPFS test complete\");\n return transport\n } catch(err) {\n console.log(\"Exception thrown in TransportIPFS.test:\", err.message);\n throw err;\n }\n }\n\n}\nTransports._transportclasses[\"IPFS\"] = TransportIPFS;\nexports = module.exports = TransportIPFS;\n\n/* WEBPACK VAR INJECTION */}.call(this, __webpack_require__(/*! ./../../../../usr/local/lib/node_modules/webpack/node_modules/buffer/index.js */ \"../../../../usr/local/lib/node_modules/webpack/node_modules/buffer/index.js\").Buffer))\n\n//# sourceURL=webpack:///./TransportIPFS.js?"); /***/ }), @@ -766,7 +766,7 @@ eval("const Url = __webpack_require__(/*! url */ \"../../../../usr/local/lib/nod /*! no static exports found */ /***/ (function(module, exports, __webpack_require__) { -eval("/* WEBPACK VAR INJECTION */(function(Buffer) {const nodefetch = __webpack_require__(/*! node-fetch */ 2); // Note, were using node-fetch-npm which had a warning in webpack see https://github.com/bitinn/node-fetch/issues/421 and is intended for clients\nconst errors = __webpack_require__(/*! ./Errors */ \"./Errors.js\"); // Standard Dweb Errors\n\n//var fetch,Headers,Request;\n//if (typeof(Window) === \"undefined\") {\nif (typeof(fetch) === \"undefined\") {\n //var fetch = require('whatwg-fetch').fetch; //Not as good as node-fetch-npm, but might be the polyfill needed for browser.safari\n //XMLHttpRequest = require(\"xmlhttprequest\").XMLHttpRequest; // Note this doesnt work if set to a var or const, needed by whatwg-fetch\n fetch = nodefetch;\n Headers = fetch.Headers; // A class\n Request = fetch.Request; // A class\n} /* else {\n // If on a browser, need to find fetch,Headers,Request in window\n console.log(\"Loading browser version of fetch,Headers,Request\");\n fetch = window.fetch;\n Headers = window.Headers;\n Request = window.Request;\n} */\n//TODO-HTTP to work on Safari or mobile will require a polyfill, see https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch for comment\n\n\nhttptools = {};\n\nhttptools.p_httpfetch = async function(httpurl, init, verbose) { // Embrace and extend \"fetch\" to check result etc.\n /*\n Fetch a url\n\n url: optional (depends on command)\n resolves to: data as text or json depending on Content-Type header\n throws: TransportError if fails to fetch\n */\n try {\n if (verbose) console.log(\"httpurl=%s init=%o\", httpurl, init);\n //console.log('CTX=',init[\"headers\"].get('Content-Type'))\n // Using window.fetch, because it doesn't appear to be in scope otherwise in the browser.\n let response = await fetch(new Request(httpurl, init));\n // fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch)\n // Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer.\n if (response.ok) {\n let contenttype = response.headers.get('Content-Type');\n if (contenttype === \"application/json\") {\n return response.json(); // promise resolving to JSON\n } else if (contenttype.startsWith(\"text\")) { // Note in particular this is used for responses to store\n return response.text();\n } else { // Typically application/octetStream when don't know what fetching\n return new Buffer(await response.arrayBuffer()); // Convert arrayBuffer to Buffer which is much more usable currently\n }\n }\n // noinspection ExceptionCaughtLocallyJS\n throw new errors.TransportError(`Transport Error ${response.status}: ${response.statusText}`);\n } catch (err) {\n // Error here is particularly unhelpful - if rejected during the COrs process it throws a TypeError\n console.log(\"Note error from fetch might be misleading especially TypeError can be Cors issue:\",httpurl);\n if (err instanceof errors.TransportError) {\n throw err;\n } else {\n throw new errors.TransportError(`Transport error thrown by ${httpurl}: ${err.message}`);\n }\n }\n}\n\n\nhttptools.p_GET = async function(httpurl, opts={}) {\n /* Locate and return a block, based on its url\n Throws TransportError if fails\n opts {\n start, end, // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes\n verbose }\n resolves to: URL that can be used to fetch the resource, of form contenthash:/contenthash/Q123\n */\n let headers = new Headers();\n if (opts.start || opts.end) headers.append(\"range\", `bytes=${opts.start || 0}-${opts.end || \"\"}`);\n let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch\n method: 'GET',\n headers: headers,\n mode: 'cors',\n cache: 'default',\n redirect: 'follow', // Chrome defaults to manual\n keepalive: true // Keep alive - mostly we'll be going back to same places a lot\n };\n return await httptools.p_httpfetch(httpurl, init, opts.verbose); // This s a real http url\n}\nhttptools.p_POST = async function(httpurl, type, data, verbose) {\n // Locate and return a block, based on its url\n // Throws TransportError if fails\n //let headers = new window.Headers();\n //headers.set('content-type',type); Doesn't work, it ignores it\n let init = {\n //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch\n //https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name for headers tat cant be set\n method: 'POST',\n headers: {}, //headers,\n //body: new Buffer(data),\n body: data,\n mode: 'cors',\n cache: 'default',\n redirect: 'follow', // Chrome defaults to manual\n keepalive: true // Keep alive - mostly we'll be going back to same places a lot\n };\n return await httptools.p_httpfetch(httpurl, init, verbose);\n}\n\nexports = module.exports = httptools;\n/* WEBPACK VAR INJECTION */}.call(this, __webpack_require__(/*! ./../../../../usr/local/lib/node_modules/webpack/node_modules/buffer/index.js */ \"../../../../usr/local/lib/node_modules/webpack/node_modules/buffer/index.js\").Buffer))\n\n//# sourceURL=webpack:///./httptools.js?"); +eval("/* WEBPACK VAR INJECTION */(function(Buffer) {const nodefetch = __webpack_require__(/*! node-fetch */ 2); // Note, were using node-fetch-npm which had a warning in webpack see https://github.com/bitinn/node-fetch/issues/421 and is intended for clients\nconst errors = __webpack_require__(/*! ./Errors */ \"./Errors.js\"); // Standard Dweb Errors\n\n//var fetch,Headers,Request;\n//if (typeof(Window) === \"undefined\") {\nif (typeof(fetch) === \"undefined\") {\n //var fetch = require('whatwg-fetch').fetch; //Not as good as node-fetch-npm, but might be the polyfill needed for browser.safari\n //XMLHttpRequest = require(\"xmlhttprequest\").XMLHttpRequest; // Note this doesnt work if set to a var or const, needed by whatwg-fetch\n fetch = nodefetch;\n Headers = fetch.Headers; // A class\n Request = fetch.Request; // A class\n} /* else {\n // If on a browser, need to find fetch,Headers,Request in window\n console.log(\"Loading browser version of fetch,Headers,Request\");\n fetch = window.fetch;\n Headers = window.Headers;\n Request = window.Request;\n} */\n//TODO-HTTP to work on Safari or mobile will require a polyfill, see https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch for comment\n\n\nhttptools = {};\n\nasync function loopfetch(req, ms, count, what) {\n let lasterr;\n while (count--) {\n try {\n return await fetch(req);\n } catch(err) {\n lasterr = err;\n console.log(\"Delaying\", what,\"by\", ms, \"because\", err.message);\n await new Promise(resolve => {setTimeout(() => { resolve(); },ms)})\n ms = ms*(1+Math.random()); // Spread out delays incase all requesting same time\n }\n }\n console.log(\"Looping\",what,\"failed\");\n throw(lasterr);\n}\n\nhttptools.p_httpfetch = async function(httpurl, init, verbose) { // Embrace and extend \"fetch\" to check result etc.\n /*\n Fetch a url\n\n url: optional (depends on command)\n resolves to: data as text or json depending on Content-Type header\n throws: TransportError if fails to fetch\n */\n try {\n if (verbose) console.log(\"httpurl=%s init=%o\", httpurl, init);\n //console.log('CTX=',init[\"headers\"].get('Content-Type'))\n // Using window.fetch, because it doesn't appear to be in scope otherwise in the browser.\n let req = new Request(httpurl, init);\n //let response = await fetch(new Request(httpurl, init)).catch(err => console.exception(err));\n let response = await loopfetch(req, 500, 10, \"fetching \"+httpurl);\n // fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch)\n // Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer.\n if (response.ok) {\n let contenttype = response.headers.get('Content-Type');\n if (contenttype === \"application/json\") {\n return response.json(); // promise resolving to JSON\n } else if (contenttype.startsWith(\"text\")) { // Note in particular this is used for responses to store\n return response.text();\n } else { // Typically application/octetStream when don't know what fetching\n return new Buffer(await response.arrayBuffer()); // Convert arrayBuffer to Buffer which is much more usable currently\n }\n }\n // noinspection ExceptionCaughtLocallyJS\n throw new errors.TransportError(`Transport Error ${response.status}: ${response.statusText}`);\n } catch (err) {\n // Error here is particularly unhelpful - if rejected during the COrs process it throws a TypeError\n console.log(\"Note error from fetch might be misleading especially TypeError can be Cors issue:\",httpurl);\n if (err instanceof errors.TransportError) {\n throw err;\n } else {\n throw new errors.TransportError(`Transport error thrown by ${httpurl}: ${err.message}`);\n }\n }\n}\n\n\nhttptools.p_GET = async function(httpurl, opts={}) {\n /* Locate and return a block, based on its url\n Throws TransportError if fails\n opts {\n start, end, // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes\n verbose }\n resolves to: URL that can be used to fetch the resource, of form contenthash:/contenthash/Q123\n */\n let headers = new Headers();\n if (opts.start || opts.end) headers.append(\"range\", `bytes=${opts.start || 0}-${opts.end || \"\"}`);\n let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch\n method: 'GET',\n headers: headers,\n mode: 'cors',\n cache: 'default',\n redirect: 'follow', // Chrome defaults to manual\n keepalive: true // Keep alive - mostly we'll be going back to same places a lot\n };\n return await httptools.p_httpfetch(httpurl, init, opts.verbose); // This s a real http url\n}\nhttptools.p_POST = async function(httpurl, type, data, verbose) {\n // Locate and return a block, based on its url\n // Throws TransportError if fails\n //let headers = new window.Headers();\n //headers.set('content-type',type); Doesn't work, it ignores it\n let init = {\n //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch\n //https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name for headers tat cant be set\n method: 'POST',\n headers: {}, //headers,\n //body: new Buffer(data),\n body: data,\n mode: 'cors',\n cache: 'default',\n redirect: 'follow', // Chrome defaults to manual\n keepalive: true // Keep alive - mostly we'll be going back to same places a lot\n };\n return await httptools.p_httpfetch(httpurl, init, verbose);\n}\n\nexports = module.exports = httptools;\n/* WEBPACK VAR INJECTION */}.call(this, __webpack_require__(/*! ./../../../../usr/local/lib/node_modules/webpack/node_modules/buffer/index.js */ \"../../../../usr/local/lib/node_modules/webpack/node_modules/buffer/index.js\").Buffer))\n\n//# sourceURL=webpack:///./httptools.js?"); /***/ }), diff --git a/httptools.js b/httptools.js index e1c9985..6146125 100644 --- a/httptools.js +++ b/httptools.js @@ -21,6 +21,22 @@ if (typeof(fetch) === "undefined") { httptools = {}; +async function loopfetch(req, ms, count, what) { + let lasterr; + while (count--) { + try { + return await fetch(req); + } catch(err) { + lasterr = err; + console.log("Delaying", what,"by", ms, "because", err.message); + await new Promise(resolve => {setTimeout(() => { resolve(); },ms)}) + ms = ms*(1+Math.random()); // Spread out delays incase all requesting same time + } + } + console.log("Looping",what,"failed"); + throw(lasterr); +} + httptools.p_httpfetch = async function(httpurl, init, verbose) { // Embrace and extend "fetch" to check result etc. /* Fetch a url @@ -33,7 +49,9 @@ httptools.p_httpfetch = async function(httpurl, init, verbose) { // Embrace and if (verbose) console.log("httpurl=%s init=%o", httpurl, init); //console.log('CTX=',init["headers"].get('Content-Type')) // Using window.fetch, because it doesn't appear to be in scope otherwise in the browser. - let response = await fetch(new Request(httpurl, init)); + let req = new Request(httpurl, init); + //let response = await fetch(new Request(httpurl, init)).catch(err => console.exception(err)); + let response = await loopfetch(req, 500, 10, "fetching "+httpurl); // fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch) // Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer. if (response.ok) {