fluencelabs/dweb-transports
1
0
Fork 0
mirror of https://github.com/fluencelabs/dweb-transports synced 2025-04-25 06:42:17 +00:00
Code Issues Projects Releases Wiki Activity

Split httptools out of TransportIPFS and use as fallback

Browse Source
This commit is contained in:
Mitra Ardron 2018-05-08 16:46:57 -07:00
parent 027f94c867
commit 8eb6faa9ad
3 changed files with 155 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

120
TransportHTTP.js
View File

@ -1,26 +1,8 @@
const errors = require('./Errors'); // Standard Dweb Errors
const Transport = require('./Transport'); // Base class for TransportXyz
const Transports = require('./Transports'); // Manage all Transports that are loaded
const nodefetch = require('node-fetch'); // Note, were using node-fetch-npm which had a warning in webpack see https://github.com/bitinn/node-fetch/issues/421 and is intended for clients
const httptools = require('./httptools'); // Expose some of the httptools so that IPFS can use it as a backup
const Url = require('url');
//var fetch,Headers,Request;
//if (typeof(Window) === "undefined") {
if (typeof(fetch) === "undefined") {
//var fetch = require('whatwg-fetch').fetch; //Not as good as node-fetch-npm, but might be the polyfill needed for browser.safari
//XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest; // Note this doesnt work if set to a var or const, needed by whatwg-fetch
console.log("Node loaded");
fetch = nodefetch;
Headers = fetch.Headers; // A class
Request = fetch.Request; // A class
} /* else {
// If on a browser, need to find fetch,Headers,Request in window
console.log("Loading browser version of fetch,Headers,Request");
fetch = window.fetch;
Headers = window.Headers;
Request = window.Request;
} */
//TODO-HTTP to work on Safari or mobile will require a polyfill, see https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch for comment
defaulthttpoptions = {
urlbase: 'https://gateway.dweb.me:443'
@ -85,84 +67,6 @@ class TransportHTTP extends Transport {
return super.p_status(verbose);
}
async p_httpfetch(httpurl, init, verbose) { // Embrace and extend "fetch" to check result etc.
/*
Fetch a url
url: optional (depends on command)
resolves to: data as text or json depending on Content-Type header
throws: TransportError if fails to fetch
*/
try {
if (verbose) console.log("httpurl=%s init=%o", httpurl, init);
//console.log('CTX=',init["headers"].get('Content-Type'))
// Using window.fetch, because it doesn't appear to be in scope otherwise in the browser.
let response = await fetch(new Request(httpurl, init));
// fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch)
// Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer.
if (response.ok) {
let contenttype = response.headers.get('Content-Type');
if (contenttype === "application/json") {
return response.json(); // promise resolving to JSON
} else if (contenttype.startsWith("text")) { // Note in particular this is used for responses to store
return response.text();
} else { // Typically application/octetStream when don't know what fetching
return new Buffer(await response.arrayBuffer()); // Convert arrayBuffer to Buffer which is much more usable currently
}
}
// noinspection ExceptionCaughtLocallyJS
throw new errors.TransportError(`Transport Error ${response.status}: ${response.statusText}`);
} catch (err) {
// Error here is particularly unhelpful - if rejected during the COrs process it throws a TypeError
console.log("Note error from fetch might be misleading especially TypeError can be Cors issue:",httpurl);
if (err instanceof errors.TransportError) {
throw err;
} else {
throw new errors.TransportError(`Transport error thrown by ${httpurl}: ${err.message}`);
}
}
}
async p_GET(httpurl, opts={}) {
/* Locate and return a block, based on its url
Throws TransportError if fails
opts {
start, end, // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
verbose }
resolves to: URL that can be used to fetch the resource, of form contenthash:/contenthash/Q123
*/
let headers = new Headers();
if (opts.start || opts.end) headers.append("range", `bytes=${opts.start || 0}-${opts.end || ""}`);
let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
method: 'GET',
headers: headers,
mode: 'cors',
cache: 'default',
redirect: 'follow', // Chrome defaults to manual
keepalive: true // Keep alive - mostly we'll be going back to same places a lot
};
return await this.p_httpfetch(httpurl, init, opts.verbose); // This s a real http url
}
async p_POST(httpurl, type, data, verbose) {
// Locate and return a block, based on its url
// Throws TransportError if fails
//let headers = new window.Headers();
//headers.set('content-type',type); Doesn't work, it ignores it
let init = {
//https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
//https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name for headers tat cant be set
method: 'POST',
headers: {}, //headers,
//body: new Buffer(data),
body: data,
mode: 'cors',
cache: 'default',
redirect: 'follow', // Chrome defaults to manual
keepalive: true // Keep alive - mostly we'll be going back to same places a lot
};
return await this.p_httpfetch(httpurl, init, verbose);
}
_cmdurl(command) {
return `${this.urlbase}/${command}`
}
@ -190,7 +94,7 @@ class TransportHTTP extends Transport {
table: "keyvaluetable",
}
} else {
return await this.p_GET(this._url(url, servercommands.rawfetch), opts);
return await httptools.p_GET(this._url(url, servercommands.rawfetch), opts);
}
}
@ -198,7 +102,7 @@ class TransportHTTP extends Transport {
// obj being loaded
// Locate and return a block, based on its url
if (!url) throw new errors.CodingError("TransportHTTP.p_rawlist: requires url");
return this.p_GET(this._url(url, servercommands.rawlist), {verbose});
return httptools.p_GET(this._url(url, servercommands.rawlist), {verbose});
}
rawreverse() { throw new errors.ToBeImplementedError("Undefined function TransportHTTP.rawreverse"); }
@ -211,7 +115,7 @@ class TransportHTTP extends Transport {
*/
//PY: res = self._sendGetPost(True, "rawstore", headers={"Content-Type": "application/octet-stream"}, urlargs=[], data=data, verbose=verbose)
console.assert(data, "TransportHttp.p_rawstore: requires data");
let res = await this.p_POST(this._cmdurl(servercommands.rawstore), "application/octet-stream", data, verbose); // resolves to URL
let res = await httptools.p_POST(this._cmdurl(servercommands.rawstore), "application/octet-stream", data, verbose); // resolves to URL
let parsedurl = Url.parse(res);
let pathparts = parsedurl.pathname.split('/');
return `contenthash:/contenthash/${pathparts.slice(-1)}`
@ -223,7 +127,7 @@ class TransportHTTP extends Transport {
if (!url || !sig) throw new errors.CodingError("TransportHTTP.p_rawadd: invalid parms",url, sig);
if (verbose) console.log("rawadd", url, sig);
let value = JSON.stringify(sig.preflight(Object.assign({},sig)))+"\n";
return this.p_POST(this._url(url, servercommands.rawadd), "application/json", value, verbose); // Returns immediately
return httptools.p_POST(this._url(url, servercommands.rawadd), "application/json", value, verbose); // Returns immediately
}
p_newlisturls(cl, {verbose=false}={}) {
@ -266,10 +170,10 @@ class TransportHTTP extends Transport {
if (verbose) console.log("p_set", url, keyvalues, value);
if (typeof keyvalues === "string") {
let kv = JSON.stringify([{key: keyvalues, value: value}]);
await this.p_POST(this._url(url, servercommands.set), "application/json", kv, verbose); // Returns immediately
await httptools.p_POST(this._url(url, servercommands.set), "application/json", kv, verbose); // Returns immediately
} else {
let kv = JSON.stringify(Object.keys(keyvalues).map((k) => ({"key": k, "value": keyvalues[k]})));
await this.p_POST(this._url(url, servercommands.set), "application/json", kv, verbose); // Returns immediately
await httptools.p_POST(this._url(url, servercommands.set), "application/json", kv, verbose); // Returns immediately
}
}
@ -279,23 +183,23 @@ class TransportHTTP extends Transport {
async p_get(url, keys, {verbose=false}={}) {
if (!url && keys) throw new errors.CodingError("TransportHTTP.p_get: requires url and at least one key");
let parmstr =Array.isArray(keys) ? keys.map(k => this._keyparm(k)).join('&') : this._keyparm(keys)
let res = await this.p_GET(this._url(url, servercommands.get, parmstr), {verbose});
let res = await httptools.p_GET(this._url(url, servercommands.get, parmstr), {verbose});
return Array.isArray(keys) ? res : res[keys]
}
async p_delete(url, keys, {verbose=false}={}) { //TODO-KEYVALUE-API need to think this one through
if (!url && keys) throw new errors.CodingError("TransportHTTP.p_get: requires url and at least one key");
let parmstr = keys.map(k => this._keyparm(k)).join('&');
await this.p_GET(this._url(url, servercommands.delete, parmstr), {verbose});
await httptools.p_GET(this._url(url, servercommands.delete, parmstr), {verbose});
}
async p_keys(url, {verbose=false}={}) {
if (!url && keys) throw new errors.CodingError("TransportHTTP.p_get: requires url and at least one key");
return await this.p_GET(this._url(url, servercommands.keys), {verbose});
return await httptools.p_GET(this._url(url, servercommands.keys), {verbose});
}
async p_getall(url, {verbose=false}={}) {
if (!url && keys) throw new errors.CodingError("TransportHTTP.p_get: requires url and at least one key");
return await this.p_GET(this._url(url, servercommands.getall), {verbose});
return await httptools.p_GET(this._url(url, servercommands.getall), {verbose});
}
/* Make sure doesnt shadow regular p_rawfetch
async p_rawfetch(url, verbose) {
@ -306,7 +210,7 @@ class TransportHTTP extends Transport {
}
*/
p_info(verbose) { return this.p_GET(`${this.urlbase}/info`, {verbose}); }
p_info(verbose) { return httptools.p_GET(`${this.urlbase}/info`, {verbose}); }
static async p_test(opts={}, verbose=false) {
if (verbose) {console.log("TransportHTTP.test")}

52
TransportIPFS.js
View File

@ -3,8 +3,9 @@ This is a shim to the IPFS library, (Lists are handled in YJS or OrbitDB)
See https://github.com/ipfs/js-ipfs but note its often out of date relative to the generic API doc.
*/
// IPFS components
const httptools = require('./httptools'); // Expose some of the httptools so that IPFS can use it as a backup
// IPFS components
const IPFS = require('ipfs');
const CID = require('cids');
//Removed next two as not needed if use "Kludge" flagged below.
@ -158,32 +159,48 @@ class TransportIPFS extends Transport {
if (typeof(url) === "string") url = Url.parse(url);
if (url && url["pathname"]) { // On browser "instanceof Url" isn't valid)
const patharr = url.pathname.split('/');
if ((url.protocol !== "ipfs:") || (patharr[1] !== 'ipfs') || (patharr.length < 3))
throw new errors.TransportError("TransportIPFS.cidFrom bad format for url should be ipfs:/ipfs/...: " + url.href);
if ((!["ipfs:","dweb:"].includes(url.protocol)) || (patharr[1] !== 'ipfs') || (patharr.length < 3))
throw new errors.TransportError("TransportIPFS.cidFrom bad format for url should be dweb: or ipfs:/ipfs/...: " + url.href);
if (patharr.length > 3)
throw new errors.TransportError("TransportIPFS.cidFrom not supporting paths in url yet, should be ipfs:/ipfs/...: " + url.href);
throw new errors.TransportError("TransportIPFS.cidFrom not supporting paths in url yet, should be dweb: or ipfs:/ipfs/...: " + url.href);
return new CID(patharr[2]);
} else {
throw new errors.CodingError("TransportIPFS.cidFrom: Cant convert url",url);
}
}
static _stringFrom(url) {
// Tool for ipfsFrom and ipfsGatewayFrom
if (url instanceof CID)
return "/ipfs/"+url.toBaseEncodedString();
if (typeof url === 'object' && url.path) { // It better be URL which unfortunately is hard to test
return url.path;
}
}
static ipfsFrom(url) {
/*
Convert to a ipfspath i.e. /ipfs/Qm....
Required because of strange differences in APIs between files.cat and dag.get see https://github.com/ipfs/js-ipfs/issues/1229
*/
if (url instanceof CID)
return "/ipfs/"+url.toBaseEncodedString();
if (typeof(url) !== "string") { // It better be URL which unfortunately is hard to test
url = url.path;
}
url = this._stringFrom(url); // Convert CID or Url to a string hopefully containing /ipfs/
if (url.indexOf('/ipfs/') > -1) {
return url.slice(url.indexOf('/ipfs/'));
}
throw new errors.CodingError(`TransportIPFS.ipfsFrom: Cant convert url ${url} into a path starting /ipfs/`);
}
static ipfsGatewayFrom(url) { //TODO-API
/*
url: CID, Url, or a string
returns: https://ipfs.io/ipfs/<cid>
*/
url = this._stringFrom(url); // Convert CID or Url to a string hopefully containing /ipfs/
if (url.indexOf('/ipfs/') > -1) {
return "https://ipfs.io" + url.slice(url.indexOf('/ipfs/'));
}
throw new errors.CodingError(`TransportIPFS.ipfsGatewayFrom: Cant convert url ${url} into a path starting /ipfs/`);
}
static multihashFrom(url) {
if (url instanceof CID)
return cid.toBaseEncodedString();
@ -207,7 +224,7 @@ class TransportIPFS extends Transport {
Returns a new Promise that resolves currently to a string.
There may also be need for a streaming version of this call, at this point undefined since we havent (currently) got a use case..
:param string url: URL of object being retrieved
:param string url: URL of object being retrieved {ipfs|dweb}:/ipfs/<cid> or /
:param boolean verbose: true for debugging output
:resolve buffer: Return the object being fetched. (may in the future return a stream and buffer externally)
:throws: TransportError if url invalid - note this happens immediately, not as a catch in the promise
@ -218,7 +235,7 @@ class TransportIPFS extends Transport {
const ipfspath = TransportIPFS.ipfsFrom(url) // Need because dag.get has different requirement than file.cat
try {
const res = await utils.p_timeout(this.ipfs.dag.get(cid), timeoutMS);
const res = await utils.p_timeout(this.ipfs.dag.get(cid), timeoutMS); // Will reject and throw TimeoutError if times out
// noinspection Annotator
if (res.remainderPath.length)
{ // noinspection ExceptionCaughtLocallyJS
@ -247,9 +264,16 @@ class TransportIPFS extends Transport {
}
if (verbose) console.log(`IPFS fetched ${buff.length} from ${ipfspath}`);
return buff;
} catch (err) {
console.log("Caught misc error in TransportIPFS.p_rawfetch");
throw err;
} catch (err) { // TimeoutError or could be some other error from IPFS etc
console.log("Caught misc error in TransportIPFS.p_rawfetch trying IPFS", err.message);
try {
return await utils.p_timeout(
httptools.p_GET(TransportIPFS.ipfsGatewayFrom(url)), // Returns a buffer
timeoutMS)
} catch (err) {
console.log("Caught misc error in TransportIPFS.p_rawfetch trying gateway", err.message);
throw err;
}
}
}

105
httptools.js Normal file
View File

@ -0,0 +1,105 @@
const nodefetch = require('node-fetch'); // Note, were using node-fetch-npm which had a warning in webpack see https://github.com/bitinn/node-fetch/issues/421 and is intended for clients
const errors = require('./Errors'); // Standard Dweb Errors
//TODO-API move separate out httptools to own part of API.md
//var fetch,Headers,Request;
//if (typeof(Window) === "undefined") {
if (typeof(fetch) === "undefined") {
//var fetch = require('whatwg-fetch').fetch; //Not as good as node-fetch-npm, but might be the polyfill needed for browser.safari
//XMLHttpRequest = require("xmlhttprequest").XMLHttpRequest; // Note this doesnt work if set to a var or const, needed by whatwg-fetch
fetch = nodefetch;
Headers = fetch.Headers; // A class
Request = fetch.Request; // A class
} /* else {
// If on a browser, need to find fetch,Headers,Request in window
console.log("Loading browser version of fetch,Headers,Request");
fetch = window.fetch;
Headers = window.Headers;
Request = window.Request;
} */
//TODO-HTTP to work on Safari or mobile will require a polyfill, see https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch for comment
httptools = {};
httptools.p_httpfetch = async function(httpurl, init, verbose) { // Embrace and extend "fetch" to check result etc.
/*
Fetch a url
url: optional (depends on command)
resolves to: data as text or json depending on Content-Type header
throws: TransportError if fails to fetch
*/
try {
if (verbose) console.log("httpurl=%s init=%o", httpurl, init);
//console.log('CTX=',init["headers"].get('Content-Type'))
// Using window.fetch, because it doesn't appear to be in scope otherwise in the browser.
let response = await fetch(new Request(httpurl, init));
// fetch throws (on Chrome, untested on Firefox or Node) TypeError: Failed to fetch)
// Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer.
if (response.ok) {
let contenttype = response.headers.get('Content-Type');
if (contenttype === "application/json") {
return response.json(); // promise resolving to JSON
} else if (contenttype.startsWith("text")) { // Note in particular this is used for responses to store
return response.text();
} else { // Typically application/octetStream when don't know what fetching
return new Buffer(await response.arrayBuffer()); // Convert arrayBuffer to Buffer which is much more usable currently
}
}
// noinspection ExceptionCaughtLocallyJS
throw new errors.TransportError(`Transport Error ${response.status}: ${response.statusText}`);
} catch (err) {
// Error here is particularly unhelpful - if rejected during the COrs process it throws a TypeError
console.log("Note error from fetch might be misleading especially TypeError can be Cors issue:",httpurl);
if (err instanceof errors.TransportError) {
throw err;
} else {
throw new errors.TransportError(`Transport error thrown by ${httpurl}: ${err.message}`);
}
}
}
httptools.p_GET = async function(httpurl, opts={}) {
/* Locate and return a block, based on its url
Throws TransportError if fails
opts {
start, end, // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
verbose }
resolves to: URL that can be used to fetch the resource, of form contenthash:/contenthash/Q123
*/
let headers = new Headers();
if (opts.start || opts.end) headers.append("range", `bytes=${opts.start || 0}-${opts.end || ""}`);
let init = { //https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
method: 'GET',
headers: headers,
mode: 'cors',
cache: 'default',
redirect: 'follow', // Chrome defaults to manual
keepalive: true // Keep alive - mostly we'll be going back to same places a lot
};
return await httptools.p_httpfetch(httpurl, init, opts.verbose); // This s a real http url
}
httptools.p_POST = async function(httpurl, type, data, verbose) {
// Locate and return a block, based on its url
// Throws TransportError if fails
//let headers = new window.Headers();
//headers.set('content-type',type); Doesn't work, it ignores it
let init = {
//https://developer.mozilla.org/en-US/docs/Web/API/WindowOrWorkerGlobalScope/fetch
//https://developer.mozilla.org/en-US/docs/Glossary/Forbidden_header_name for headers tat cant be set
method: 'POST',
headers: {}, //headers,
//body: new Buffer(data),
body: data,
mode: 'cors',
cache: 'default',
redirect: 'follow', // Chrome defaults to manual
keepalive: true // Keep alive - mostly we'll be going back to same places a lot
};
return await httptools.p_httpfetch(httpurl, init, verbose);
}
exports = module.exports = httptools;