v0.1.42 Improve behavior when cant reach gateway

README.md

@@ -126,16 +126,17 @@ See [Dweb document index](./DOCUMENTINDEX.md) for a list of the repos that make
 
 ### Release Notes
 
-* 0.1.33: Bug fixes; support for gatewayUrls (for dweb-mirror)
+* 0.1.42: Better behavior when cant see gateway
-* 0.1.35: package update (note wont work with latest versions of yjs or uglify)
+* 0.1.41: Remove createReadStream for browser (it was added for node in 0.1.40), add fetch(url,opts,cb)
-* 0.1.36: Made httptools accessable at Transports.httptools so it doesnt have to be separately 'require'd
-* 0.1.37: IPFS - dont stop it if we didnt start it (were stopping via API)
-* 0.1.37: Start move to unpromisify pattern v5
-* 0.1.37: IPFS - updated to (significant) v0.34.0 API changes
-* 0.1.38: httptools - adds retries
-* 0.1.38: WOLK - added to the library
-* 0.1.39: WOLK - updated wolk.js module to fix bugs
 * 0.1.40: Bug fix in httpfetch({count=0}),
 * 0.1.40: Added support for "seed" and tested in IPFS
 * 0.1.40: WOLK - moved to their production sys and master branch
-* 0.1.41: Remove createReadStream for browser (it was added for node in 0.1.40), add fetch(url,opts,cb)
+* 0.1.39: WOLK - updated wolk.js module to fix bugs
+* 0.1.38: httptools - adds retries
+* 0.1.38: WOLK - added to the library
+* 0.1.37: IPFS - dont stop it if we didnt start it (were stopping via API)
+* 0.1.37: Start move to unpromisify pattern v5
+* 0.1.37: IPFS - updated to (significant) v0.34.0 API changes
+* 0.1.36: Made httptools accessable at Transports.httptools so it doesnt have to be separately 'require'd
+* 0.1.35: package update (note wont work with latest versions of yjs or uglify)
+* 0.1.33: Bug fixes; support for gatewayUrls (for dweb-mirror)

Transport.js

@@ -92,7 +92,11 @@ class Transport {
         return this.status;
     }
 
-    supports(url, func) {
+    connected() {
+        // True if connected (status==STATUS_CONNECTED==0) should not need subclassing
+        return ! this.status;
+    }
+    supports(url, func) { //TODO-API
         /*
         Determine if this transport supports a certain set of URLs and a func
 
@@ -111,6 +115,13 @@ class Transport {
             && (!func || this.supportFunctions.includes(func)))
     }
 
+    validFor(url, func) { //TODO-API
+        // By default a transport can handle a url and a func if its connected and supports that url/func
+        // This shouldnt need subclassing, an exception is HTTP which only applies "connected" against urls heading for the gateway
+        return this.connected() && this.supports(url, func);
+    }
+
+
     p_rawstore(data, opts) {
         /*
         Store a blob of data onto the decentralised transport.

TransportHTTP.js

@@ -8,7 +8,7 @@ const stringify = require('canonical-json');
 
 
 defaulthttpoptions = {
-    urlbase: 'https://dweb.me:443'
+    urlbase: 'https://dweb.me'
 };
 
 servercommands = {  // What the server wants to see to return each of these
@@ -29,7 +29,7 @@ class TransportHTTP extends Transport {
     constructor(options) {
         super(options); // These are now options.http
         this.options = options;
-        this.urlbase = options.urlbase;
+        this.urlbase = options.urlbase; // e.g. https://dweb.me
         this.supportURLs = ['contenthash', 'http','https'];
         this.supportFunctions = ['fetch', 'store', 'add', 'list', 'reverse', 'newlisturls', "get", "set", "keys", "getall", "delete", "newtable", "newdatabase"]; //Does not support: listmonitor - reverse is disabled somewhere not sure if here or caller
         if (typeof window === "undefined") {
@@ -86,6 +86,12 @@ class TransportHTTP extends Transport {
         url = url + (parmstr ? "?"+parmstr : "");
         return url;
     }
+
+    validFor(url, func) {
+        // Overrides Transport.prototype.validFor because HTTP's connection test is only really for dweb.me
+        // in particular this allows urls like https://be-api.us.archive.org
+        return (this.connected() || (url.protocol.startsWith("http") && ! url.href.startsWith(this.urlbase))) && this.supports(url, func);
+    }
     // noinspection JSCheckFunctionSignatures
     async p_rawfetch(url, opts={}) {
         /*
@@ -304,7 +310,7 @@ class TransportHTTP extends Transport {
     }
     */
 
-    p_info() { return httptools.p_GET(`${this.urlbase}/info`); }
+    p_info() { return httptools.p_GET(`${this.urlbase}/info`, {retries: 5}); } // Try info, but dont wait more than approx 10secs
 
     static async p_test(opts={}) {
         {console.log("TransportHTTP.test")}

Transports.js

@@ -61,14 +61,14 @@ class Transports {
             console.error("Transports.validFor called with invalid arguments: urls=", urls, "func=", func); // FOr debugging old calling patterns with [ undefined ]
             return [];
         }
-        if (!(urls && urls.length > 0)) {
+        if (!(urls && urls.length > 0)) { // No url supplied we are just checking which transports support this function on no url.
-            return this._connected().filter((t) => (t.supports(undefined, func)))
+            return this._transports.filter((t) => (t.validFor(undefined, func)))
                 .map((t) => [undefined, t]);
         } else {
             return [].concat(
                 ...urls.map((url) => typeof url === 'string' ? Url.parse(url) : url) // parse URLs once
                     .map((url) =>
-                        this._connected().filter((t) => (t.supports(url, func))) // [ t1, t2 ]
+                        this._transports.filter((t) => (t.validFor(url, func))) // [ t1, t2 ]
                             .map((t) => [url, t]))); // [[ u, t1], [u, t2]]
         }
     }
@@ -338,34 +338,34 @@ class Transports {
         wanturl True if want the URL of the stream (for service workers)
         returns:    f(opts) => stream returning bytes from opts.start || start of file to opts.end-1 || end of file
          */
+        // Find all the transports that CAN support this request
         let tt = this.validFor(urls, "createReadStream", {}); //[ [Url,t],[Url,t]]  // Can pass options TODO-STREAM support options in validFor
         if (!tt.length) {
-            debug("Opening stream to %o failed: no transports available", urls);
+            debug("Opening stream from %o failed: no transports available", urls);
             throw new errors.TransportError("Transports.p_createReadStream cant find any transport for urls: " + urls);
         }
         //With multiple transports, it should return when the first one returns something.
         let errs = [];
-        // Until we have transport ordering, try randomly TODO Transport ordering
 
-        //Debugging: preferredTransports = [] // ["WEBTORRENT", "IPFS", "HTTP"];
+        // Select first from preferredTransports in the order presented, then the rest at random
         tt.sort((a,b) =>
             ((preferredTransports.indexOf(a[1].name)+1) || 999+Math.random())  - ((preferredTransports.indexOf(b[1].name)+1) || 999+Math.random())
         );
 
         for (const [url, t] of tt) {
             try {
-                debug("Opening stream to %s via %s", url.href, t.name);
+                debug("Opening stream from %s via %s", url.href, t.name);
                 let res = await t.p_f_createReadStream(url, {wanturl} );
-                debug("Opening stream to %s via %s succeeded", url.href, t.name);
+                debug("Opening stream from %s via %s succeeded", url.href, t.name);
                 return res;
             } catch (err) {
                 errs.push(err);
-                debug("Opening stream to %s via %s failed: %s", url.href, t.name, err.message);
+                debug("Opening stream from %s via %s failed: %s", url.href, t.name, err.message);
                 // Don't throw anything here, loop round for next, only throw if drop out bottom
                 //TODO-MULTI-GATEWAY potentially copy from success to failed URLs.
             }
         }
-        debug("Opening stream to %o failed on all transports", urls);
+        debug("Opening stream from %o failed on all transports", urls);
         throw new errors.TransportError(errs.map((err)=>err.message).join(', '));  //Throw err with combined messages if none succeed
     }
     static createReadStream(urls, opts, cb) {

httptools.js

@@ -42,7 +42,7 @@ async function loopfetch(req, ms, count, what) {
             lasterr = err;
             debug("Delaying %s by %d ms because %s", what, ms, err.message);
             await new Promise(resolve => {setTimeout(() => { resolve(); },ms)})
-            ms = ms*(1+Math.random()); // Spread out delays incase all requesting same time
+            ms = Math.floor(ms*(1+Math.random())); // Spread out delays incase all requesting same time
         }
     }
     console.warn("loopfetch of",what,"failed");

package.json

@@ -11,8 +11,9 @@
     "url": "https://github.com/internetarchive/dweb-transports/issues"
   },
   "dependencies": {
-    "canonical-json": "latest",
+    "canonical-json": "^0.0.4",
     "cids": "^0.5.7",
+    "debug": "^4.1.1",
     "gun": "^0.9.9999991",
     "ipfs": "^0.34.4",
     "ipfs-http-client": "^29.1.1",
@@ -52,5 +53,5 @@
     "test": "cd src; node ./test.js",
     "help": "echo 'test (test it)'; echo 'build (creates dweb-transports-bundle)'"
   },
-  "version": "0.1.41"
+  "version": "0.1.42"
 }