Add streaming to TransportHTTP and p_GET and p_httpfetch

API.md

@@ -432,7 +432,7 @@ relay        If first transport fails, try and retrieve on 2nd, then store on 1s
 A utility class to support HTTP with or without TransportHTTP
 e.g. `httptools.http().p_httpfetch("http://foo.com/bar", {method: 'GET'} )`
 
-##### p_httpfetch(url, init, verbose)
+##### p_httpfetch(url, init, {verbose)}
 Fetch a url.
 If the result
 

TransportHTTP.js

@@ -144,6 +144,63 @@ class TransportHTTP extends Transport {
         return [u,u];
     }
 
+    // ============================== Stream support
+
+    /*
+      Code disabled until have a chance to test it with <VIDEO> tag etc, problem is that it returns p_createReadStream whch is async
+      if need sync, look at WebTorrent and how it buffers through a stream which can be returned immediately
+     */
+    async p_f_createReadStream(url, {verbose=false, wanturl=false}={}) {
+        /*
+        Fetch bytes progressively, using a node.js readable stream, based on a url of the form:
+        No assumption is made about the data in terms of size or structure.
+
+        This is the initialisation step, which returns a function suitable for <VIDEO>
+
+        Returns a new Promise that resolves to function for a node.js readable stream.
+
+        Node.js readable stream docs: https://nodejs.org/api/stream.html#stream_readable_streams
+
+        :param string url: URL of object being retrieved of form  magnet:xyzabc/path/to/file  (Where xyzabc is the typical magnet uri contents)
+        :param boolean verbose: true for debugging output
+        :resolves to: f({start, end}) => stream (The readable stream.)
+        :throws:        TransportError if url invalid - note this happens immediately, not as a catch in the promise
+         */
+        if (verbose) console.log(this.name, "p_f_createreadstream %o", Url.parse(url).href);
+        try {
+            let self = this;
+            if (wanturl) {
+                return url;
+            } else {
+                return function (opts) { return self.p_createReadStream(url, opts, verbose); };
+            }
+        } catch(err) {
+            console.warn(`p_f_createReadStream failed on ${Url.parse(url).href} ${err.message}`);
+            throw(err);
+        }
+    }
+
+
+    async p_createReadStream(url, opts, verbose) {
+        /*
+        The function, encapsulated and inside another function by p_f_createReadStream (see docs)
+        NOTE THIS PROBABLY WONT WORK FOR <VIDEO> tags, but shouldnt be using it there anyway
+
+        :param file:    Webtorrent "file" as returned by webtorrentfindfile
+        :param opts: { start: byte to start from; end: optional end byte }
+        :param boolean verbose: true for debugging output
+        :resolves to stream: The readable stream.
+         */
+        if (verbose) console.log(this.name, "createreadstream %o %o", Url.parse(url).href, opts);
+        try {
+            return await httptools.p_GET(this._url(url, servercommands.rawfetch), Object.assign({wantstream: true}, opts));
+        } catch(err) {
+            console.warn(this.name, "caught error", err);
+            throw err;
+        }
+    }
+
+
     // ============================== Key Value support
 
 

httptools.js

@@ -22,6 +22,15 @@ if (typeof(fetch) === "undefined") {
 httptools = {};
 
 async function loopfetch(req, ms, count, what) {
+    /*
+    A workaround for a nasty Chrome issue which fails if there is a (cross-origin?) fetch of more than 6 files.  See other WORKAROUND-CHROME-CROSSORIGINFETCH
+    Loops at longer and longer intervals trying
+    req:        Request
+    ms:         Initial wait between polls
+    count:      Max number of times to try (0 means just once)
+    what:       Name of what retrieving for log (usually file name or URL)
+    returns Response:
+     */
     let lasterr;
     let loopguard = (typeof window != "undefined") && window.loopguard; // Optional global parameter, will cancel any loops if changes
     while (count-- && (loopguard === ((typeof window != "undefined") && window.loopguard)) ) {
@@ -43,11 +52,12 @@ async function loopfetch(req, ms, count, what) {
     }
 }
 
-httptools.p_httpfetch = async function(httpurl, init, verbose) { // Embrace and extend "fetch" to check result etc.
+httptools.p_httpfetch = async function(httpurl, init, {verbose=false, wantstream=false}={}) { // Embrace and extend "fetch" to check result etc.
     /*
     Fetch a url
 
-    url: optional (depends on command)
+    httpurl: optional (depends on command)
+    init:   {headers}
     resolves to: data as text or json depending on Content-Type header
     throws: TransportError if fails to fetch
      */
@@ -62,7 +72,9 @@ httptools.p_httpfetch = async function(httpurl, init, verbose) { // Embrace and
         // Note response.body gets a stream and response.blob gets a blob and response.arrayBuffer gets a buffer.
         if (response.ok) {
             let contenttype = response.headers.get('Content-Type');
-            if (contenttype === "application/json") {
+            if (wantstream) {
+                return response.body; // Note property while json() or text() are functions
+            } else if (contenttype === "application/json") {
                 return response.json(); // promise resolving to JSON
             } else if (contenttype.startsWith("text")) { // Note in particular this is used for responses to store
                 return response.text();
@@ -89,6 +101,7 @@ httptools.p_GET = async function(httpurl, opts={}) {
         Throws TransportError if fails
         opts {
             start, end,     // Range of bytes wanted - inclusive i.e. 0,1023 is 1024 bytes
+            wantstream,     // Return a stream rather than data
             verbose }
         resolves to: URL that can be used to fetch the resource, of form contenthash:/contenthash/Q123
     */
@@ -102,7 +115,7 @@ httptools.p_GET = async function(httpurl, opts={}) {
         redirect: 'follow',  // Chrome defaults to manual
         keepalive: true    // Keep alive - mostly we'll be going back to same places a lot
     };
-    return await httptools.p_httpfetch(httpurl, init, opts.verbose); // This s a real http url
+    return await httptools.p_httpfetch(httpurl, init, {verbose: opts.verbose, wantstream: opts.wantstream}); // This s a real http url
 }
 httptools.p_POST = async function(httpurl, type, data, verbose) {
     // Locate and return a block, based on its url
@@ -121,7 +134,7 @@ httptools.p_POST = async function(httpurl, type, data, verbose) {
         redirect: 'follow',  // Chrome defaults to manual
         keepalive: true    // Keep alive - mostly we'll be going back to same places a lot
     };
-    return await httptools.p_httpfetch(httpurl, init, verbose);
+    return await httptools.p_httpfetch(httpurl, init, {verbose});
 }
 
 exports = module.exports = httptools;

package.json

@@ -33,7 +33,7 @@
     "chai": "latest",
     "uglifyjs-webpack-plugin": "latest",
     "watchify": "^3.11.0",
-    "webpack-cli": "^3.0.8"
+    "webpack-cli": "^3.1.0"
   },
   "homepage": "https://github.com/internetarchive/dweb-transports#readme",
   "keywords": [],