New TLSF, see #15

README.md

@@ -20,7 +20,7 @@ A few early examples to get an idea:
   A PSON decoder implemented in AssemblyScript.
 
 * **[TLSF memory allocator](./examples/tlsf)**<br />
-  A port of TLSF to AssemblyScript.
+  An implementation of the TLSF memory allocator in AssemblyScript.
 
 * **[μgc garbage collector](./examples/ugc)**<br />
   A port of μgc to AssemblyScript.

bin/asc.js

@@ -1,7 +1,9 @@
-var fs = require("fs");
+#!/usr/bin/env node
-var path = require("path");
+const fs = require("fs");
-var minimist = require("minimist");
+const path = require("path");
-var glob = require("glob");
+const minimist = require("minimist");
+const glob = require("glob");
+const { SourceMapConsumer, SourceMapGenerator } = require("source-map");
 
 var assemblyscript;
 var isDev = true;
@@ -15,8 +17,8 @@ try {
   assemblyscript = require("../src");
 }
 
-var conf = require("./asc.json");
+const conf = require("./asc.json");
-var opts = {};
+const opts = {};
 
 Object.keys(conf).forEach(key => {
   var opt = conf[key];
@@ -30,9 +32,10 @@ Object.keys(conf).forEach(key => {
     (opts.boolean || (opts.boolean = [])).push(key);
 });
 
-var args = minimist(process.argv.slice(2), opts);
+const args = minimist(process.argv.slice(2), opts);
+const indent = 24;
+
 var version = require("../package.json").version;
-var indent = 24;
 if (isDev) version += "-dev";
 
 if (args.version) {
@@ -43,7 +46,7 @@ if (args.version) {
 }
 
 if (args.help || args._.length < 1) {
-  var options = [];
+  const options = [];
   Object.keys(conf).forEach(name => {
     var option = conf[name];
     var text = " ";
@@ -79,6 +82,7 @@ var parser = null;
 var readTime = 0;
 var readCount = 0;
 var writeTime = 0;
+var writeCount = 0;
 var parseTime = 0;
 var compileTime = 0;
 var validateTime = 0;
@@ -122,7 +126,7 @@ libDirs.forEach(libDir => {
       var nextText = fs.readFileSync(path.join(libDir, file), { encoding: "utf8" });
       ++readCount;
       var time = measure(() => {
-        parser = assemblyscript.parseFile(nextText, "std:" + file, parser, false);
+        parser = assemblyscript.parseFile(nextText, ".std/" + file, parser, false);
       });
       parseTime += time;
       notIoTime += time;
@@ -139,12 +143,14 @@ args._.forEach(filename => {
   try {
     readTime += measure(() => {
       entryText = fs.readFileSync(entryPath + ".ts", { encoding: "utf8" });
+      entryPath += ".ts";
     });
     ++readCount;
   } catch (e) {
     try {
       readTime += measure(() => {
         entryText = fs.readFileSync(entryPath + "/index.ts", { encoding: "utf8" });
+        entryPath += "/index.ts";
       });
       ++readCount;
       entryPath = entryPath + "/index";
@@ -154,7 +160,7 @@ args._.forEach(filename => {
     }
   }
 
-  var nextPath;
+  var nextFile;
   var nextText;
 
   // Load entry text
@@ -162,13 +168,14 @@ args._.forEach(filename => {
     parser = assemblyscript.parseFile(entryText, entryPath, parser, true);
   });
 
-  while ((nextPath = parser.nextFile()) != null) {
+  while ((nextFile = parser.nextFile()) != null) {
     var found = false;
-    if (nextPath.startsWith("std:")) {
+    if (nextFile.startsWith(".std/")) {
       for (var i = 0; i < libDirs.length; ++i) {
         readTime += measure(() => {
           try {
-            nextText = fs.readFileSync(libDirs[i] + "/" + nextPath.substring(4) + ".ts", { encoding: "utf8" });
+            nextText = fs.readFileSync(path.join(libDirs[i], nextFile.substring(4) + ".ts"), { encoding: "utf8" });
+            nextFile = nextFile + ".ts";
             found = true;
           } catch (e) {}
         });
@@ -179,7 +186,8 @@ args._.forEach(filename => {
     } else {
       readTime += measure(() => {
         try {
-          nextText = fs.readFileSync(nextPath + "/index.ts", { encoding: "utf8" });
+          nextText = fs.readFileSync(nextFile + ".ts", { encoding: "utf8" });
+          nextFile = nextFile + ".ts";
           found = true;
         } catch (e) {}
       });
@@ -187,7 +195,8 @@ args._.forEach(filename => {
       if (!found) {
         readTime += measure(() => {
           try {
-            nextText = fs.readFileSync(nextPath + ".ts", { encoding: "utf8" });
+            nextText = fs.readFileSync(nextFile + "/index.ts", { encoding: "utf8" });
+            nextFile = nextFile + "/index.ts";
             found = true;
           } catch (e) {}
         });
@@ -195,11 +204,11 @@ args._.forEach(filename => {
       }
     }
     if (!found) {
-      console.error("Imported file '" + nextPath + ".ts' not found.");
+      console.error("Imported file '" + nextFile + ".ts' not found.");
       process.exit(1);
     }
     parseTime += measure(() => {
-      assemblyscript.parseFile(nextText, nextPath, parser);
+      assemblyscript.parseFile(nextText, nextFile, parser);
     });
   }
   checkDiagnostics(parser);
@@ -210,6 +219,7 @@ assemblyscript.setTarget(options, 0);
 assemblyscript.setNoTreeShaking(options, args.noTreeShaking);
 assemblyscript.setNoAssert(options, args.noAssert);
 assemblyscript.setNoMemory(options, args.noMemory);
+assemblyscript.setSourceMap(options, args.sourceMap != null);
 
 var module;
 compileTime += measure(() => {
@@ -221,6 +231,7 @@ if (args.validate)
   validateTime += measure(() => {
     if (!module.validate()) {
       module.dispose();
+      console.error("Validation failed");
       process.exit(1);
     }
   });
@@ -234,7 +245,7 @@ else if (args.trapMode === "js")
     module.runPasses([ "trap-mode-js" ]);
   });
 else if (args.trapMode !== "allow") {
-  console.log("Unsupported trap mode: " + args.trapMode);
+  console.error("Unsupported trap mode: " + args.trapMode);
   process.exit(1);
 }
 
@@ -298,6 +309,40 @@ if (runPasses.length)
     module.runPasses(runPasses.map(pass => pass.trim()));
   });
 
+function processSourceMap(sourceMap, sourceMapURL) {
+  var json = JSON.parse(sourceMap);
+  return SourceMapConsumer.with(sourceMap, sourceMapURL, consumer => {
+    var generator = SourceMapGenerator.fromSourceMap(consumer);
+    json.sources.forEach(name => {
+      var text, found = false;
+      if (name.startsWith(".std/")) {
+        for (var i = 0, k = libDirs.length; i < k; ++i) {
+          readTime += measure(() => {
+            try {
+              text = fs.readFileSync(path.join(libDirs[i], name.substring(4)), { encoding: "utf8" });
+              found = true;
+            } catch (e) {}
+          });
+          ++readCount;
+        }
+      } else {
+        readTime += measure(() => {
+          try {
+            text = fs.readFileSync(name, { encoding: "utf8" });
+            found = true;
+          } catch (e) {}
+        });
+        ++readCount;
+      }
+      if (found)
+        generator.setSourceContent(name, text);
+      else
+        console.error("No source content found for file '" + name + "'.");
+    });
+    return generator.toString();
+  });
+}
+
 if (!args.noEmit) {
   var hasOutput = false;
 
@@ -310,8 +355,25 @@ if (!args.noEmit) {
       args.binaryFile = args.outFile;
   }
   if (args.binaryFile != null && args.binaryFile.length) {
+    var sourceMapURL = args.sourceMap != null
+      ? args.sourceMap.length
+        ? args.sourceMap
+        : path.basename(args.binaryFile) + ".map"
+      : null;
+    var binary;
     writeTime += measure(() => {
-      fs.writeFileSync(args.binaryFile, module.toBinary());
+      binary = module.toBinary(sourceMapURL); // FIXME: 'not a valid URL' in FF
+      fs.writeFileSync(args.binaryFile, binary.output);
+    });
+    ++writeCount;
+    if (binary.sourceMap != null)
+      processSourceMap(binary.sourceMap).then(sourceMap => {
+        writeTime += measure(() => {
+          fs.writeFileSync(path.join(path.dirname(args.binaryFile), path.basename(sourceMapURL)), sourceMap, { encoding: "utf8" });
+        }, err => {
+          throw err;
+        });
+        ++writeCount;
       });
     hasOutput = true;
   }
@@ -319,38 +381,43 @@ if (!args.noEmit) {
     writeTime += measure(() => {
       fs.writeFileSync(args.textFile, module.toText(), { encoding: "utf8" });
     });
+    ++writeCount;
     hasOutput = true;
   }
   if (args.asmjsFile != null && args.asmjsFile.length) {
     writeTime += measure(() => {
       fs.writeFileSync(args.asmjsFile, module.toAsmjs(), { encoding: "utf8" });
     });
+    ++writeCount;
     hasOutput = true;
   }
   if (!hasOutput) {
-    if (args.binaryFile === "")
+    if (args.binaryFile === "") {
       writeTime += measure(() => {
         process.stdout.write(Buffer.from(module.toBinary()));
       });
-    else if (args.asmjsFile === "")
+      ++writeCount;
+    } else if (args.asmjsFile === "") {
       writeTime += measure(() => {
         module.printAsmjs();
       });
-    else
+      ++writeCount;
+    } else {
       writeTime += measure(() => {
         module.print();
       });
+      ++writeCount;
+    }
   }
 }
 
 module.dispose();
 
-if (args.measure)
+if (args.measure) process.on("beforeExit", () => console.error([
-  console.error([
   "I/O Read  : " + (readTime     ? (readTime     / 1e6).toFixed(3) + " ms (" + readCount + " files)" : "N/A"),
-    "I/O Write : " + (writeTime    ? (writeTime    / 1e6).toFixed(3) + " ms" : "N/A"),
+  "I/O Write : " + (writeTime    ? (writeTime    / 1e6).toFixed(3) + " ms (" + writeCount + " files)" : "N/A"),
   "Parse     : " + (parseTime    ? (parseTime    / 1e6).toFixed(3) + " ms" : "N/A"),
   "Compile   : " + (compileTime  ? (compileTime  / 1e6).toFixed(3) + " ms" : "N/A"),
   "Validate  : " + (validateTime ? (validateTime / 1e6).toFixed(3) + " ms" : "N/A"),
   "Optimize  : " + (optimizeTime ? (optimizeTime / 1e6).toFixed(3) + " ms" : "N/A")
-  ].join("\n"));
+].join("\n")));

bin/asc.json

@@ -30,7 +30,7 @@
     "type": "number"
   },
   "shrinkLevel": {
-    "desc": "How much to focus on shrinking code size. [0-2]",
+    "desc": "How much to focus on shrinking code size. [0-2, s=1, z=2]",
     "type": "number"
   },
   "validate": {
@@ -58,6 +58,13 @@
     "type": "string",
     "aliases": [ "a" ]
   },
+  "sourceMap": {
+    "desc": [
+      "Enables source map generation. Optionally takes the URL",
+      "used to reference the source map from the binary file."
+    ],
+    "type": "string"
+  },
   "noTreeShaking": {
     "desc": "Disables compiler-level tree-shaking.",
     "type": "boolean"

examples/tlsf/README.md

@@ -1,7 +1,8 @@
-TLSF memory allocator
+![](https://s.gravatar.com/avatar/f105de3decfafc734b8eabe9a960b25d?size=48) TLSF
-=====================
+=================
 
-A port of [Matt Conte's implementation](https://github.com/mattconte/tlsf) of the [TLSF](http://www.gii.upv.es/tlsf/) memory allocator to AssemblyScript.
+An implementation of the [Two Level Segregate Fit](http://www.gii.upv.es/tlsf/main/docs)
+memory allocator in AssemblyScript.
 
 Instructions
 ------------

examples/tlsf/assembly/LICENSE

@@ -1,36 +0,0 @@
-tlsf.ts is based on https://github.com/mattconte/tlsf
-
-Two Level Segregated Fit memory allocator, version 3.1.
-Written by Matthew Conte
-  http://tlsf.baisoku.org
-
-Based on the original documentation by Miguel Masmano:
-  http://www.gii.upv.es/tlsf/main/docs
-
-This implementation was written to the specification
-of the document, therefore no GPL restrictions apply.
-
-Copyright (c) 2006-2016, Matthew Conte
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-* Redistributions of source code must retain the above copyright
-  notice, this list of conditions and the following disclaimer.
-* Redistributions in binary form must reproduce the above copyright
-  notice, this list of conditions and the following disclaimer in the
-  documentation and/or other materials provided with the distribution.
-* Neither the name of the copyright holder nor the
-  names of its contributors may be used to endorse or promote products
-  derived from this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-DISCLAIMED. IN NO EVENT SHALL MATTHEW CONTE BE LIABLE FOR ANY
-DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

examples/tlsf/package.json

@@ -4,8 +4,8 @@
   "private": true,
   "scripts": {
     "build": "npm run build:untouched && npm run build:optimized",
-    "build:untouched": "asc assembly/tlsf.ts -t tlsf.untouched.wast -b tlsf.untouched.wasm --validate",
+    "build:untouched": "asc assembly/tlsf.ts -t tlsf.untouched.wast -b tlsf.untouched.wasm --validate --sourceMap --measure",
-    "build:optimized": "asc -O3 assembly/tlsf.ts -b tlsf.optimized.wasm -t tlsf.optimized.wast --validate --noDebug --noAssert",
+    "build:optimized": "asc -O3 assembly/tlsf.ts -b tlsf.optimized.wasm -t tlsf.optimized.wast --validate --noDebug --noAssert --sourceMap --measure",
     "test": "node tests"
   }
 }

examples/tlsf/tests/index.js

@@ -1,61 +1,28 @@
 var fs = require("fs");
 
-function test(file) {
+var runner = require("./runner");
-  console.log("Testing '" + file + "' ...");
 
-  var tlsf = new WebAssembly.Instance(WebAssembly.Module(fs.readFileSync(__dirname + "/../" + file)), {
+function test(file) {
+  console.log("Testing '" + file + "' ...\n");
+
+  var exports = new WebAssembly.Instance(WebAssembly.Module(fs.readFileSync(__dirname + "/../" + file)), {
     env: {
-      log_i: function(i) { i == -1 ? console.log("---") : console.log("log_i -> " + i); }
+      abort: function(msg, file, line, column) {
+        throw Error("Assertion failed: " + (msg ? "'" + getString(msg) + "' " : "") + "at " + getString(file) + ":" + line + ":" + column);
+      },
+      log: function(ptr) { console.log(getString(ptr)); },
+      logi: function(i) { console.log(i); }
     }
   }).exports;
 
-  try {
+  function getString(ptr) {
-    var memSize = 0;
+    var len = new Uint32Array(exports.memory.buffer, ptr)[0];
-    var ptr = 0;
+    var str = new Uint16Array(exports.memory.buffer, ptr + 4).subarray(0, len);
-    for (var j = 0; j < 10000; ++j) {
+    return String.fromCharCode.apply(String, str);
-      if (!j || !((j + 1) % 1000))
-        console.log("run #" + (j + 1));
-      ptr;
-      var ptrs = [];
-      // allocate some blocks of unusual sizes
-      for (var i = 0; i < 2048; ++i) {
-        var size = i * 61;
-        ptr = tlsf.allocate_memory(size);
-        if (tlsf.set_memory)
-          tlsf.set_memory(ptr, ptr % 256, size); // slow
-        // immediately free every 4th
-        if (!(i % 4)) {
-          tlsf.free_memory(ptr);
-        } else {
-          ptrs.push(ptr);
-          // randomly free random blocks (if not the first run that determines max memory)
-          if (j && Math.random() < 0.25) {
-            ptr = ptrs.splice((Math.random() * ptrs.length)|0, 1)[0];
-            tlsf.free_memory(ptr);
-          }
-        }
-      }
-      if (tlsf.check)
-        tlsf.check();
-      if (tlsf.check_pool)
-        tlsf.check_pool(0);
-      // clean up by randomly freeing all blocks
-      while (ptrs.length) {
-        ptr = ptrs.splice((Math.random() * ptrs.length)|0, 1)[0];
-        tlsf.free_memory(ptr);
-      }
-      if (memSize && memSize != tlsf.memory.buffer.byteLength)
-        throw new Error("memory should not grow multiple times: " + memSize + " != " + tlsf.memory.buffer.byteLength);
-      memSize = tlsf.memory.buffer.byteLength;
-      if (tlsf.check)
-        tlsf.check();
-      if (tlsf.check_pool)
-        tlsf.check_pool(0);
-    }
-  } finally {
-    // mem(tlsf.memory, 0, 4096);
-    console.log("memSize=" + memSize);
   }
+
+  runner(exports, 10, 20000); // picked so I/O isn't the bottleneck
+  console.log("mem final: " + exports.memory.buffer.byteLength);
   console.log();
 }
 

examples/tlsf/tests/runner.js

@@ -0,0 +1,91 @@
+function runner(tlsf, runs, allocs) {
+  var ptrs = [];
+
+  function randomAlloc(maxSize) {
+    if (!maxSize) maxSize = 8192;
+    var size = ((Math.random() * maxSize) >>> 0) + 1;
+    size = (size + 3) & ~3;
+    var ptr = tlsf.allocate_memory(size);
+    if (!ptr) throw Error();
+    if (ptrs.indexOf(ptr) >= 0) throw Error();
+    if (tlsf.set_memory)
+      tlsf.set_memory(ptr, 0xdc, size);
+    ptrs.push(ptr);
+    return ptr;
+  }
+
+  function preciseFree(ptr) {
+    var idx = ptrs.indexOf(ptr);
+    if (idx < 0) throw Error();
+    var ptr = ptrs[idx];
+    ptrs.splice(idx, 1);
+    if (typeof ptr !== "number") throw Error();
+    tlsf.free_memory(ptr);
+  }
+
+  function randomFree() {
+    var idx = (Math.random() * ptrs.length) >>> 0;
+    var ptr = ptrs[idx];
+    if (typeof ptr !== "number") throw Error();
+    ptrs.splice(idx, 1);
+    tlsf.free_memory(ptr);
+  }
+
+  // remember the smallest possible memory address
+  var base = tlsf.allocate_memory(64);
+  console.log("base: " + base);
+  tlsf.free_memory(base);
+  console.log("mem initial: " + tlsf.memory.buffer.byteLength);
+
+  for (var j = 0; j < runs; ++j) {
+    console.log("run " + (j + 1) + " (" + allocs + " allocations) ...");
+    for (var i = 0; i < allocs; ++i) {
+      var ptr = randomAlloc();
+
+      // immediately free every 4th
+      if (!(i % 4)) preciseFree(ptr);
+
+      // occasionally free random blocks
+      else if (ptrs.length && Math.random() < 0.33) randomFree();
+
+      // ^ sums up to clearing about half the blocks half-way
+    }
+    // free the rest, randomly
+    while (ptrs.length) randomFree();
+
+    // should now be possible to reuse the entire first page (remember: sl+1)
+    // e.g. with base 3088 (3048 optimized due to static memory):
+    var size = 0x10000 - base - 4 - 1008;
+    // 61436 (1110111111111100b) -> fl = 15, sl = 27
+    // 61437 (61440 aligned, 1111000000000000b) -> fl = 15, sl = 28
+    // NOTE that this calculation will be different if static memory changes
+    var ptr = tlsf.allocate_memory(size);
+    tlsf.set_memory(ptr, 0xac, size);
+    if (ptr !== base) throw Error("expected " + base + " but got " + ptr);
+    tlsf.free_memory(ptr);
+  }
+
+  mem(tlsf.memory, 0, 0x10000); // should end in 02 00 00 00 (tail LEFT_FREE)
+}
+
+function mem(memory, offset, count) {
+  if (!offset) offset = 0;
+  if (!count) count = 1024;
+  var mem = new Uint8Array(memory.buffer, offset);
+  var stackTop = new Uint32Array(memory.buffer, 4, 1)[0];
+  var hex = [];
+  for (var i = 0; i < count; ++i) {
+    var o = (offset + i).toString(16);
+    while (o.length < 4) o = "0" + o;
+    if ((i & 15) === 0) {
+      hex.push("\n" + o + ":");
+    }
+    var h = mem[i].toString(16);
+    if (h.length < 2) h = "0" + h;
+    hex.push(h);
+  }
+  console.log(hex.join(" ") + " ...");
+}
+
+if (typeof module === "object" && typeof exports === "object")
+  module.exports = runner;