Test formatting; Wire webpack loader to asc

This commit is contained in:
dcodeIO 2018-02-05 17:10:14 +01:00
parent a0b39da7cf
commit 41c0f2c6c3
5 changed files with 244 additions and 201 deletions

View File

@ -1,10 +1,10 @@
const fs = require("fs");
const path = require("path");
const fs = require("fs");
const os = require("os");
// Use distribution files if present, otherwise run the sources directly
var assemblyscript;
var isDev = true;
const { assemblyscript, isDev } = (function bootstrap() {
var assemblyscript, isDev;
try {
assemblyscript = require("../dist/assemblyscript.js");
isDev = false;
@ -13,7 +13,10 @@ try {
require("ts-node").register({ project: require("path").join(__dirname, "..", "src") });
require("../src/glue/js");
assemblyscript = require("../src");
isDev = true;
}
return { assemblyscript, isDev };
})();
// Common constants
const VERSION = require("../package.json").version + (isDev ? "-dev" : "");
@ -36,19 +39,18 @@ function main(argv, options, callback) {
const stderr = options.stderr || process.stderr;
// Record compilation times
var stats = createStats();
const stats = createStats();
const args = parseArguments(argv);
const indent = 24;
// Use default callback is none is provided
// Use default callback if none is provided
if (!callback) callback = function defaultCallback(err) {
var code = 0;
if (err) {
stderr.write(err + os.EOL);
code = 1;
} else if (args.measure)
printStats(stats, stderr);
}
return code;
};
@ -71,7 +73,7 @@ function main(argv, options, callback) {
text += " ";
if (Array.isArray(option.desc)) {
opts.push(text + option.desc[0] + option.desc.slice(1).map(line => {
for (var i = 0; i < indent; ++i)
for (let i = 0; i < indent; ++i)
line = " " + line;
return os.EOL + line;
}).join(""));
@ -100,10 +102,9 @@ function main(argv, options, callback) {
// Include custom library components (with or without stdlib)
if (args.lib) {
if (Array.isArray(args.lib))
Array.prototype.push.apply(libDirs, args.lib.map(dir));
else
libDirs.push(args.lib);
if (typeof args.lib === "string")
args.lib = args.lib.split(",");
Array.prototype.push.apply(libDirs, args.lib.map(trim));
}
// Begin parsing
@ -113,114 +114,75 @@ function main(argv, options, callback) {
for (let i = 0, k = args._.length; i < k; ++i) {
const filename = args._[i];
let entryPath = filename.replace(/\\/g, "/").replace(/(\.ts|\/)$/, "");
let entryText;
let sourcePath = filename.replace(/\\/g, "/").replace(/(\.ts|\/)$/, "");
// Try entryPath.ts, then entryPath/index.ts
try {
stats.readTime += measure(() => {
entryText = fs.readFileSync(path.join(baseDir, entryPath) + ".ts", { encoding: "utf8" });
entryPath += ".ts";
});
++stats.readCount;
} catch (e) {
try {
stats.readTime += measure(() => {
entryText = fs.readFileSync(path.join(baseDir, entryPath, "index.ts"), { encoding: "utf8" });
entryPath += "/index.ts";
});
++stats.readCount;
entryPath = entryPath + "/index";
} catch (e) {
return callback(Error("Entry file '" + entryPath + ".ts' not found."));
}
}
let sourceText = readFile(path.join(baseDir, sourcePath) + ".ts");
if (sourceText === null) {
sourceText = readFile(path.join(baseDir, sourcePath, "index.ts"));
if (sourceText === null)
return callback(Error("Entry file '" + sourcePath + ".ts' not found."));
else
sourcePath += "/index.ts";
} else
sourcePath += ".ts";
stats.parseTime += measure(() => {
parser = assemblyscript.parseFile(entryText, entryPath, parser, true);
});
stats.parseCount++;
stats.parseTime += measure(() => parser = assemblyscript.parseFile(sourceText, sourcePath, parser, true));
let nextFile;
let nextText;
while ((nextFile = parser.nextFile()) != null) {
while ((sourcePath = parser.nextFile()) != null) {
let found = false;
// Load library file if explicitly requested
if (nextFile.startsWith(LIBRARY_PREFIX)) {
for (let i = 0; i < libDirs.length; ++i) {
stats.readTime += measure(() => {
try {
nextText = fs.readFileSync(path.join(libDirs[i], nextFile.substring(4) + ".ts"), { encoding: "utf8" });
nextFile = nextFile + ".ts";
found = true;
} catch (e) {}
});
++stats.readCount;
if (found)
if (sourcePath.startsWith(LIBRARY_PREFIX)) {
for (let i = 0, k = libDirs.length; i < k; ++i) {
sourceText = readFile(path.join(libDirs[i], sourcePath.substring(LIBRARY_PREFIX.length) + ".ts"));
if (sourceText !== null) {
sourcePath += ".ts";
break;
}
}
// Otherwise try nextFile.ts, nextFile/index.ts, (lib)/nextFile.ts
} else {
stats.readTime += measure(() => {
try {
nextText = fs.readFileSync(path.join(baseDir, nextFile + ".ts"), { encoding: "utf8" });
nextFile = nextFile + ".ts";
found = true;
} catch (e) {}
});
++stats.readCount;
if (!found) {
stats.readTime += measure(() => {
try {
nextText = fs.readFileSync(path.join(baseDir, nextFile, "index.ts"), { encoding: "utf8" });
nextFile = nextFile + "/index.ts";
found = true;
} catch (e) {}
});
++stats.readCount;
}
if (!found) {
for (let i = 0; i < libDirs.length; ++i) {
stats.readTime += measure(() => {
try {
nextText = fs.readFileSync(path.join(libDirs[i], nextFile + ".ts"), { encoding: "utf8" });
nextFile = LIBRARY_PREFIX + nextFile + ".ts";
found = true;
} catch (e) {}
});
++stats.readCount;
if (found)
sourceText = readFile(path.join(baseDir, sourcePath + ".ts"));
if (sourceText === null) {
sourceText = readFile(path.join(baseDir, sourcePath, "index.ts"));
if (sourceText === null) {
for (let i = 0, k =libDirs.length; i < k; ++i) {
sourceText = readFile(path.join(libDirs[i], sourcePath + ".ts"));
if (sourceText !== null) {
sourcePath = LIBRARY_PREFIX + sourcePath + ".ts";
break;
}
}
if (sourceText === null)
return callback(Error("Import file '" + sourcePath + ".ts' not found."));
} else
sourcePath += "/index.ts";
} else
sourcePath += ".ts";
}
if (!found)
return callback(Error("Import file '" + nextFile + ".ts' not found."));
stats.parseTime += measure(() => {
assemblyscript.parseFile(nextText, nextFile, parser);
});
stats.parseCount++;
stats.parseTime += measure(() => assemblyscript.parseFile(sourceText, sourcePath, parser));
}
if (checkDiagnostics(parser, stderr))
return callback(Error("Parse error"));
}
// Include library components
// Include (other) library components
for (let i = 0, k = libDirs.length; i < k; ++i) {
let libDir = libDirs[i];
let notReadTime = 0;
stats.readTime += measure(() => {
require("glob").sync("*.ts", { cwd: libDir }).forEach(file => {
var nextText = fs.readFileSync(path.join(libDir, file), { encoding: "utf8" });
++stats.readCount;
var time = measure(() => {
parser = assemblyscript.parseFile(nextText, LIBRARY_PREFIX + file, parser, false);
});
stats.parseTime += time;
notReadTime += time;
});
}) - notReadTime;
let libFiles;
stats.readTime += measure(() => { libFiles = require("glob").sync("*.ts", { cwd: libDir }) });
for (let j = 0, l = libFiles.length; j < l; ++j) {
let libPath = libFiles[j];
let libText = readFile(path.join(libDir, libPath));
if (libText === null)
return callback(Error("Library file '" + libPath + "' could not be read."));
stats.parseCount++;
stats.parseTime += measure(() => { parser = assemblyscript.parseFile(libText, LIBRARY_PREFIX + libPath, parser, false); });
}
}
// Begin compilation
@ -232,9 +194,8 @@ function main(argv, options, callback) {
assemblyscript.setSourceMap(compilerOptions, args.sourceMap != null);
var module;
stats.compileTime += measure(() => {
module = assemblyscript.compile(parser, compilerOptions);
});
stats.compileCount++;
stats.compileTime += measure(() => module = assemblyscript.compile(parser, compilerOptions));
if (checkDiagnostics(parser, stderr)) {
if (module) module.dispose();
return callback(Error("Compile error"));
@ -242,6 +203,7 @@ function main(argv, options, callback) {
// Validate the module if requested
if (args.validate) {
stats.validateCount++;
stats.validateTime += measure(() => {
if (!module.validate()) {
module.dispose();
@ -252,8 +214,10 @@ function main(argv, options, callback) {
// Set Binaryen-specific options
if (args.trapMode === "clamp") {
stats.optimizeCount++;
stats.optimizeTime += measure(() => module.runPasses([ "trap-mode-clamp" ]));
} else if (args.trapMode === "js") {
stats.optimizeCount++;
stats.optimizeTime += measure(() => module.runPasses([ "trap-mode-js" ]));
} else if (args.trapMode !== "allow") {
module.dispose();
@ -311,12 +275,16 @@ function main(argv, options, callback) {
}
// Optimize the module if requested
if (optimizeLevel >= 0)
if (optimizeLevel >= 0) {
stats.optimizeCount++;
stats.optimizeTime += measure(() => module.optimize());
}
// Run additional passes if requested
if (runPasses.length)
if (runPasses.length) {
stats.optimizeCount++;
stats.optimizeTime += measure(() => module.runPasses(runPasses.map(pass => pass.trim())));
}
// Prepare output
if (!args.noEmit) {
@ -340,14 +308,13 @@ function main(argv, options, callback) {
: null;
let binary;
stats.writeTime += measure(() => binary = module.toBinary(sourceMapURL));
stats.emitCount++;
stats.emitTime += measure(() => binary = module.toBinary(sourceMapURL));
if (args.binaryFile.length) {
stats.writeTime += measure(() => fs.writeFileSync(path.join(baseDir, args.binaryFile), binary.output));
++stats.writeCount;
writeFile(path.join(baseDir, args.binaryFile), binary.output);
} else {
stats.writeTime += measure(() => stdout.write(Buffer.from(binary.output)));
++stats.writeCount;
writeStdout(binary.output);
hasStdout = true;
}
@ -357,65 +324,89 @@ function main(argv, options, callback) {
let sourceMap = JSON.parse(binary.sourceMap);
sourceMap.sourceRoot = SOURCEMAP_ROOT;
sourceMap.sources.forEach((name, index) => {
var text, found = false;
let text = null;
if (name.startsWith(LIBRARY_PREFIX)) {
for (var i = 0, k = libDirs.length; i < k; ++i) {
stats.readTime += measure(() => {
try {
text = fs.readFileSync(path.join(libDirs[i], name.substring(LIBRARY_PREFIX.length)), { encoding: "utf8" });
found = true;
} catch (e) {}
});
++stats.readCount;
for (let i = 0, k = libDirs.length; i < k; ++i) {
text = readFile(path.join(libDirs[i], name.substring(LIBRARY_PREFIX.length)));
if (text !== null) break;
}
} else {
stats.readTime += measure(() => {
try {
text = fs.readFileSync(path.join(baseDir, name), { encoding: "utf8" });
found = true;
} catch (e) {}
});
++stats.readCount;
}
if (!found)
} else
text = readFile(path.join(baseDir, name));
if (text === null)
return callback(Error("Source file '" + name + "' not found."));
(sourceMap.sourceContents || (sourceMap.sourceContents = []))[index] = text;
});
stats.writeTime += measure(() => fs.writeFileSync(path.join(baseDir, path.dirname(args.binaryFile), path.basename(sourceMapURL)), JSON.stringify(sourceMap), { encoding: "utf8" }));
++stats.writeCount;
writeFile(path.join(baseDir, path.dirname(args.binaryFile), path.basename(sourceMapURL)), JSON.stringify(sourceMap));
} else {
stderr.write("Cannot write source map because binary already uses stdout." + os.EOL);
stderr.write("Cannot write source map because binary already occupies stdout." + os.EOL);
}
}
}
// Write text
if (args.textFile != null) {
if (args.textFile.length) {
stats.writeTime += measure(() => fs.writeFileSync(path.join(baseDir, args.textFile), module.toText(), { encoding: "utf8" }));
++stats.writeCount;
if (args.textFile != null || (args.binaryFile == null && args.asmjsFile == null)) {
let text;
if (args.textFile && args.textFile.length) {
stats.emitCount++;
stats.emitTime += measure(() => text = module.toText());
writeFile(path.join(baseDir, args.textFile), text);
} else if (!hasStdout) {
stats.writeTime += measure(() => stdout.write(module.toText()));
++stats.writeCount;
stats.emitCount++;
stats.emitTime += measure(() => text = module.toText());
writeStdout(text);
hasStdout = true;
}
}
// Write asm.js
if (args.asmjsFile != null && args.asmjsFile.length) {
let asm;
if (args.asmjsFile.length) {
stats.writeTime += measure(() => fs.writeFileSync(path.join(baseDir, args.asmjsFile), module.toAsmjs(), { encoding: "utf8" }));
++stats.writeCount;
stats.emitCount++;
stats.emitTime += measure(() => asm = module.toAsmjs());
writeFile(path.join(baseDir, args.asmjsFile), asm);
} else if (!hasStdout) {
stats.writeTime += measure(() => stdout.write(Buffer.from(module.toBinary().output)));
++stats.writeCount;
stats.emitCount++;
stats.emitTime += measure(() => asm = module.toAsmjs());
writeStdout(asm);
hasStdout = true;
}
}
}
module.dispose();
if (args.measure)
printStats(stats, stderr);
return callback(null);
function readFile(filename) {
try {
var text;
stats.readCount++;
stats.readTime += measure(() => text = fs.readFileSync(filename, { encoding: "utf8" }));
return text;
} catch (e) {
return null;
}
}
function writeFile(filename, contents) {
try {
stats.writeCount++;
stats.writeTime += measure(() => fs.writeFileSync(filename, contents, typeof contents === "string" ? { encoding: "utf8" } : undefined));
return true;
} catch (e) {
return false;
}
}
function writeStdout(contents) {
if (!writeStdout.used) {
stats.writeCount++;
writeStdout.used = true;
}
stats.writeTime += measure(() => stdout.write(contents, typeof contents === "string" ? { encoding: "utf8" } : undefined));
}
}
exports.main = main;
@ -457,9 +448,15 @@ function createStats() {
writeTime: 0,
writeCount: 0,
parseTime: 0,
parseCount: 0,
compileTime: 0,
compileCount: 0,
emitTime: 0,
emitCount: 0,
validateTime: 0,
optimizeTime: 0
validateCount: 0,
optimizeTime: 0,
optimizeCount: 0
};
}
@ -478,10 +475,11 @@ function printStats(stats, output) {
(output || process.stdout).write([
"I/O Read : " + (stats.readTime ? (stats.readTime / 1e6).toFixed(3) + " ms (" + stats.readCount + " files)" : "N/A"),
"I/O Write : " + (stats.writeTime ? (stats.writeTime / 1e6).toFixed(3) + " ms (" + stats.writeCount + " files)" : "N/A"),
"Parse : " + (stats.parseTime ? (stats.parseTime / 1e6).toFixed(3) + " ms" : "N/A"),
"Compile : " + (stats.compileTime ? (stats.compileTime / 1e6).toFixed(3) + " ms" : "N/A"),
"Validate : " + (stats.validateTime ? (stats.validateTime / 1e6).toFixed(3) + " ms" : "N/A"),
"Optimize : " + (stats.optimizeTime ? (stats.optimizeTime / 1e6).toFixed(3) + " ms" : "N/A")
"Parse : " + (stats.parseTime ? (stats.parseTime / 1e6).toFixed(3) + " ms (" + stats.parseCount + " times)" : "N/A"),
"Compile : " + (stats.compileTime ? (stats.compileTime / 1e6).toFixed(3) + " ms (" + stats.compileCount + " times)" : "N/A"),
"Emit : " + (stats.emitTime ? (stats.emitTime / 1e6).toFixed(3) + " ms (" + stats.emitCount + " times)" : "N/A"),
"Validate : " + (stats.validateTime ? (stats.validateTime / 1e6).toFixed(3) + " ms (" + stats.validateCount + " times)" : "N/A"),
"Optimize : " + (stats.optimizeTime ? (stats.optimizeTime / 1e6).toFixed(3) + " ms (" + stats.optimizeCount + " times)" : "N/A")
].join(os.EOL) + os.EOL);
}

View File

@ -95,13 +95,13 @@ class Block {
// ├─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┴─┤ ┐
// │ 0 | flMap S│ ◄────┐
// ╞═══════════════════════════════════════════════════════════════╡ │
// │ slMap[0] (small blocks) │ ◄─┐ │
// │ slMap[0] S │ ◄─┐ │
// ├───────────────────────────────────────────────────────────────┤ │ │
// │ slMap[1] │ ◄─┤ │
// ├───────────────────────────────────────────────────────────────┤ u32 │
// │ ... │ ◄─┤ │
// ├───────────────────────────────────────────────────────────────┤ │ │
// │ slMap[22] * │ ◄─┘ │
// │ slMap[22] P │ ◄─┘ │
// ╞═══════════════════════════════════════════════════════════════╡ usize
// │ head[0] │ ◄────┤
// ├───────────────────────────────────────────────────────────────┤ │
@ -109,7 +109,7 @@ class Block {
// ├───────────────────────────────────────────────────────────────┤ │
// │ head[736] │ ◄────┘
// └───────────────────────────────────────────────────────────────┘ SIZE ┘
// *: Possibly followed by padding if 64-bit
// S: Small blocks map, P: Possibly padded if 64-bit
assert((1 << SL_BITS) <= 32); // second level must fit into 32 bits
@ -398,7 +398,7 @@ function ffs<T>(word: T): T {
return ctz<T>(word); // differs from ffs only for 0
}
/** Determins the last (LSB to MSB) set bit's index of a word. */
/** Determines the last (LSB to MSB) set bit's index of a word. */
function fls<T>(word: T): T {
assert(word != 0); // word cannot be 0
const inv: T = (sizeof<T>() << 3) - 1;

View File

@ -1,18 +1,13 @@
// set up decoding table
var s64 = new Array(123);
for (var i = 0; i < 64;) s64[i < 26 ? i + 65 : i < 52 ? i + 71 : i < 62 ? i - 4 : i - 59 | 43] = i++;
module.exports = function decode(string) {
// determine buffer length
var length = string.length;
if (length) {
var n = 0, p = length;
while (--p % 4 > 1 && string.charCodeAt(p) === 61) ++n;
length = Math.ceil(length * 3) / 4 - n;
}
// decode to buffer
var buffer = new Uint8Array(length);
var j = 0, o = 0, t;
for (var i = 0, k = string.length; i < k;) {
@ -27,6 +22,5 @@ module.exports = function decode(string) {
}
}
if (j === 1) throw Error();
return buffer;
};

View File

@ -1,21 +1,58 @@
var base64 = require("@protobufjs/base64");
const fs = require("fs");
const path = require("path");
const asc = require("assemblyscript/bin/asc.js");
const base64 = require("@protobufjs/base64");
const MAGIC = Buffer.from([ 0x00, 0x61, 0x73, 0x6D ]);
module.exports = loader;
function loader(buffer) {
var data = base64.encode(buffer, 0, buffer.length);
var code = [
if (MAGIC.compare(target, 0, 4) !== 0)
return compile.call(this);
else
return bundle.call(this, buffer);
}
loader.raw = true;
function compile() {
const basePath = this.resourcePath.replace(/\.\w+$/, "");
const args = [
path.basename(this.resourcePath),
"--baseDir", path.dirname(this.resourcePath),
"--binaryFile", basePath + ".wasm",
"--textFile", basePath + ".wast",
"--validate",
"--optimize"
];
if (this.sourceMap)
args.push("--sourceMap");
asc.main(args, err => {
if (err)
return this.callback(err);
fs.readFile(basePath + ".wasm", (err, binary) => {
if (err)
return this.callback(err);
if (!this.sourceMap)
return this.callback(null, bundle(binary));
fs.readFile(basePath + ".wasm.map", (err, sourceMap) => {
if (err)
return this.callback(err);
return this.callback(null, bundle(binary), sourceMap.toString("utf8"));
});
});
});
}
function bundle(binary) {
const data = base64.encode(binary, 0, binary.wasm);
return [
'var data = "' + data + '", wasm;',
'module.exports = function AssemblyScriptModule(options) {',
' if (!wasm)',
' wasm = new WebAssembly.Module(require("@assemblyscript/webpack/decode")(data));',
' return new WebAssembly.Instance(wasm, options && options.imports || {}).exports;',
'};'
];
return code.join("\n") + "\n";
].join("\n") + "\n";
}
loader.raw = true;
Object.defineProperties(module.exports = loader, {
__esModule: { value: true },
default: { value: loader }
});

View File

@ -4,6 +4,7 @@ const os = require("os");
const chalk = require("chalk");
const glob = require("glob");
const minimist = require("minimist");
const diff = require("./util/diff");
const asc = require("../bin/asc.js");
@ -13,9 +14,12 @@ const args = minimist(process.argv.slice(2), {
});
if (args.help) {
console.log("Usage: npm run test:compiler -- [test1, test2 ...] [--create]\n");
console.log("Runs all tests if no tests have been specified.");
console.log("Recreates affected fixtures if --create is specified.");
console.log([
"Usage: npm run test:compiler -- [test1, test2 ...] [--create]",
"",
"Runs all tests if no tests have been specified.",
"Recreates affected fixtures if --create is specified."
].join(os.EOL) + os.EOL);
process.exit(0);
}
@ -38,7 +42,7 @@ if (args._.length) {
// TODO: asc's callback is synchronous here. This might change.
tests.forEach(filename => {
console.log(chalk.whiteBright("Testing compiler/" + filename));
console.log(chalk.whiteBright("Testing compiler/" + filename) + "\n");
const basename = filename.replace(/\.ts$/, "");
@ -47,50 +51,59 @@ tests.forEach(filename => {
var failed = false;
// TODO: also stdout/stderr and diff it (-> expected failures)
// TODO: also save stdout/stderr and diff it (-> expected failures)
// Build unoptimized
asc.main( [
filename,
"--baseDir", basedir,
"-t", // -> stdout
"--sourceMap"
"--validate",
"--sourceMap",
"--measure",
"--textFile" // -> stdout
], {
stdout: stdout,
stderr: stderr
}, err => {
console.log();
if (err)
stderr.write(err + os.EOL);
if (args.create) {
fs.writeFileSync(path.join(basedir, basename + ".wast"), stdout.toString(), { encoding: "utf8" });
console.log("Recreated fixture.");
console.log("- " + chalk.yellow("Created fixture"));
} else {
let actual = stdout.toString();
let expected = fs.readFileSync(path.join(basedir, basename + ".wast"), { encoding: "utf8" });
let diffs = diff(basename + ".wast", expected, actual);
if (diffs !== null) {
console.log(diffs);
console.log(chalk.red("diff ERROR"));
console.log("- " + chalk.red("diff ERROR"));
failed = true;
} else
console.log(chalk.green("diff OK"));
console.log("- " + chalk.green("diff OK"));
}
console.log();
stdout.length = 0;
stderr.length = 0;
stderr.print = false;
// Build optimized
asc.main([
var cmd = [
filename,
"--baseDir", basedir,
"-t", basename + ".optimized.wast",
"-b", // -> stdout
"-O"
], {
"--validate",
"--optimize",
"--measure",
"--binaryFile" // -> stdout
];
if (args.create) cmd.push(
"--textFile", basename + ".optimized.wast"
);
asc.main(cmd, {
stdout: stdout,
stderr: stderr
}, err => {
console.log();
if (err)
stderr.write(err + os.EOL);
@ -109,9 +122,9 @@ tests.forEach(filename => {
externalConstant: 2
}
});
console.log(chalk.green("instantiate OK"));
console.log("- " + chalk.green("instantiate OK"));
} catch (e) {
console.log(chalk.red("instantiate ERROR: ") + e);
console.log("- " + chalk.red("instantiate ERROR: ") + e);
failed = true;
}
@ -129,10 +142,11 @@ function createMemoryStream(print) {
stream.write = function(chunk) {
if (typeof chunk === "string") {
this.push(Buffer.from(chunk, "utf8"));
if (stream.print)
process.stderr.write(chunk);
} else
} else {
this.push(chunk);
}
if (stream.print)
process.stderr.write(chunk.toString().replace(/^(?!$)/mg, " "));
};
stream.toBuffer = function() {
return Buffer.concat(this);