#! /usr/bin/env python # Copyright 2016 WebAssembly Community Group participants # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import argparse import glob import itertools import multiprocessing import os import shutil import subprocess import sys import tempfile verbose = False DIR_BLACKLIST = ['misc', 'ldso'] BLACKLIST = [ 'puts.c', # Prefer the JS version for now 'abort.c', # Perfer the JS version for now '_Exit.c', # Perfer the JS version for now '__init_tls.c', ] # Files that contain weak reference which are not suppoered by s2wasm WEAK_BLACKLIST = [ 'exit.c', '__libc_start_main.c', ] CFLAGS = ['-std=c99', '-D_XOPEN_SOURCE=700', '-Werror', '-Wno-empty-body', '-Wno-incompatible-library-redeclaration', '-Wno-shift-op-parentheses', '-Wno-tautological-unsigned-zero-compare', '-Wno-tautological-constant-out-of-range-compare', '-Wno-tautological-unsigned-enum-zero-compare', '-Wno-ignored-attributes', '-Wno-format', '-Wno-bitwise-op-parentheses', '-Wno-logical-op-parentheses', '-Wno-string-plus-int', '-Wno-pointer-sign', '-Wno-dangling-else', '-Wno-absolute-value', '-Wno-parentheses', '-Wno-unknown-pragmas'] def check_output(cmd, **kwargs): cwd = kwargs.get('cwd', os.getcwd()) if verbose: c = ' '.join('"' + c + '"' if ' ' in c else c for c in cmd) print ' `%s`, cwd=`%s`' % (c, cwd) return subprocess.check_output(cmd, cwd=cwd) def change_extension(path, new_extension): return path[:path.rfind('.')] + new_extension def create_version(musl, outdir): """musl's Makefile creates version.h""" script = os.path.join(musl, 'tools', 'version.sh') version = check_output(['sh', script], cwd=musl).strip() outroot = os.path.join(outdir, 'src', 'internal') if not os.path.exists(outroot): os.makedirs(outroot) with open(os.path.join(outroot, 'version.h'), 'w') as v: v.write('#define VERSION "%s"\n' % version) def build_headers(musl, arch, outdir): """Emulate musl's Makefile build of alltypes.h and syscall.h""" outroot = os.path.join(outdir, 'include', 'bits') if not os.path.exists(outroot): os.makedirs(outroot) mkalltypes = os.path.join(musl, 'tools', 'mkalltypes.sed') inbits = os.path.join(musl, 'arch', arch, 'bits', 'alltypes.h.in') intypes = os.path.join(musl, 'include', 'alltypes.h.in') out = check_output(['sed', '-f', mkalltypes, inbits, intypes]) with open(os.path.join(outroot, 'alltypes.h'), 'w') as o: o.write(out) insyscall = os.path.join(musl, 'arch', arch, 'bits', 'syscall.h.in') out = check_output(['sed', '-n', '-e', 's/__NR_/SYS_/p', insyscall]) with open(os.path.join(outroot, 'syscall.h'), 'w') as o: o.write(open(insyscall).read()) o.write(out) def musl_sources(musl_root, include_weak): """musl sources to be built.""" sources = [] for d in os.listdir(os.path.join(musl_root, 'src')): if d in DIR_BLACKLIST: continue base = os.path.join(musl_root, 'src', d) pattern = os.path.join(base, '*.c') for f in glob.glob(pattern): if os.path.basename(f) in BLACKLIST: continue if not include_weak and os.path.basename(f) in WEAK_BLACKLIST: continue sources.append(f) return sorted(sources) def includes(musl, arch, outdir): """Include path.""" includes = [ os.path.join(musl, 'arch', arch), os.path.join(musl, 'arch', 'generic'), os.path.join(outdir, 'src', 'internal'), os.path.join(musl, 'src', 'internal'), os.path.join(outdir, 'include'), os.path.join(musl, 'include')] return list(itertools.chain(*zip(['-I'] * len(includes), includes))) class Compiler(object): """Compile source files.""" def __init__(self, out, clang_dir, musl, arch, tmpdir): self.out = out self.outbase = os.path.basename(self.out) self.clang_dir = clang_dir self.musl = musl self.arch = arch self.tmpdir = tmpdir self.compiled = [] def compile(self, sources): if verbose: self.compiled = sorted([self(source) for source in sources]) else: pool = multiprocessing.Pool() self.compiled = sorted(pool.map(self, sources)) pool.close() pool.join() class ObjCompiler(Compiler): def __init__(self, out, clang_dir, musl, arch, tmpdir): super(ObjCompiler, self).__init__(out, clang_dir, musl, arch, tmpdir) def __call__(self, src): target = 'wasm32-unknown-unknown-wasm' compile_cmd = [os.path.join(self.clang_dir, 'clang'), '-target', target, '-Os', '-c', '-nostdinc'] compile_cmd += includes(self.musl, self.arch, self.tmpdir) compile_cmd += CFLAGS check_output(compile_cmd + [src], cwd=self.tmpdir) return os.path.basename(src)[:-1] + 'o' # .c -> .o def binary(self): if os.path.exists(self.out): os.remove(self.out) check_output([os.path.join(self.clang_dir, 'llvm-ar'), 'rcs', self.out] + self.compiled, cwd=self.tmpdir) class AsmCompiler(Compiler): def __init__(self, out, clang_dir, musl, arch, tmpdir, binaryen_dir, sexpr_wasm): super(AsmCompiler, self).__init__(out, clang_dir, musl, arch, tmpdir) self.binaryen_dir = binaryen_dir self.sexpr_wasm = sexpr_wasm def __call__(self, src): target = 'wasm32-unknown-unknown' compile_cmd = [os.path.join(self.clang_dir, 'clang'), '-target', target, '-Os', '-emit-llvm', '-S', '-nostdinc'] compile_cmd += includes(self.musl, self.arch, self.tmpdir) compile_cmd += CFLAGS check_output(compile_cmd + [src], cwd=self.tmpdir) return os.path.basename(src)[:-1] + 'll' # .c -> .ll def binary(self): max_memory = str(16 * 1024 * 1024) bytecode = change_extension(self.out, '.bc') assembly = os.path.join(self.tmpdir, self.outbase + '.s') check_output([os.path.join(self.clang_dir, 'llvm-link'), '-o', bytecode] + self.compiled, cwd=self.tmpdir) check_output([os.path.join(self.clang_dir, 'llc'), bytecode, '-o', assembly], cwd=self.tmpdir) check_output([os.path.join(self.binaryen_dir, 's2wasm'), assembly, '--ignore-unknown', '-o', self.out, '--import-memory', '-m', max_memory], cwd=self.tmpdir) if self.sexpr_wasm: check_output([self.sexpr_wasm, self.out, '-o', change_extension(self.out, '.wasm')], cwd=self.tmpdir) def run(clang_dir, binaryen_dir, sexpr_wasm, musl, arch, out, compile_to_wasm): objdir = os.path.join(os.path.dirname(out), 'obj') if os.path.isdir(objdir): shutil.rmtree(objdir) os.mkdir(objdir) create_version(musl, objdir) build_headers(musl, arch, objdir) sources = musl_sources(musl, include_weak=compile_to_wasm) if compile_to_wasm: compiler = ObjCompiler(out, clang_dir, musl, arch, objdir) else: compiler = AsmCompiler(out, clang_dir, musl, arch, objdir, binaryen_dir, sexpr_wasm) compiler.compile(sources) compiler.binary() if compile_to_wasm: compiler.compile([os.path.join(musl, 'crt', 'crt1.c')]) shutil.copy(os.path.join(objdir, compiler.compiled[0]), os.path.dirname(out)) def getargs(): parser = argparse.ArgumentParser(description='Build a hacky wasm libc.') parser.add_argument('--clang_dir', type=str, required=True, help='Clang binary directory') parser.add_argument('--binaryen_dir', type=str, required=True, help='binaryen binary directory') parser.add_argument('--sexpr_wasm', type=str, required=False, help='sexpr-wasm binary') parser.add_argument('--musl', type=str, required=True, help='musl libc root directory') parser.add_argument('--arch', type=str, default='wasm32', help='architecture to target') parser.add_argument('--out', '-o', type=str, default=os.path.join(os.getcwd(), 'musl.wast'), help='Output file') parser.add_argument('--verbose', default=False, action='store_true', help='Verbose') parser.add_argument('--compile-to-wasm', default=False, action='store_true', help='Use clang to compile directly to wasm') return parser.parse_args() def main(): global verbose args = getargs() if args.verbose: verbose = True return run(args.clang_dir, args.binaryen_dir, args.sexpr_wasm, args.musl, args.arch, args.out, args.compile_to_wasm) if __name__ == '__main__': sys.exit(main())