Don't use JSON for custom section format

This commit migrates away from using Serde for the custom section in
wasm executables. This is a refactoring of a purely-internal data
structure to `wasm-bindgen` and should have no visible functional change
on users.

The motivation for this commit is two fold:

* First, the compile times using `serde_json` and `serde_derive` for the
  syntax extension isn't the most fun.
* Second, eventually we're going to want to stablize the layout of the
  custom section, and it's highly unlikely to be json!

Primarily, though, the intention of this commit is to improve the
cold-cache compile time of `wasm-bindgen` by ensuring that for new users
this project builds as quickly as possible. By removing some heavyweight
dependencies from the procedural macro, `serde`, `serde_derive`, and
`serde_json`, we're able to get a pretty nice build time improvement for
the `wasm-bindgen` crate itself:

|             | single-core build | parallel build |
|-------------|-------------------|----------------|
| master      |             36.5s |          17.3s |
| this commit |             20.5s |          11.8s |

These are't really end-all-be-all wins but they're much better
especially on the spectrum of weaker CPUs (in theory modeled by the
single-core case showing we have 42% less CPU work in theory).

crates/backend/Cargo.toml

@@ -19,6 +19,5 @@ lazy_static = "1.0.0"
 log = "0.4"
 proc-macro2 = "0.4.8"
 quote = '0.6'
-serde_json = "1.0"
-syn = { version = '0.15', features = ['full', 'visit'] }
+syn = { version = '0.15', features = ['full'] }
 wasm-bindgen-shared = { path = "../shared", version = "=0.2.25" }

crates/backend/src/ast.rs

@@ -1,9 +1,8 @@
+use Diagnostic;
 use proc_macro2::{Ident, Span};
 use shared;
 use syn;
 
-use Diagnostic;
-
 /// An abstract syntax tree representing a rust program. Contains
 /// extra information for joining up this rust code with javascript.
 #[cfg_attr(feature = "extra-traits", derive(Debug, PartialEq))]
@@ -260,36 +259,6 @@ pub struct DictionaryField {
     pub ty: syn::Type,
 }
 
-impl Program {
-    pub(crate) fn shared(&self) -> Result<shared::Program, Diagnostic> {
-        let mut errors = Vec::new();
-        let mut imports = Vec::new();
-        for import in self.imports.iter() {
-            match import.shared() {
-                Ok(i) => imports.push(i),
-                Err(e) => errors.push(e),
-            }
-        }
-        Diagnostic::from_vec(errors)?;
-        Ok(shared::Program {
-            exports: self.exports.iter().map(|a| a.shared()).collect(),
-            structs: self.structs.iter().map(|a| a.shared()).collect(),
-            enums: self.enums.iter().map(|a| a.shared()).collect(),
-            imports,
-            version: shared::version(),
-            schema_version: shared::SCHEMA_VERSION.to_string(),
-        })
-    }
-}
-
-impl Function {
-    fn shared(&self) -> shared::Function {
-        shared::Function {
-            name: self.name.to_string(),
-        }
-    }
-}
-
 impl Export {
     /// Mangles a rust -> javascript export, so that the created Ident will be unique over function
     /// name and class name, if the function belongs to a javascript class.
@@ -314,51 +283,6 @@ impl Export {
             None => shared::free_function_export_name(&fn_name),
         }
     }
-
-    fn shared(&self) -> shared::Export {
-        let (method, consumed) = match self.method_self {
-            Some(MethodSelf::ByValue) => (true, true),
-            Some(_) => (true, false),
-            None => (false, false),
-        };
-        shared::Export {
-            class: self.class.as_ref().map(|s| s.to_string()),
-            method,
-            consumed,
-            is_constructor: self.is_constructor,
-            function: self.function.shared(),
-            comments: self.comments.clone(),
-        }
-    }
-}
-
-impl Enum {
-    fn shared(&self) -> shared::Enum {
-        shared::Enum {
-            name: self.name.to_string(),
-            variants: self.variants.iter().map(|v| v.shared()).collect(),
-            comments: self.comments.clone(),
-        }
-    }
-}
-
-impl Variant {
-    fn shared(&self) -> shared::EnumVariant {
-        shared::EnumVariant {
-            name: self.name.to_string(),
-            value: self.value,
-        }
-    }
-}
-
-impl Import {
-    fn shared(&self) -> Result<shared::Import, Diagnostic> {
-        Ok(shared::Import {
-            module: self.module.clone(),
-            js_namespace: self.js_namespace.as_ref().map(|s| s.to_string()),
-            kind: self.kind.shared()?,
-        })
-    }
 }
 
 impl ImportKind {
@@ -371,27 +295,18 @@ impl ImportKind {
             ImportKind::Enum(_) => false,
         }
     }
-
-    fn shared(&self) -> Result<shared::ImportKind, Diagnostic> {
-        Ok(match *self {
-            ImportKind::Function(ref f) => shared::ImportKind::Function(f.shared()?),
-            ImportKind::Static(ref f) => shared::ImportKind::Static(f.shared()),
-            ImportKind::Type(ref f) => shared::ImportKind::Type(f.shared()),
-            ImportKind::Enum(ref f) => shared::ImportKind::Enum(f.shared()),
-        })
-    }
 }
 
 impl ImportFunction {
     /// If the rust object has a `fn xxx(&self) -> MyType` method, get the name for a getter in
     /// javascript (in this case `xxx`, so you can write `val = obj.xxx`)
-    fn infer_getter_property(&self) -> String {
-        self.function.name.to_string()
+    pub fn infer_getter_property(&self) -> &str {
+        &self.function.name
     }
 
     /// If the rust object has a `fn set_xxx(&mut self, MyType)` style method, get the name
     /// for a setter in javascript (in this case `xxx`, so you can write `obj.xxx = val`)
-    fn infer_setter_property(&self) -> Result<String, Diagnostic> {
+    pub fn infer_setter_property(&self) -> Result<String, Diagnostic> {
         let name = self.function.name.to_string();
 
         // if `#[wasm_bindgen(js_name = "...")]` is used then that explicitly
@@ -410,102 +325,4 @@ impl ImportFunction {
         }
         Ok(name[4..].to_string())
     }
-
-    fn shared(&self) -> Result<shared::ImportFunction, Diagnostic> {
-        let shared_operation = |operation: &Operation| -> Result<_, Diagnostic> {
-            let is_static = operation.is_static;
-            let kind = match &operation.kind {
-                OperationKind::Regular => shared::OperationKind::Regular,
-                OperationKind::Getter(g) => {
-                    let g = g.as_ref().map(|g| g.to_string());
-                    shared::OperationKind::Getter(g.unwrap_or_else(|| self.infer_getter_property()))
-                }
-                OperationKind::Setter(s) => {
-                    let s = s.as_ref().map(|s| s.to_string());
-                    shared::OperationKind::Setter(match s {
-                        Some(s) => s,
-                        None => self.infer_setter_property()?,
-                    })
-                }
-                OperationKind::IndexingGetter => shared::OperationKind::IndexingGetter,
-                OperationKind::IndexingSetter => shared::OperationKind::IndexingSetter,
-                OperationKind::IndexingDeleter => shared::OperationKind::IndexingDeleter,
-            };
-            Ok(shared::Operation { is_static, kind })
-        };
-
-        let method = match self.kind {
-            ImportFunctionKind::Method {
-                ref class,
-                ref kind,
-                ..
-            } => {
-                let kind = match kind {
-                    MethodKind::Constructor => shared::MethodKind::Constructor,
-                    MethodKind::Operation(op) => {
-                        shared::MethodKind::Operation(shared_operation(op)?)
-                    }
-                };
-                Some(shared::MethodData {
-                    class: class.clone(),
-                    kind,
-                })
-            }
-            ImportFunctionKind::Normal => None,
-        };
-
-        Ok(shared::ImportFunction {
-            shim: self.shim.to_string(),
-            catch: self.catch,
-            variadic: self.variadic,
-            method,
-            structural: self.structural,
-            function: self.function.shared(),
-        })
-    }
-}
-
-impl ImportStatic {
-    fn shared(&self) -> shared::ImportStatic {
-        shared::ImportStatic {
-            name: self.js_name.to_string(),
-            shim: self.shim.to_string(),
-        }
-    }
-}
-
-impl ImportType {
-    fn shared(&self) -> shared::ImportType {
-        shared::ImportType {
-            name: self.js_name.clone(),
-            instanceof_shim: self.instanceof_shim.clone(),
-            vendor_prefixes: self.vendor_prefixes.iter().map(|s| s.to_string()).collect(),
-        }
-    }
-}
-
-impl ImportEnum {
-    fn shared(&self) -> shared::ImportEnum {
-        shared::ImportEnum {}
-    }
-}
-
-impl Struct {
-    fn shared(&self) -> shared::Struct {
-        shared::Struct {
-            name: self.name.to_string(),
-            fields: self.fields.iter().map(|s| s.shared()).collect(),
-            comments: self.comments.clone(),
-        }
-    }
-}
-
-impl StructField {
-    fn shared(&self) -> shared::StructField {
-        shared::StructField {
-            name: self.name.to_string(),
-            readonly: self.readonly,
-            comments: self.comments.clone(),
-        }
-    }
 }

crates/backend/src/codegen.rs

@@ -1,14 +1,14 @@
 use std::collections::HashSet;
-use std::sync::atomic::{AtomicUsize, Ordering, ATOMIC_USIZE_INIT};
 use std::sync::Mutex;
+use std::sync::atomic::{AtomicUsize, Ordering, ATOMIC_USIZE_INIT};
 
 use proc_macro2::{Ident, Literal, Span, TokenStream};
 use quote::ToTokens;
-use serde_json;
 use shared;
 use syn;
 
 use ast;
+use encode;
 use util::ShortHash;
 use Diagnostic;
 
@@ -87,20 +87,20 @@ impl TryToTokens for ast::Program {
         );
         let generated_static_name = Ident::new(&generated_static_name, Span::call_site());
 
-        let description = serde_json::to_string(&self.shared()?).unwrap();
+        // See comments in `crates/cli-support/src/lib.rs` about what this
+        // `schema_version` is.
+        let prefix_json = format!(r#"{{"schema_version":"{}","version":"{}"}}"#,
+                                  shared::SCHEMA_VERSION,
+                                  shared::version());
+        let mut bytes = Vec::new();
+        bytes.push((prefix_json.len() >> 0) as u8);
+        bytes.push((prefix_json.len() >> 8) as u8);
+        bytes.push((prefix_json.len() >> 16) as u8);
+        bytes.push((prefix_json.len() >> 24) as u8);
+        bytes.extend_from_slice(prefix_json.as_bytes());
+        bytes.extend_from_slice(&encode::encode(self)?);
 
-        // Each JSON blob is prepended with the length of the JSON blob so when
-        // all these sections are concatenated in the final wasm file we know
-        // how to extract all the JSON pieces, so insert the byte length here.
-        // The value is little-endian.
-        let generated_static_length = description.len() + 4;
-        let mut bytes = vec![
-            (description.len() >> 0) as u8,
-            (description.len() >> 8) as u8,
-            (description.len() >> 16) as u8,
-            (description.len() >> 24) as u8,
-        ];
-        bytes.extend_from_slice(description.as_bytes());
+        let generated_static_length = bytes.len();
         let generated_static_value = syn::LitByteStr::new(&bytes, Span::call_site());
 
         (quote! {

crates/backend/src/encode.rs

@@ -0,0 +1,368 @@
+use std::cell::RefCell;
+use std::collections::HashMap;
+
+use proc_macro2::{Ident, Span};
+
+use Diagnostic;
+use ast;
+
+pub fn encode(program: &ast::Program) -> Result<Vec<u8>, Diagnostic> {
+    let mut e = Encoder::new();
+    let i = Interner::new();
+    shared_program(program, &i)?.encode(&mut e);
+    Ok(e.finish())
+}
+
+struct Interner {
+    map: RefCell<HashMap<Ident, String>>,
+}
+
+impl Interner {
+    fn new() -> Interner {
+        Interner { map: RefCell::new(HashMap::new()) }
+    }
+
+    fn intern(&self, s: &Ident) -> &str {
+        let mut map = self.map.borrow_mut();
+        if let Some(s) = map.get(s) {
+            return unsafe { &*(&**s as *const str) }
+        }
+        map.insert(s.clone(), s.to_string());
+        unsafe { &*(&*map[s] as *const str) }
+    }
+
+    fn intern_str(&self, s: &str) -> &str {
+        self.intern(&Ident::new(s, Span::call_site()))
+    }
+}
+
+fn shared_program<'a>(prog: &'a ast::Program, intern: &'a Interner)
+    -> Result<Program<'a>, Diagnostic>
+{
+    Ok(Program {
+        exports: prog.exports.iter().map(|a| shared_export(a, intern)).collect(),
+        structs: prog.structs.iter().map(|a| shared_struct(a, intern)).collect(),
+        enums: prog.enums.iter().map(|a| shared_enum(a, intern)).collect(),
+        imports: prog.imports.iter()
+            .map(|a| shared_import(a, intern))
+            .collect::<Result<Vec<_>, _>>()?,
+        // version: shared::version(),
+        // schema_version: shared::SCHEMA_VERSION.to_string(),
+    })
+}
+
+fn shared_export<'a>(export: &'a ast::Export, intern: &'a Interner) -> Export<'a> {
+    let (method, consumed) = match export.method_self {
+        Some(ast::MethodSelf::ByValue) => (true, true),
+        Some(_) => (true, false),
+        None => (false, false),
+    };
+    Export {
+        class: export.class.as_ref().map(|s| intern.intern(s)),
+        method,
+        consumed,
+        is_constructor: export.is_constructor,
+        function: shared_function(&export.function, intern),
+        comments: export.comments.iter().map(|s| &**s).collect(),
+    }
+}
+
+fn shared_function<'a>(func: &'a ast::Function, _intern: &'a Interner) -> Function<'a> {
+    Function {
+        name: &func.name,
+    }
+}
+
+fn shared_enum<'a>(e: &'a ast::Enum, intern: &'a Interner) -> Enum<'a> {
+    Enum {
+        name: intern.intern(&e.name),
+        variants: e.variants.iter().map(|v| shared_variant(v, intern)).collect(),
+        comments: e.comments.iter().map(|s| &**s).collect(),
+    }
+}
+
+fn shared_variant<'a>(v: &'a ast::Variant, intern: &'a Interner) -> EnumVariant<'a> {
+    EnumVariant {
+        name: intern.intern(&v.name),
+        value: v.value,
+    }
+}
+
+fn shared_import<'a>(i: &'a ast::Import, intern: &'a Interner)
+    -> Result<Import<'a>, Diagnostic>
+{
+    Ok(Import {
+        module: i.module.as_ref().map(|s| &**s),
+        js_namespace: i.js_namespace.as_ref().map(|s| intern.intern(s)),
+        kind: shared_import_kind(&i.kind, intern)?,
+    })
+}
+
+fn shared_import_kind<'a>(i: &'a ast::ImportKind, intern: &'a Interner)
+    -> Result<ImportKind<'a>, Diagnostic>
+{
+    Ok(match i {
+        ast::ImportKind::Function(f) => ImportKind::Function(shared_import_function(f, intern)?),
+        ast::ImportKind::Static(f) => ImportKind::Static(shared_import_static(f, intern)),
+        ast::ImportKind::Type(f) => ImportKind::Type(shared_import_type(f, intern)),
+        ast::ImportKind::Enum(f) => ImportKind::Enum(shared_import_enum(f, intern)),
+    })
+}
+
+fn shared_import_function<'a>(i: &'a ast::ImportFunction, intern: &'a Interner)
+    -> Result<ImportFunction<'a>, Diagnostic>
+{
+    let method = match &i.kind {
+        ast::ImportFunctionKind::Method { class, kind, ..  } => {
+            let kind = match kind {
+                ast::MethodKind::Constructor => MethodKind::Constructor,
+                ast::MethodKind::Operation(ast::Operation { is_static, kind }) => {
+                    let is_static = *is_static;
+                    let kind = match kind {
+                        ast::OperationKind::Regular => OperationKind::Regular,
+                        ast::OperationKind::Getter(g) => {
+                            let g = g.as_ref().map(|g| intern.intern(g));
+                            OperationKind::Getter(
+                                g.unwrap_or_else(|| i.infer_getter_property()),
+                            )
+                        }
+                        ast::OperationKind::Setter(s) => {
+                            let s = s.as_ref().map(|s| intern.intern(s));
+                            OperationKind::Setter(match s {
+                                Some(s) => s,
+                                None => intern.intern_str(&i.infer_setter_property()?),
+                            })
+                        }
+                        ast::OperationKind::IndexingGetter => OperationKind::IndexingGetter,
+                        ast::OperationKind::IndexingSetter => OperationKind::IndexingSetter,
+                        ast::OperationKind::IndexingDeleter => OperationKind::IndexingDeleter,
+                    };
+                    MethodKind::Operation(Operation { is_static, kind })
+                }
+            };
+            Some(MethodData {
+                class,
+                kind,
+            })
+        }
+        ast::ImportFunctionKind::Normal => None,
+    };
+
+    Ok(ImportFunction {
+        shim: intern.intern(&i.shim),
+        catch: i.catch,
+        method,
+        structural: i.structural,
+        function: shared_function(&i.function, intern),
+        variadic: i.variadic,
+    })
+}
+
+fn shared_import_static<'a>(i: &'a ast::ImportStatic, intern: &'a Interner)
+    -> ImportStatic<'a>
+{
+    ImportStatic {
+        name: &i.js_name,
+        shim: intern.intern(&i.shim),
+    }
+}
+
+fn shared_import_type<'a>(i: &'a ast::ImportType, intern: &'a Interner)
+    -> ImportType<'a>
+{
+    ImportType {
+        name: &i.js_name,
+        instanceof_shim: &i.instanceof_shim,
+        vendor_prefixes: i.vendor_prefixes.iter()
+            .map(|x| intern.intern(x))
+            .collect(),
+    }
+}
+
+fn shared_import_enum<'a>(_i: &'a ast::ImportEnum, _intern: &'a Interner)
+    -> ImportEnum
+{
+    ImportEnum {}
+}
+
+fn shared_struct<'a>(s: &'a ast::Struct, intern: &'a Interner) -> Struct<'a> {
+    Struct {
+        name: intern.intern(&s.name),
+        fields: s.fields.iter().map(|s| shared_struct_field(s, intern)).collect(),
+        comments: s.comments.iter().map(|s| &**s).collect(),
+    }
+}
+
+fn shared_struct_field<'a>(s: &'a ast::StructField, intern: &'a Interner)
+    -> StructField<'a>
+{
+    StructField {
+        name: intern.intern(&s.name),
+        readonly: s.readonly,
+        comments: s.comments.iter().map(|s| &**s).collect(),
+    }
+}
+
+trait Encode {
+    fn encode(&self, dst: &mut Encoder);
+}
+
+struct Encoder {
+    dst: Vec<u8>,
+}
+
+impl Encoder {
+    fn new() -> Encoder {
+        Encoder {
+            dst: vec![0, 0, 0, 0],
+        }
+    }
+
+    fn finish(mut self) -> Vec<u8> {
+        let len = self.dst.len() - 4;
+        self.dst[0] = (len >> 0) as u8;
+        self.dst[1] = (len >> 8) as u8;
+        self.dst[2] = (len >> 16) as u8;
+        self.dst[3] = (len >> 24) as u8;
+        self.dst
+    }
+
+    fn byte(&mut self, byte: u8) {
+        self.dst.push(byte);
+    }
+}
+
+impl Encode for bool {
+    fn encode(&self, dst: &mut Encoder) {
+        dst.byte(*self as u8);
+    }
+}
+
+impl Encode for u32 {
+    fn encode(&self, dst: &mut Encoder) {
+        let mut val = *self;
+        while (val >> 7) != 0 {
+            dst.byte((val as u8) | 0x80);
+            val >>= 7;
+        }
+        assert_eq!(val >> 7, 0);
+        dst.byte(val as u8);
+    }
+}
+
+impl Encode for usize {
+    fn encode(&self, dst: &mut Encoder) {
+        assert!(*self <= u32::max_value() as usize);
+        (*self as u32).encode(dst);
+    }
+}
+
+impl<'a> Encode for &'a [u8] {
+    fn encode(&self, dst: &mut Encoder) {
+        self.len().encode(dst);
+        dst.dst.extend_from_slice(*self);
+    }
+}
+
+impl<'a> Encode for &'a str {
+    fn encode(&self, dst: &mut Encoder) {
+        self.as_bytes().encode(dst);
+    }
+}
+
+impl<T: Encode> Encode for Vec<T> {
+    fn encode(&self, dst: &mut Encoder) {
+        self.len().encode(dst);
+        for item in self {
+            item.encode(dst);
+        }
+    }
+}
+
+impl<T: Encode> Encode for Option<T> {
+    fn encode(&self, dst: &mut Encoder) {
+        match self {
+            None => dst.byte(0),
+            Some(val) => {
+                dst.byte(1);
+                val.encode(dst)
+            }
+        }
+    }
+}
+
+macro_rules! encode_struct {
+    ($name:ident ($($lt:tt)*) $($field:ident: $ty:ty,)*) => {
+        struct $name $($lt)* {
+            $($field: $ty,)*
+        }
+
+        impl $($lt)* Encode for $name $($lt)* {
+            fn encode(&self, _dst: &mut Encoder) {
+                $(self.$field.encode(_dst);)*
+            }
+        }
+    }
+}
+
+macro_rules! encode_enum {
+    ($name:ident ($($lt:tt)*) $($fields:tt)*) => (
+        enum $name $($lt)* { $($fields)* }
+
+        impl$($lt)* Encode for $name $($lt)* {
+            fn encode(&self, dst: &mut Encoder) {
+                use self::$name::*;
+                encode_enum!(@arms self dst (0) () $($fields)*)
+            }
+        }
+    );
+
+    (@arms $me:ident $dst:ident ($cnt:expr) ($($arms:tt)*)) => (
+        encode_enum!(@expr match $me { $($arms)* })
+    );
+
+    (@arms $me:ident $dst:ident ($cnt:expr) ($($arms:tt)*) $name:ident, $($rest:tt)*) => (
+        encode_enum!(
+            @arms
+            $me
+            $dst
+            ($cnt+1)
+            ($($arms)* $name => $dst.byte($cnt),)
+            $($rest)*
+        )
+    );
+
+    (@arms $me:ident $dst:ident ($cnt:expr) ($($arms:tt)*) $name:ident($t:ty), $($rest:tt)*) => (
+        encode_enum!(
+            @arms
+            $me
+            $dst
+            ($cnt+1)
+            ($($arms)* $name(val) => { $dst.byte($cnt); val.encode($dst) })
+            $($rest)*
+        )
+    );
+
+    (@expr $e:expr) => ($e);
+}
+
+macro_rules! encode_api {
+    () => ();
+    (struct $name:ident<'a> { $($fields:tt)* } $($rest:tt)*) => (
+        encode_struct!($name (<'a>) $($fields)*);
+        encode_api!($($rest)*);
+    );
+    (struct $name:ident { $($fields:tt)* } $($rest:tt)*) => (
+        encode_struct!($name () $($fields)*);
+        encode_api!($($rest)*);
+    );
+    (enum $name:ident<'a> { $($variants:tt)* } $($rest:tt)*) => (
+        encode_enum!($name (<'a>) $($variants)*);
+        encode_api!($($rest)*);
+    );
+    (enum $name:ident { $($variants:tt)* } $($rest:tt)*) => (
+        encode_enum!($name () $($variants)*);
+        encode_api!($($rest)*);
+    );
+}
+shared_api!(encode_api);

crates/backend/src/lib.rs

@@ -5,16 +5,16 @@
 )]
 #![doc(html_root_url = "https://docs.rs/wasm-bindgen-backend/0.2")]
 
-#[macro_use]
-extern crate lazy_static;
 #[macro_use]
 extern crate log;
 extern crate proc_macro2;
 #[macro_use]
 extern crate quote;
-extern crate serde_json;
 extern crate syn;
+#[macro_use]
+extern crate lazy_static;
 
+#[macro_use]
 extern crate wasm_bindgen_shared as shared;
 
 pub use codegen::TryToTokens;
@@ -25,5 +25,6 @@ mod error;
 
 pub mod ast;
 mod codegen;
+mod encode;
 pub mod defined;
 pub mod util;