Move out parser and codegen traits.

lib/runtime-core/src/codegen.rs

@@ -0,0 +1,240 @@
+use crate::{
+    backend::RunnableModule,
+    structures::Map,
+    types::{FuncIndex, FuncSig, SigIndex},
+    backend::{sys::Memory, Backend, CacheGen, Compiler, CompilerConfig, Token},
+    cache::{Artifact, Error as CacheError},
+    error::{CompileError, CompileResult},
+    module::{ModuleInfo, ModuleInner},
+    parse::LoadError,
+};
+use wasmparser::{Operator, Type as WpType};
+use std::fmt::Debug;
+use smallvec::SmallVec;
+use std::marker::PhantomData;
+
+pub enum Event<'a, 'b> {
+    Internal(InternalEvent),
+    Wasm(&'b Operator<'a>),
+}
+
+#[derive(Copy, Clone, Debug)]
+pub enum InternalEvent {
+    FunctionBegin,
+    FunctionEnd,
+    Trace,
+}
+
+pub trait ModuleCodeGenerator<FCG: FunctionCodeGenerator<E>, RM: RunnableModule, E: Debug> {
+    fn new() -> Self;
+    fn backend_id() -> Backend;
+    fn check_precondition(&mut self, module_info: &ModuleInfo) -> Result<(), E>;
+
+    /// Creates a new function and returns the function-scope code generator for it.
+    fn next_function(&mut self) -> Result<&mut FCG, E>;
+    fn finalize(self, module_info: &ModuleInfo) -> Result<RM, E>;
+    fn feed_signatures(&mut self, signatures: Map<SigIndex, FuncSig>) -> Result<(), E>;
+
+    /// Sets function signatures.
+    fn feed_function_signatures(
+        &mut self,
+        assoc: Map<FuncIndex, SigIndex>,
+    ) -> Result<(), E>;
+
+    /// Adds an import function.
+    fn feed_import_function(&mut self) -> Result<(), E>;
+}
+
+pub struct StreamingCompiler<
+    MCG: ModuleCodeGenerator<FCG, RM, E>,
+    FCG: FunctionCodeGenerator<E>,
+    RM: RunnableModule + 'static,
+    E: Debug,
+    CGEN: Fn() -> MiddlewareChain,
+> {
+    middleware_chain_generator: CGEN,
+    _phantom_mcg: PhantomData<MCG>,
+    _phantom_fcg: PhantomData<FCG>,
+    _phantom_rm: PhantomData<RM>,
+    _phantom_e: PhantomData<E>,
+}
+
+pub struct SimpleStreamingCompilerGen<
+    MCG: ModuleCodeGenerator<FCG, RM, E>,
+    FCG: FunctionCodeGenerator<E>,
+    RM: RunnableModule + 'static,
+    E: Debug,
+> {
+    _phantom_mcg: PhantomData<MCG>,
+    _phantom_fcg: PhantomData<FCG>,
+    _phantom_rm: PhantomData<RM>,
+    _phantom_e: PhantomData<E>,
+}
+
+impl<
+    MCG: ModuleCodeGenerator<FCG, RM, E>,
+    FCG: FunctionCodeGenerator<E>,
+    RM: RunnableModule + 'static,
+    E: Debug,
+> SimpleStreamingCompilerGen<MCG, FCG, RM, E> {
+    pub fn new() -> StreamingCompiler<MCG, FCG, RM, E, impl Fn() -> MiddlewareChain> {
+        StreamingCompiler::new(|| MiddlewareChain::new())
+    }
+}
+
+impl<
+    MCG: ModuleCodeGenerator<FCG, RM, E>,
+    FCG: FunctionCodeGenerator<E>,
+    RM: RunnableModule + 'static,
+    E: Debug,
+    CGEN: Fn() -> MiddlewareChain,
+> StreamingCompiler<MCG, FCG, RM, E, CGEN> {
+    pub fn new(chain_gen: CGEN) -> Self {
+        Self {
+            middleware_chain_generator: chain_gen,
+            _phantom_mcg: PhantomData,
+            _phantom_fcg: PhantomData,
+            _phantom_rm: PhantomData,
+            _phantom_e: PhantomData,
+        }
+    }
+}
+
+impl<
+    MCG: ModuleCodeGenerator<FCG, RM, E>,
+    FCG: FunctionCodeGenerator<E>,
+    RM: RunnableModule + 'static,
+    E: Debug,
+    CGEN: Fn() -> MiddlewareChain,
+>Compiler for StreamingCompiler<MCG, FCG, RM, E, CGEN> {
+    fn compile(
+        &self,
+        wasm: &[u8],
+        compiler_config: CompilerConfig,
+        _: Token,
+    ) -> CompileResult<ModuleInner> {
+        struct Placeholder;
+        impl CacheGen for Placeholder {
+            fn generate_cache(&self) -> Result<(Box<[u8]>, Memory), CacheError> {
+                Err(CacheError::Unknown(
+                    "the singlepass backend doesn't support caching yet".to_string(),
+                ))
+            }
+        }
+
+        let mut mcg = MCG::new();
+        let mut chain = (self.middleware_chain_generator)();
+        let info = crate::parse::read_module(wasm, MCG::backend_id(), &mut mcg, &mut chain, &compiler_config)?;
+        let exec_context = mcg.finalize(&info).map_err(|x| CompileError::InternalError {
+            msg: format!("{:?}", x),
+        })?;
+        Ok(ModuleInner {
+            cache_gen: Box::new(Placeholder),
+            runnable_module: Box::new(exec_context),
+            info: info,
+        })
+    }
+
+    unsafe fn from_cache(&self, _artifact: Artifact, _: Token) -> Result<ModuleInner, CacheError> {
+        Err(CacheError::Unknown(
+            "the singlepass backend doesn't support caching yet".to_string(),
+        ))
+    }
+}
+
+pub struct EventSink<'a, 'b> {
+    buffer: SmallVec<[Event<'a, 'b>; 2]>
+}
+
+impl<'a, 'b> EventSink<'a, 'b> {
+    pub fn push(&mut self, ev: Event<'a, 'b>) {
+        self.buffer.push(ev);
+    }
+}
+
+pub struct MiddlewareChain {
+    chain: Vec<Box<GenericFunctionMiddleware>>,
+}
+
+impl MiddlewareChain {
+    pub fn new() -> MiddlewareChain {
+        MiddlewareChain {
+            chain: vec! [],
+        }
+    }
+
+    pub fn push<M: FunctionMiddleware + 'static>(&mut self, m: M) {
+        self.chain.push(Box::new(m));
+    }
+
+    pub(crate) fn run<E: Debug, FCG: FunctionCodeGenerator<E>>(&mut self, fcg: Option<&mut FCG>, ev: Event, module_info: &ModuleInfo) -> Result<(), String> {
+        let mut sink = EventSink {
+            buffer: SmallVec::new(),
+        };
+        sink.push(ev);
+        for m in &mut self.chain {
+            let prev: SmallVec<[Event; 2]> = sink.buffer.drain().collect();
+            for ev in prev {
+                m.feed_event(ev, module_info, &mut sink)?;
+            }
+        }
+        if let Some(fcg) = fcg {
+            for ev in sink.buffer {
+                fcg.feed_event(ev, module_info).map_err(|x| format!("{:?}", x))?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+pub trait FunctionMiddleware {
+    type Error: Debug;
+    fn feed_event(
+        &mut self,
+        op: Event,
+        module_info: &ModuleInfo,
+        sink: &mut EventSink,
+    ) -> Result<(), Self::Error>;
+}
+
+pub(crate) trait GenericFunctionMiddleware {
+    fn feed_event(
+        &mut self,
+        op: Event,
+        module_info: &ModuleInfo,
+        sink: &mut EventSink,
+    ) -> Result<(), String>;
+}
+
+impl<E: Debug, T: FunctionMiddleware<Error = E>> GenericFunctionMiddleware for T {
+    fn feed_event(
+        &mut self,
+        op: Event,
+        module_info: &ModuleInfo,
+        sink: &mut EventSink,
+    ) -> Result<(), String> {
+        <Self as FunctionMiddleware>::feed_event(self, op, module_info, sink).map_err(|x| format!("{:?}", x))
+    }
+}
+
+/// The function-scope code generator trait.
+pub trait FunctionCodeGenerator<E: Debug> {
+    /// Sets the return type.
+    fn feed_return(&mut self, ty: WpType) -> Result<(), E>;
+
+    /// Adds a parameter to the function.
+    fn feed_param(&mut self, ty: WpType) -> Result<(), E>;
+
+    /// Adds `n` locals to the function.
+    fn feed_local(&mut self, ty: WpType, n: usize) -> Result<(), E>;
+
+    /// Called before the first call to `feed_opcode`.
+    fn begin_body(&mut self) -> Result<(), E>;
+
+    /// Called for each operator.
+    fn feed_event(&mut self, op: Event, module_info: &ModuleInfo) -> Result<(), E>;
+
+    /// Finalizes the function.
+    fn finalize(&mut self) -> Result<(), E>;
+}

lib/runtime-core/src/lib.rs

@@ -31,6 +31,8 @@ pub mod units;
 pub mod vm;
 #[doc(hidden)]
 pub mod vmcalls;
+pub mod codegen;
+pub mod parse;
 
 use self::error::CompileResult;
 #[doc(inline)]

lib/runtime-core/src/parse.rs

@@ -1,6 +1,6 @@
-use crate::codegen::{CodegenError, FunctionCodeGenerator, ModuleCodeGenerator};
+use crate::codegen::*;
 use hashbrown::HashMap;
-use wasmer_runtime_core::{
+use crate::{
     backend::{Backend, CompilerConfig, RunnableModule},
     module::{
         DataInitializer, ExportIndex, ImportName, ModuleInfo, StringTable, StringTableBuilder,
@@ -13,17 +13,27 @@ use wasmer_runtime_core::{
         TableIndex, Type, Value,
     },
     units::Pages,
+    error::CompileError,
 };
 use wasmparser::{
     BinaryReaderError, Data, DataKind, Element, ElementKind, Export, ExternalKind, FuncType,
     Import, ImportSectionEntryType, InitExpr, ModuleReader, Operator, SectionCode, Type as WpType,
     WasmDecoder,
 };
+use std::fmt::Debug;
 
 #[derive(Debug)]
 pub enum LoadError {
     Parse(BinaryReaderError),
-    Codegen(CodegenError),
+    Codegen(String),
+}
+
+impl From<LoadError> for CompileError {
+    fn from(other: LoadError) -> CompileError {
+        CompileError::InternalError {
+            msg: format!("{:?}", other),
+        }
+    }
 }
 
 impl From<BinaryReaderError> for LoadError {
@@ -32,20 +42,16 @@ impl From<BinaryReaderError> for LoadError {
     }
 }
 
-impl From<CodegenError> for LoadError {
-    fn from(other: CodegenError) -> LoadError {
-        LoadError::Codegen(other)
-    }
-}
-
 pub fn read_module<
-    MCG: ModuleCodeGenerator<FCG, RM>,
+    MCG: ModuleCodeGenerator<FCG, RM, E>,
-    FCG: FunctionCodeGenerator,
+    FCG: FunctionCodeGenerator<E>,
     RM: RunnableModule,
+    E: Debug,
 >(
     wasm: &[u8],
     backend: Backend,
     mcg: &mut MCG,
+    middlewares: &mut MiddlewareChain,
     compiler_config: &CompilerConfig,
 ) -> Result<ModuleInfo, LoadError> {
     let mut info = ModuleInfo {
@@ -116,7 +122,7 @@ pub fn read_module<
                         let sigindex = SigIndex::new(sigindex as usize);
                         info.imported_functions.push(import_name);
                         info.func_assoc.push(sigindex);
-                        mcg.feed_import_function()?;
+                        mcg.feed_import_function().map_err(|x| LoadError::Codegen(format!("{:?}", x)))?;
                     }
                     ImportSectionEntryType::Table(table_ty) => {
                         assert_eq!(table_ty.element_type, WpType::AnyFunc);
@@ -186,12 +192,14 @@ pub fn read_module<
                 if func_count == 0 {
                     info.namespace_table = namespace_builder.take().unwrap().finish();
                     info.name_table = name_builder.take().unwrap().finish();
-                    mcg.feed_signatures(info.signatures.clone())?;
+                    mcg.feed_signatures(info.signatures.clone()).map_err(|x| LoadError::Codegen(format!("{:?}", x)))?;
-                    mcg.feed_function_signatures(info.func_assoc.clone())?;
+                    mcg.feed_function_signatures(info.func_assoc.clone()).map_err(|x| LoadError::Codegen(format!("{:?}", x)))?;
-                    mcg.check_precondition(&info)?;
+                    mcg.check_precondition(&info).map_err(|x| LoadError::Codegen(format!("{:?}", x)))?;
                 }
 
-                let fcg = mcg.next_function()?;
+                let fcg = mcg.next_function().map_err(|x| LoadError::Codegen(format!("{:?}", x)))?;
+                middlewares.run(Some(fcg), Event::Internal(InternalEvent::FunctionBegin), &info).map_err(|x| LoadError::Codegen(x))?;
+
                 let sig = info
                     .signatures
                     .get(
@@ -202,10 +210,10 @@ pub fn read_module<
                     )
                     .unwrap();
                 for ret in sig.returns() {
-                    fcg.feed_return(type_to_wp_type(*ret))?;
+                    fcg.feed_return(type_to_wp_type(*ret)).map_err(|x| LoadError::Codegen(format!("{:?}", x)))?;
                 }
                 for param in sig.params() {
-                    fcg.feed_param(type_to_wp_type(*param))?;
+                    fcg.feed_param(type_to_wp_type(*param)).map_err(|x| LoadError::Codegen(format!("{:?}", x)))?;
                 }
 
                 let mut body_begun = false;
@@ -216,21 +224,22 @@ pub fn read_module<
                         ParserState::Error(err) => return Err(LoadError::Parse(err)),
                         ParserState::FunctionBodyLocals { ref locals } => {
                             for &(count, ty) in locals.iter() {
-                                fcg.feed_local(ty, count as usize)?;
+                                fcg.feed_local(ty, count as usize).map_err(|x| LoadError::Codegen(format!("{:?}", x)))?;
                             }
                         }
                         ParserState::CodeOperator(ref op) => {
                             if !body_begun {
                                 body_begun = true;
-                                fcg.begin_body()?;
+                                fcg.begin_body().map_err(|x| LoadError::Codegen(format!("{:?}", x)))?;
                             }
-                            fcg.feed_opcode(op, &info)?;
+                            middlewares.run(Some(fcg), Event::Wasm(op), &info).map_err(|x| LoadError::Codegen(x))?;
                         }
                         ParserState::EndFunctionBody => break,
                         _ => unreachable!(),
                     }
                 }
-                fcg.finalize()?;
+                middlewares.run(Some(fcg), Event::Internal(InternalEvent::FunctionEnd), &info).map_err(|x| LoadError::Codegen(x))?;
+                fcg.finalize().map_err(|x| LoadError::Codegen(format!("{:?}", x)))?;
             }
             ParserState::BeginActiveElementSectionEntry(table_index) => {
                 let table_index = TableIndex::new(table_index as usize);

lib/singlepass-backend/Cargo.toml

@@ -9,7 +9,7 @@ edition = "2018"
 
 [dependencies]
 wasmer-runtime-core = { path = "../runtime-core", version = "0.3.0" }
-wasmparser = "0.28.0"
+wasmparser = "0.29.2"
 dynasm = "0.3.1"
 dynasmrt = "0.3.1"
 lazy_static = "1.2.0"

lib/singlepass-backend/src/codegen.rs

@@ -1,51 +0,0 @@
-use wasmer_runtime_core::{
-    backend::RunnableModule,
-    module::ModuleInfo,
-    structures::Map,
-    types::{FuncIndex, FuncSig, SigIndex},
-};
-use wasmparser::{Operator, Type as WpType};
-
-pub trait ModuleCodeGenerator<FCG: FunctionCodeGenerator, RM: RunnableModule> {
-    fn check_precondition(&mut self, module_info: &ModuleInfo) -> Result<(), CodegenError>;
-
-    /// Creates a new function and returns the function-scope code generator for it.
-    fn next_function(&mut self) -> Result<&mut FCG, CodegenError>;
-    fn finalize(self, module_info: &ModuleInfo) -> Result<RM, CodegenError>;
-    fn feed_signatures(&mut self, signatures: Map<SigIndex, FuncSig>) -> Result<(), CodegenError>;
-
-    /// Sets function signatures.
-    fn feed_function_signatures(
-        &mut self,
-        assoc: Map<FuncIndex, SigIndex>,
-    ) -> Result<(), CodegenError>;
-
-    /// Adds an import function.
-    fn feed_import_function(&mut self) -> Result<(), CodegenError>;
-}
-
-/// The function-scope code generator trait.
-pub trait FunctionCodeGenerator {
-    /// Sets the return type.
-    fn feed_return(&mut self, ty: WpType) -> Result<(), CodegenError>;
-
-    /// Adds a parameter to the function.
-    fn feed_param(&mut self, ty: WpType) -> Result<(), CodegenError>;
-
-    /// Adds `n` locals to the function.
-    fn feed_local(&mut self, ty: WpType, n: usize) -> Result<(), CodegenError>;
-
-    /// Called before the first call to `feed_opcode`.
-    fn begin_body(&mut self) -> Result<(), CodegenError>;
-
-    /// Called for each operator.
-    fn feed_opcode(&mut self, op: &Operator, module_info: &ModuleInfo) -> Result<(), CodegenError>;
-
-    /// Finalizes the function.
-    fn finalize(&mut self) -> Result<(), CodegenError>;
-}
-
-#[derive(Debug)]
-pub struct CodegenError {
-    pub message: &'static str,
-}

lib/singlepass-backend/src/codegen_x64.rs

@@ -1,6 +1,5 @@
 #![allow(clippy::forget_copy)] // Used by dynasm.
 
-use super::codegen::*;
 use crate::emitter_x64::*;
 use crate::machine::*;
 use crate::protect_unix;
@@ -11,7 +10,7 @@ use smallvec::SmallVec;
 use std::ptr::NonNull;
 use std::{any::Any, collections::HashMap, sync::Arc};
 use wasmer_runtime_core::{
-    backend::RunnableModule,
+    backend::{Backend, RunnableModule},
     memory::MemoryType,
     module::ModuleInfo,
     structures::{Map, TypedIndex},
@@ -22,6 +21,7 @@ use wasmer_runtime_core::{
     },
     vm::{self, LocalGlobal, LocalMemory, LocalTable},
     vmcalls,
+    codegen::*,
 };
 use wasmparser::{Operator, Type as WpType};
 
@@ -262,8 +262,13 @@ impl RunnableModule for X64ExecutionContext {
     }
 }
 
-impl X64ModuleCodeGenerator {
+#[derive(Debug)]
-    pub fn new() -> X64ModuleCodeGenerator {
+pub struct CodegenError {
+    pub message: &'static str,
+}
+
+impl ModuleCodeGenerator<X64FunctionCode, X64ExecutionContext, CodegenError> for X64ModuleCodeGenerator {
+    fn new() -> X64ModuleCodeGenerator {
         X64ModuleCodeGenerator {
             functions: vec![],
             signatures: None,
@@ -273,9 +278,11 @@ impl X64ModuleCodeGenerator {
             func_import_count: 0,
         }
     }
-}
 
-impl ModuleCodeGenerator<X64FunctionCode, X64ExecutionContext> for X64ModuleCodeGenerator {
+    fn backend_id() -> Backend {
+        Backend::Singlepass
+    }
+
     fn check_precondition(&mut self, _module_info: &ModuleInfo) -> Result<(), CodegenError> {
         Ok(())
     }
@@ -1335,7 +1342,7 @@ impl X64FunctionCode {
     }
 }
 
-impl FunctionCodeGenerator for X64FunctionCode {
+impl FunctionCodeGenerator<CodegenError> for X64FunctionCode {
     fn feed_return(&mut self, ty: WpType) -> Result<(), CodegenError> {
         self.returns.push(ty);
         Ok(())
@@ -1377,7 +1384,13 @@ impl FunctionCodeGenerator for X64FunctionCode {
         Ok(())
     }
 
-    fn feed_opcode(&mut self, op: &Operator, module_info: &ModuleInfo) -> Result<(), CodegenError> {
+    fn feed_event(&mut self, ev: Event, module_info: &ModuleInfo) -> Result<(), CodegenError> {
+        let op = match ev {
+            Event::Wasm(x) => x,
+            Event::Internal(x) => {
+                return Ok(());
+            }
+        };
         //println!("{:?} {}", op, self.value_stack.len());
         let was_unreachable;
 

lib/singlepass-backend/src/lib.rs

@@ -18,74 +18,18 @@ extern crate byteorder;
 #[macro_use]
 extern crate smallvec;
 
-mod codegen;
 mod codegen_x64;
 mod emitter_x64;
 mod machine;
-mod parse;
 mod protect_unix;
 
-use crate::codegen::{CodegenError, ModuleCodeGenerator};
+pub use codegen_x64::X64ModuleCodeGenerator as ModuleCodeGenerator;
-use crate::parse::LoadError;
+pub use codegen_x64::X64FunctionCode as FunctionCodeGenerator;
-use wasmer_runtime_core::{
-    backend::{sys::Memory, Backend, CacheGen, Compiler, CompilerConfig, Token},
-    cache::{Artifact, Error as CacheError},
-    error::{CompileError, CompileResult},
-    module::{ModuleInfo, ModuleInner},
-};
 
-struct Placeholder;
+use wasmer_runtime_core::codegen::SimpleStreamingCompilerGen;
-impl CacheGen for Placeholder {
+pub type SinglePassCompiler = SimpleStreamingCompilerGen<
-    fn generate_cache(&self) -> Result<(Box<[u8]>, Memory), CacheError> {
+    codegen_x64::X64ModuleCodeGenerator,
-        Err(CacheError::Unknown(
+    codegen_x64::X64FunctionCode,
-            "the singlepass backend doesn't support caching yet".to_string(),
+    codegen_x64::X64ExecutionContext,
-        ))
+    codegen_x64::CodegenError,
-    }
+>;
-}
-
-pub struct SinglePassCompiler {}
-impl SinglePassCompiler {
-    pub fn new() -> Self {
-        Self {}
-    }
-}
-
-impl Compiler for SinglePassCompiler {
-    fn compile(
-        &self,
-        wasm: &[u8],
-        compiler_config: CompilerConfig,
-        _: Token,
-    ) -> CompileResult<ModuleInner> {
-        let mut mcg = codegen_x64::X64ModuleCodeGenerator::new();
-        let info = parse::read_module(wasm, Backend::Singlepass, &mut mcg, &compiler_config)?;
-        let exec_context = mcg.finalize(&info)?;
-        Ok(ModuleInner {
-            cache_gen: Box::new(Placeholder),
-            runnable_module: Box::new(exec_context),
-            info: info,
-        })
-    }
-
-    unsafe fn from_cache(&self, _artifact: Artifact, _: Token) -> Result<ModuleInner, CacheError> {
-        Err(CacheError::Unknown(
-            "the singlepass backend doesn't support caching yet".to_string(),
-        ))
-    }
-}
-
-impl From<CodegenError> for CompileError {
-    fn from(other: CodegenError) -> CompileError {
-        CompileError::InternalError {
-            msg: other.message.into(),
-        }
-    }
-}
-
-impl From<LoadError> for CompileError {
-    fn from(other: LoadError) -> CompileError {
-        CompileError::InternalError {
-            msg: format!("{:?}", other),
-        }
-    }
-}