diff --git a/Cargo.lock b/Cargo.lock index bcd7f4790..55ccf93c9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2363,6 +2363,7 @@ dependencies = [ "serde-bench 0.0.7 (registry+https://github.com/rust-lang/crates.io-index)", "serde_bytes 0.10.5 (registry+https://github.com/rust-lang/crates.io-index)", "serde_derive 1.0.90 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 0.6.9 (registry+https://github.com/rust-lang/crates.io-index)", "wasmparser 0.29.2 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.3.7 (registry+https://github.com/rust-lang/crates.io-index)", ] diff --git a/lib/runtime-core/Cargo.toml b/lib/runtime-core/Cargo.toml index 41d472626..321ee501f 100644 --- a/lib/runtime-core/Cargo.toml +++ b/lib/runtime-core/Cargo.toml @@ -17,6 +17,7 @@ indexmap = "1.0.2" errno = "0.2.4" libc = "0.2.49" hex = "0.3.2" +smallvec = "0.6.9" # Dependencies for caching. [dependencies.serde] diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index 1044a7ee5..c6ecffcb0 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -1,10 +1,8 @@ use crate::{ - backing::ImportBacking, error::CompileResult, - error::RuntimeResult, module::ModuleInner, typed_func::Wasm, - types::{FuncIndex, LocalFuncIndex, SigIndex, Value}, + types::{LocalFuncIndex, SigIndex}, vm, }; @@ -67,59 +65,25 @@ pub trait Compiler { unsafe fn from_cache(&self, cache: Artifact, _: Token) -> Result; } -/// The functionality exposed by this trait is expected to be used -/// for calling functions exported by a webassembly module from -/// host code only. -pub trait ProtectedCaller: Send + Sync { - /// This calls the exported function designated by `local_func_index`. - /// Important to note, this supports calling imported functions that are - /// then exported. - /// - /// It's invalid to attempt to call a local function that isn't exported and - /// the implementation is expected to check for that. The implementation - /// is also expected to check for correct parameter types and correct - /// parameter number. - /// - /// The `returns` parameter is filled with dummy values when passed in and upon function - /// return, will be filled with the return values of the wasm function, as long as the - /// call completed successfully. - /// - /// The existance of the Token parameter ensures that this can only be called from - /// within the runtime crate. - /// - /// TODO(lachlan): Now that `get_wasm_trampoline` exists, `ProtectedCaller::call` - /// can be removed. That should speed up calls a little bit, since sanity checks - /// would only occur once. - fn call( - &self, - module: &ModuleInner, - func_index: FuncIndex, - params: &[Value], - import_backing: &ImportBacking, - vmctx: *mut vm::Ctx, - _: Token, - ) -> RuntimeResult>; - - /// A wasm trampoline contains the necesarry data to dynamically call an exported wasm function. - /// Given a particular signature index, we are returned a trampoline that is matched with that - /// signature and an invoke function that can call the trampoline. - fn get_wasm_trampoline(&self, module: &ModuleInner, sig_index: SigIndex) -> Option; - - fn get_early_trapper(&self) -> Box; -} - pub trait UserTrapper { unsafe fn do_early_trap(&self, data: Box) -> !; } -pub trait FuncResolver: Send + Sync { +pub trait RunnableModule: Send + Sync { /// This returns a pointer to the function designated by the `local_func_index` /// parameter. - fn get( + fn get_func( &self, - module: &ModuleInner, + info: &ModuleInfo, local_func_index: LocalFuncIndex, ) -> Option>; + + /// A wasm trampoline contains the necesarry data to dynamically call an exported wasm function. + /// Given a particular signature index, we are returned a trampoline that is matched with that + /// signature and an invoke function that can call the trampoline. + fn get_trampoline(&self, info: &ModuleInfo, sig_index: SigIndex) -> Option; + + fn get_early_trapper(&self) -> Box; } pub trait CacheGen: Send + Sync { diff --git a/lib/runtime-core/src/backing.rs b/lib/runtime-core/src/backing.rs index 48f2d2256..b79180ad6 100644 --- a/lib/runtime-core/src/backing.rs +++ b/lib/runtime-core/src/backing.rs @@ -72,8 +72,8 @@ impl LocalBacking { (0..module.info.func_assoc.len() - module.info.imported_functions.len()) .map(|index| { module - .func_resolver - .get(module, LocalFuncIndex::new(index)) + .runnable_module + .get_func(&module.info, LocalFuncIndex::new(index)) .unwrap() .as_ptr() as *const _ }) @@ -216,8 +216,8 @@ impl LocalBacking { let (func, ctx) = match func_index.local_or_import(&module.info) { LocalOrImport::Local(local_func_index) => ( module - .func_resolver - .get(module, local_func_index) + .runnable_module + .get_func(&module.info, local_func_index) .unwrap() .as_ptr() as *const vm::Func, @@ -255,8 +255,8 @@ impl LocalBacking { let (func, ctx) = match func_index.local_or_import(&module.info) { LocalOrImport::Local(local_func_index) => ( module - .func_resolver - .get(module, local_func_index) + .runnable_module + .get_func(&module.info, local_func_index) .unwrap() .as_ptr() as *const vm::Func, diff --git a/lib/runtime-core/src/instance.rs b/lib/runtime-core/src/instance.rs index 1249998cc..2cdc738da 100644 --- a/lib/runtime-core/src/instance.rs +++ b/lib/runtime-core/src/instance.rs @@ -1,18 +1,19 @@ use crate::{ - backend::Token, + backend::{RunnableModule, Token}, backing::{ImportBacking, LocalBacking}, - error::{CallError, CallResult, ResolveError, ResolveResult, Result}, + error::{CallError, CallResult, ResolveError, ResolveResult, Result, RuntimeError}, export::{Context, Export, ExportIter, FuncPointer}, global::Global, import::{ImportObject, LikeNamespace}, memory::Memory, - module::{ExportIndex, Module, ModuleInner}, + module::{ExportIndex, Module, ModuleInfo, ModuleInner}, sig_registry::SigRegistry, table::Table, - typed_func::{Func, Wasm, WasmTypeList}, - types::{FuncIndex, FuncSig, GlobalIndex, LocalOrImport, MemoryIndex, TableIndex, Value}, + typed_func::{Func, Wasm, WasmTrapInfo, WasmTypeList}, + types::{FuncIndex, FuncSig, GlobalIndex, LocalOrImport, MemoryIndex, TableIndex, Type, Value}, vm, }; +use smallvec::{smallvec, SmallVec}; use std::{mem, ptr::NonNull, sync::Arc}; pub(crate) struct InstanceInner { @@ -82,7 +83,45 @@ impl Instance { }; if let Some(start_index) = instance.module.info.start_func { - instance.call_with_index(start_index, &[])?; + // We know that the start function takes no arguments and returns no values. + // Therefore, we can call it without doing any signature checking, etc. + + let func_ptr = match start_index.local_or_import(&instance.module.info) { + LocalOrImport::Local(local_func_index) => instance + .module + .runnable_module + .get_func(&instance.module.info, local_func_index) + .unwrap(), + LocalOrImport::Import(import_func_index) => NonNull::new( + instance.inner.import_backing.vm_functions[import_func_index].func as *mut _, + ) + .unwrap(), + }; + + let ctx_ptr = match start_index.local_or_import(&instance.module.info) { + LocalOrImport::Local(_) => instance.inner.vmctx, + LocalOrImport::Import(imported_func_index) => { + instance.inner.import_backing.vm_functions[imported_func_index].vmctx + } + }; + + let sig_index = *instance + .module + .info + .func_assoc + .get(start_index) + .expect("broken invariant, incorrect func index"); + + let wasm_trampoline = instance + .module + .runnable_module + .get_trampoline(&instance.module.info, sig_index) + .expect("wasm trampoline"); + + let start_func: Func<(), (), Wasm> = + unsafe { Func::from_raw_parts(wasm_trampoline, func_ptr, ctx_ptr) }; + + start_func.call()?; } Ok(instance) @@ -147,15 +186,15 @@ impl Instance { let func_wasm_inner = self .module - .protected_caller - .get_wasm_trampoline(&self.module, sig_index) + .runnable_module + .get_trampoline(&self.module.info, sig_index) .unwrap(); let func_ptr = match func_index.local_or_import(&self.module.info) { LocalOrImport::Local(local_func_index) => self .module - .func_resolver - .get(&self.module, local_func_index) + .runnable_module + .get_func(&self.module.info, local_func_index) .unwrap(), LocalOrImport::Import(import_func_index) => NonNull::new( self.inner.import_backing.vm_functions[import_func_index].func as *mut _, @@ -245,7 +284,7 @@ impl Instance { /// # Ok(()) /// # } /// ``` - pub fn call(&self, name: &str, args: &[Value]) -> CallResult> { + pub fn call(&self, name: &str, params: &[Value]) -> CallResult> { let export_index = self.module .info @@ -264,7 +303,19 @@ impl Instance { .into()); }; - self.call_with_index(func_index, args) + let mut results = Vec::new(); + + call_func_with_index( + &self.module.info, + &*self.module.runnable_module, + &self.inner.import_backing, + self.inner.vmctx, + func_index, + params, + &mut results, + )?; + + Ok(results) } /// Returns an immutable reference to the @@ -295,45 +346,6 @@ impl Instance { } } -impl Instance { - fn call_with_index(&self, func_index: FuncIndex, args: &[Value]) -> CallResult> { - let sig_index = *self - .module - .info - .func_assoc - .get(func_index) - .expect("broken invariant, incorrect func index"); - let signature = &self.module.info.signatures[sig_index]; - - if !signature.check_param_value_types(args) { - Err(ResolveError::Signature { - expected: signature.clone(), - found: args.iter().map(|val| val.ty()).collect(), - })? - } - - let vmctx = match func_index.local_or_import(&self.module.info) { - LocalOrImport::Local(_) => self.inner.vmctx, - LocalOrImport::Import(imported_func_index) => { - self.inner.import_backing.vm_functions[imported_func_index].vmctx - } - }; - - let token = Token::generate(); - - let returns = self.module.protected_caller.call( - &self.module, - func_index, - args, - &self.inner.import_backing, - vmctx, - token, - )?; - - Ok(returns) - } -} - impl InstanceInner { pub(crate) fn get_export_from_index( &self, @@ -382,8 +394,8 @@ impl InstanceInner { let (func_ptr, ctx) = match func_index.local_or_import(&module.info) { LocalOrImport::Local(local_func_index) => ( module - .func_resolver - .get(&module, local_func_index) + .runnable_module + .get_func(&module.info, local_func_index) .expect("broken invariant, func resolver not synced with module.exports") .cast() .as_ptr() as *const _, @@ -452,6 +464,128 @@ impl LikeNamespace for Instance { } } +#[must_use] +fn call_func_with_index( + info: &ModuleInfo, + runnable: &dyn RunnableModule, + import_backing: &ImportBacking, + local_ctx: *mut vm::Ctx, + func_index: FuncIndex, + args: &[Value], + rets: &mut Vec, +) -> CallResult<()> { + rets.clear(); + + let sig_index = *info + .func_assoc + .get(func_index) + .expect("broken invariant, incorrect func index"); + + let signature = &info.signatures[sig_index]; + let num_results = signature.returns().len(); + rets.reserve(num_results); + + if !signature.check_param_value_types(args) { + Err(ResolveError::Signature { + expected: signature.clone(), + found: args.iter().map(|val| val.ty()).collect(), + })? + } + + let func_ptr = match func_index.local_or_import(info) { + LocalOrImport::Local(local_func_index) => { + runnable.get_func(info, local_func_index).unwrap() + } + LocalOrImport::Import(import_func_index) => { + NonNull::new(import_backing.vm_functions[import_func_index].func as *mut _).unwrap() + } + }; + + let ctx_ptr = match func_index.local_or_import(info) { + LocalOrImport::Local(_) => local_ctx, + LocalOrImport::Import(imported_func_index) => { + import_backing.vm_functions[imported_func_index].vmctx + } + }; + + let raw_args: SmallVec<[u64; 8]> = args + .iter() + .map(|v| match v { + Value::I32(i) => *i as u64, + Value::I64(i) => *i as u64, + Value::F32(f) => f.to_bits() as u64, + Value::F64(f) => f.to_bits(), + }) + .collect(); + + let Wasm { + trampoline, + invoke, + invoke_env, + } = runnable + .get_trampoline(info, sig_index) + .expect("wasm trampoline"); + + let run_wasm = |result_space: *mut u64| unsafe { + let mut trap_info = WasmTrapInfo::Unknown; + + let success = invoke( + trampoline, + ctx_ptr, + func_ptr, + raw_args.as_ptr(), + result_space, + &mut trap_info, + invoke_env, + ); + + if success { + Ok(()) + } else { + Err(RuntimeError::Trap { + msg: trap_info.to_string().into(), + }) + } + }; + + let raw_to_value = |raw, ty| match ty { + Type::I32 => Value::I32(raw as i32), + Type::I64 => Value::I64(raw as i64), + Type::F32 => Value::F32(f32::from_bits(raw as u32)), + Type::F64 => Value::F64(f64::from_bits(raw)), + }; + + match signature.returns() { + &[] => { + run_wasm(0 as *mut u64)?; + Ok(()) + } + &[ty] => { + let mut result = 0u64; + + run_wasm(&mut result)?; + + rets.push(raw_to_value(result, ty)); + + Ok(()) + } + result_tys @ _ => { + let mut results: SmallVec<[u64; 8]> = smallvec![0; num_results]; + + run_wasm(results.as_mut_ptr())?; + + rets.extend( + results + .iter() + .zip(result_tys.iter()) + .map(|(&raw, &ty)| raw_to_value(raw, ty)), + ); + + Ok(()) + } + } +} + /// A representation of an exported WebAssembly function. pub struct DynFunc<'a> { pub(crate) signature: Arc, @@ -484,32 +618,19 @@ impl<'a> DynFunc<'a> { /// # } /// ``` pub fn call(&self, params: &[Value]) -> CallResult> { - if !self.signature.check_param_value_types(params) { - Err(ResolveError::Signature { - expected: (*self.signature).clone(), - found: params.iter().map(|val| val.ty()).collect(), - })? - } + let mut results = Vec::new(); - let vmctx = match self.func_index.local_or_import(&self.module.info) { - LocalOrImport::Local(_) => self.instance_inner.vmctx, - LocalOrImport::Import(imported_func_index) => { - self.instance_inner.import_backing.vm_functions[imported_func_index].vmctx - } - }; - - let token = Token::generate(); - - let returns = self.module.protected_caller.call( - &self.module, + call_func_with_index( + &self.module.info, + &*self.module.runnable_module, + &self.instance_inner.import_backing, + self.instance_inner.vmctx, self.func_index, params, - &self.instance_inner.import_backing, - vmctx, - token, + &mut results, )?; - Ok(returns) + Ok(results) } pub fn signature(&self) -> &FuncSig { @@ -520,8 +641,8 @@ impl<'a> DynFunc<'a> { match self.func_index.local_or_import(&self.module.info) { LocalOrImport::Local(local_func_index) => self .module - .func_resolver - .get(self.module, local_func_index) + .runnable_module + .get_func(&self.module.info, local_func_index) .unwrap() .as_ptr(), LocalOrImport::Import(import_func_index) => { diff --git a/lib/runtime-core/src/module.rs b/lib/runtime-core/src/module.rs index 65817399d..9e9e93052 100644 --- a/lib/runtime-core/src/module.rs +++ b/lib/runtime-core/src/module.rs @@ -1,5 +1,5 @@ use crate::{ - backend::{Backend, FuncResolver, ProtectedCaller}, + backend::{Backend, RunnableModule}, cache::{Artifact, Error as CacheError}, error, import::ImportObject, @@ -22,9 +22,7 @@ use std::sync::Arc; /// This is used to instantiate a new WebAssembly module. #[doc(hidden)] pub struct ModuleInner { - pub func_resolver: Box, - pub protected_caller: Box, - + pub runnable_module: Box, pub cache_gen: Box, pub info: ModuleInfo, @@ -96,7 +94,7 @@ impl Module { pub(crate) fn new(inner: Arc) -> Self { unsafe { EARLY_TRAPPER - .with(|ucell| *ucell.get() = Some(inner.protected_caller.get_early_trapper())); + .with(|ucell| *ucell.get() = Some(inner.runnable_module.get_early_trapper())); } Module { inner } } diff --git a/lib/runtime-core/src/typed_func.rs b/lib/runtime-core/src/typed_func.rs index d18055ba1..5e38db628 100644 --- a/lib/runtime-core/src/typed_func.rs +++ b/lib/runtime-core/src/typed_func.rs @@ -71,9 +71,9 @@ pub type Invoke = unsafe extern "C" fn( /// as well as the environment that the invoke function may or may not require. #[derive(Copy, Clone)] pub struct Wasm { - trampoline: Trampoline, - invoke: Invoke, - invoke_env: Option>, + pub(crate) trampoline: Trampoline, + pub(crate) invoke: Invoke, + pub(crate) invoke_env: Option>, } impl Wasm { diff --git a/lib/runtime-core/src/vm.rs b/lib/runtime-core/src/vm.rs index 3b2ca9938..1d955d40d 100644 --- a/lib/runtime-core/src/vm.rs +++ b/lib/runtime-core/src/vm.rs @@ -544,42 +544,23 @@ mod vm_ctx_tests { fn generate_module() -> ModuleInner { use super::Func; - use crate::backend::{ - sys::Memory, Backend, CacheGen, FuncResolver, ProtectedCaller, Token, UserTrapper, - }; + use crate::backend::{sys::Memory, Backend, CacheGen, RunnableModule, UserTrapper}; use crate::cache::Error as CacheError; - use crate::error::RuntimeResult; use crate::typed_func::Wasm; - use crate::types::{FuncIndex, LocalFuncIndex, SigIndex, Value}; + use crate::types::{LocalFuncIndex, SigIndex}; use hashbrown::HashMap; use std::ptr::NonNull; struct Placeholder; - impl FuncResolver for Placeholder { - fn get( + impl RunnableModule for Placeholder { + fn get_func( &self, - _module: &ModuleInner, + _module: &ModuleInfo, _local_func_index: LocalFuncIndex, ) -> Option> { None } - } - impl ProtectedCaller for Placeholder { - fn call( - &self, - _module: &ModuleInner, - _func_index: FuncIndex, - _params: &[Value], - _import_backing: &ImportBacking, - _vmctx: *mut Ctx, - _: Token, - ) -> RuntimeResult> { - Ok(vec![]) - } - fn get_wasm_trampoline( - &self, - _module: &ModuleInner, - _sig_index: SigIndex, - ) -> Option { + + fn get_trampoline(&self, _module: &ModuleInfo, _sig_index: SigIndex) -> Option { unimplemented!() } fn get_early_trapper(&self) -> Box { @@ -596,8 +577,7 @@ mod vm_ctx_tests { } ModuleInner { - func_resolver: Box::new(Placeholder), - protected_caller: Box::new(Placeholder), + runnable_module: Box::new(Placeholder), cache_gen: Box::new(Placeholder), info: ModuleInfo { memories: Map::new(),