From fdb2a675794f632711e216d9314b8f105f1562c9 Mon Sep 17 00:00:00 2001 From: Syrus Date: Fri, 22 Feb 2019 11:42:30 -0800 Subject: [PATCH 1/5] Added cache into standalone app --- install.sh | 2 +- lib/clif-backend/src/module.rs | 2 - lib/runtime-core/src/cache.rs | 4 +- lib/runtime-core/src/module.rs | 10 ++++- lib/runtime-core/src/vm.rs | 2 - lib/runtime/src/cache.rs | 18 ++++----- src/bin/wasmer.rs | 69 +++++++++++++++++++++++++++++++++- 7 files changed, 85 insertions(+), 22 deletions(-) diff --git a/install.sh b/install.sh index 196fab7d5..04b5f364a 100755 --- a/install.sh +++ b/install.sh @@ -130,7 +130,7 @@ wasmer_link() { printf "$cyan> Adding to bash profile...$reset\n" WASMER_PROFILE="$(wasmer_detect_profile)" LOAD_STR="\n# Wasmer\nexport WASMER_DIR=\"\$HOME/.wasmer\"\n[ -s \"\$WASMER_DIR/wasmer.sh\" ] && source \"\$WASMER_DIR/wasmer.sh\" # This loads wasmer\n" - SOURCE_STR="# Wasmer config\nexport WASMER_DIR=\"\$HOME/.wasmer\"\nexport PATH=\"\$HOME/.wasmer/bin:\$PATH\"\n" + SOURCE_STR="# Wasmer config\nexport WASMER_DIR=\"\$HOME/.wasmer\"\nexport WASMER_CACHE_DIR=\"\$WASMER_DIR/cache\"\nexport PATH=\"\$HOME/.wasmer/bin:\$PATH\"\n" # We create the wasmer.sh file echo "$SOURCE_STR" > "$HOME/.wasmer/wasmer.sh" diff --git a/lib/clif-backend/src/module.rs b/lib/clif-backend/src/module.rs index 6f95ea828..38a657625 100644 --- a/lib/clif-backend/src/module.rs +++ b/lib/clif-backend/src/module.rs @@ -50,8 +50,6 @@ impl Module { namespace_table: StringTable::new(), name_table: StringTable::new(), - - wasm_hash: WasmHash::generate(wasm), }, } } diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index 3d894bb09..e47a944d3 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -67,7 +67,6 @@ struct ArtifactHeader { magic: [u8; 8], // [W, A, S, M, E, R, \0, \0] version: u64, data_len: u64, - wasm_hash: [u8; 32], // Sha256 of the wasm in binary format. } impl ArtifactHeader { @@ -170,7 +169,6 @@ impl Artifact { magic: WASMER_CACHE_MAGIC, version: CURRENT_CACHE_VERSION, data_len: 0, - wasm_hash: self.inner.info.wasm_hash.into_array(), }; let mut buffer = cache_header.as_slice().to_vec(); @@ -195,5 +193,5 @@ pub trait Cache { type StoreError; fn load(&self, key: WasmHash) -> Result; - fn store(&mut self, module: Module) -> Result; + fn store(&mut self, key: WasmHash, module: Module) -> Result<(), Self::StoreError>; } diff --git a/lib/runtime-core/src/module.rs b/lib/runtime-core/src/module.rs index 130082491..cac375b66 100644 --- a/lib/runtime-core/src/module.rs +++ b/lib/runtime-core/src/module.rs @@ -56,8 +56,6 @@ pub struct ModuleInfo { pub namespace_table: StringTable, pub name_table: StringTable, - - pub wasm_hash: WasmHash, } /// A compiled WebAssembly module. @@ -116,6 +114,14 @@ impl Module { } } +impl Clone for Module { + fn clone(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), + } + } +} + impl ModuleInner {} #[doc(hidden)] diff --git a/lib/runtime-core/src/vm.rs b/lib/runtime-core/src/vm.rs index ab386401b..71400af70 100644 --- a/lib/runtime-core/src/vm.rs +++ b/lib/runtime-core/src/vm.rs @@ -565,8 +565,6 @@ mod vm_ctx_tests { namespace_table: StringTable::new(), name_table: StringTable::new(), - - wasm_hash: WasmHash::generate(&[]), }, } } diff --git a/lib/runtime/src/cache.rs b/lib/runtime/src/cache.rs index 09a93356b..2489598c0 100644 --- a/lib/runtime/src/cache.rs +++ b/lib/runtime/src/cache.rs @@ -19,19 +19,18 @@ pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash}; /// # Usage: /// /// ```rust -/// use wasmer_runtime::cache::{Cache, FileSystemCache}; +/// use wasmer_runtime::cache::{Cache, FileSystemCache, WasmHash}; /// /// # use wasmer_runtime::{Module, error::CacheError}; -/// fn store_and_load_module(module: Module) -> Result { +/// fn store_module(module: Module) -> Result { /// // Create a new file system cache. /// // This is unsafe because we can't ensure that the artifact wasn't /// // corrupted or tampered with. /// let mut fs_cache = unsafe { FileSystemCache::new("some/directory/goes/here")? }; -/// // Store a module into the cache. -/// // The returned `key` is equivalent to `module.info().wasm_hash`. -/// let key = fs_cache.store(module)?; -/// // Load the module back from the cache with the `key`. -/// fs_cache.load(key) +/// // Compute a key for a given WebAssembly module +/// let key = WasmHash::generate(module) +/// // Store a module into the cache given a key +/// fs_cache.store(key, module)?; /// } /// ``` pub struct FileSystemCache { @@ -92,8 +91,7 @@ impl Cache for FileSystemCache { unsafe { wasmer_runtime_core::load_cache_with(serialized_cache, super::default_compiler()) } } - fn store(&mut self, module: Module) -> Result { - let key = module.info().wasm_hash; + fn store(&mut self, key: WasmHash, module: Module) -> Result<(), CacheError> { let filename = key.encode(); let mut new_path_buf = self.path.clone(); new_path_buf.push(filename); @@ -104,6 +102,6 @@ impl Cache for FileSystemCache { let mut file = File::create(new_path_buf)?; file.write_all(&buffer)?; - Ok(key) + Ok(()) } } diff --git a/src/bin/wasmer.rs b/src/bin/wasmer.rs index 2ec707f83..59d753e1a 100644 --- a/src/bin/wasmer.rs +++ b/src/bin/wasmer.rs @@ -1,5 +1,7 @@ extern crate structopt; +use std::env; +use std::fs; use std::fs::File; use std::io; use std::io::Read; @@ -11,6 +13,8 @@ use structopt::StructOpt; use wasmer::webassembly::InstanceABI; use wasmer::*; use wasmer_emscripten; +use wasmer_runtime::cache::{Cache as BaseCache, FileSystemCache, WasmHash}; +use wasmer_runtime::error::CacheError; #[derive(Debug, StructOpt)] #[structopt(name = "wasmer", about = "Wasm execution runtime.")] @@ -20,6 +24,10 @@ enum CLIOptions { #[structopt(name = "run")] Run(Run), + /// Wasmer cache + #[structopt(name = "cache")] + Cache(Cache), + /// Update wasmer to the latest version #[structopt(name = "self-update")] SelfUpdate, @@ -39,6 +47,15 @@ struct Run { args: Vec, } +#[derive(Debug, StructOpt)] +enum Cache { + #[structopt(name = "clean")] + Clean, + + #[structopt(name = "dir")] + Dir, +} + /// Read the contents of a file fn read_file_contents(path: &PathBuf) -> Result, io::Error> { let mut buffer: Vec = Vec::new(); @@ -49,6 +66,18 @@ fn read_file_contents(path: &PathBuf) -> Result, io::Error> { Ok(buffer) } +fn get_cache_dir() -> PathBuf { + match env::var("WASMER_CACHE_DIR") { + Ok(dir) => PathBuf::from(dir), + Err(_) => { + // We use a temporal directory for saving cache files + let mut temp_dir = env::temp_dir(); + temp_dir.push("wasmer"); + temp_dir + } + } +} + /// Execute a wasm/wat file fn execute_wasm(options: &Run) -> Result<(), String> { let wasm_path = &options.path; @@ -66,8 +95,34 @@ fn execute_wasm(options: &Run) -> Result<(), String> { .map_err(|e| format!("Can't convert from wast to wasm: {:?}", e))?; } - let module = webassembly::compile(&wasm_binary[..]) - .map_err(|e| format!("Can't compile module: {:?}", e))?; + let hash = WasmHash::generate(&wasm_binary); + + let wasmer_cache_dir = get_cache_dir(); + + // We create a new cache instance. + // It could be possible to use any other kinds of caching, as long as they + // implement the Cache trait (with save and load functions) + let mut cache = unsafe { + FileSystemCache::new(wasmer_cache_dir).map_err(|e| format!("Cache error: {:?}", e))? + }; + + // cache.load will return the Module if it's able to deserialize it properly, and an error if: + // * The file is not found + // * The file exists, but it's corrupted or can't be converted to a module + let module = match cache.load(hash) { + Ok(module) => { + // We are able to load the module from cache + module + } + Err(_) => { + let module = webassembly::compile(&wasm_binary[..]) + .map_err(|e| format!("Can't compile module: {:?}", e))?; + + // We save the module into a cache file + cache.store(hash, module.clone()).unwrap(); + module + } + }; let (_abi, import_object, _em_globals) = if wasmer_emscripten::is_emscripten_module(&module) { let mut emscripten_globals = wasmer_emscripten::EmscriptenGlobals::new(&module); @@ -119,5 +174,15 @@ fn main() { CLIOptions::SelfUpdate => { println!("Self update is not supported on Windows. Use install instructions on the Wasmer homepage: https://wasmer.io"); } + CLIOptions::Cache(cache) => match cache { + Cache::Clean => { + let cache_dir = get_cache_dir(); + fs::remove_dir_all(cache_dir.clone()).expect("Can't remove cache dir"); + fs::create_dir(cache_dir.clone()).expect("Can't create cache dir"); + } + Cache::Dir => { + println!("{}", get_cache_dir().to_string_lossy()); + } + }, } } From 84042cecb334ad62d8521595901c3548e675cd9e Mon Sep 17 00:00:00 2001 From: Syrus Date: Fri, 22 Feb 2019 12:00:30 -0800 Subject: [PATCH 2/5] Fixed wrong syntax --- lib/runtime/src/cache.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/runtime/src/cache.rs b/lib/runtime/src/cache.rs index 2489598c0..8f42e9e49 100644 --- a/lib/runtime/src/cache.rs +++ b/lib/runtime/src/cache.rs @@ -27,8 +27,8 @@ pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash}; /// // This is unsafe because we can't ensure that the artifact wasn't /// // corrupted or tampered with. /// let mut fs_cache = unsafe { FileSystemCache::new("some/directory/goes/here")? }; -/// // Compute a key for a given WebAssembly module -/// let key = WasmHash::generate(module) +/// // Compute a key for a given WebAssembly binary +/// let key = WasmHash::generate(&[]); /// // Store a module into the cache given a key /// fs_cache.store(key, module)?; /// } From 13e1cb11691a047850f8d9e889d098883b420801 Mon Sep 17 00:00:00 2001 From: Syrus Date: Fri, 22 Feb 2019 12:01:03 -0800 Subject: [PATCH 3/5] =?UTF-8?q?Added=20a=20=E2=80=94disable-cache=20cli=20?= =?UTF-8?q?flag=20for=20wasmer=20run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/bin/wasmer.rs | 60 +++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 23 deletions(-) diff --git a/src/bin/wasmer.rs b/src/bin/wasmer.rs index 59d753e1a..6b8c9b54b 100644 --- a/src/bin/wasmer.rs +++ b/src/bin/wasmer.rs @@ -38,6 +38,10 @@ struct Run { #[structopt(short = "d", long = "debug")] debug: bool, + // Disable the cache + #[structopt(long = "disable-cache")] + disable_cache: bool, + /// Input file #[structopt(parse(from_os_str))] path: PathBuf, @@ -95,33 +99,43 @@ fn execute_wasm(options: &Run) -> Result<(), String> { .map_err(|e| format!("Can't convert from wast to wasm: {:?}", e))?; } - let hash = WasmHash::generate(&wasm_binary); + let module = if !options.disable_cache { + // If we have cache enabled - let wasmer_cache_dir = get_cache_dir(); + // We generate a hash for the given binary, so we can use it as key + // for the Filesystem cache + let hash = WasmHash::generate(&wasm_binary); - // We create a new cache instance. - // It could be possible to use any other kinds of caching, as long as they - // implement the Cache trait (with save and load functions) - let mut cache = unsafe { - FileSystemCache::new(wasmer_cache_dir).map_err(|e| format!("Cache error: {:?}", e))? - }; + let wasmer_cache_dir = get_cache_dir(); - // cache.load will return the Module if it's able to deserialize it properly, and an error if: - // * The file is not found - // * The file exists, but it's corrupted or can't be converted to a module - let module = match cache.load(hash) { - Ok(module) => { - // We are able to load the module from cache - module - } - Err(_) => { - let module = webassembly::compile(&wasm_binary[..]) - .map_err(|e| format!("Can't compile module: {:?}", e))?; + // We create a new cache instance. + // It could be possible to use any other kinds of caching, as long as they + // implement the Cache trait (with save and load functions) + let mut cache = unsafe { + FileSystemCache::new(wasmer_cache_dir).map_err(|e| format!("Cache error: {:?}", e))? + }; - // We save the module into a cache file - cache.store(hash, module.clone()).unwrap(); - module - } + // cache.load will return the Module if it's able to deserialize it properly, and an error if: + // * The file is not found + // * The file exists, but it's corrupted or can't be converted to a module + let module = match cache.load(hash) { + Ok(module) => { + // We are able to load the module from cache + module + } + Err(_) => { + let module = webassembly::compile(&wasm_binary[..]) + .map_err(|e| format!("Can't compile module: {:?}", e))?; + + // We save the module into a cache file + cache.store(hash, module.clone()).unwrap(); + module + } + }; + module + } else { + webassembly::compile(&wasm_binary[..]) + .map_err(|e| format!("Can't compile module: {:?}", e))? }; let (_abi, import_object, _em_globals) = if wasmer_emscripten::is_emscripten_module(&module) { From b3dd47bce8d91fe1fffddbf1c0b0e9636226f045 Mon Sep 17 00:00:00 2001 From: Syrus Date: Fri, 22 Feb 2019 12:06:22 -0800 Subject: [PATCH 4/5] Fixed runtime example --- lib/runtime/src/cache.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/runtime/src/cache.rs b/lib/runtime/src/cache.rs index 8f42e9e49..c174af4ac 100644 --- a/lib/runtime/src/cache.rs +++ b/lib/runtime/src/cache.rs @@ -30,7 +30,8 @@ pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash}; /// // Compute a key for a given WebAssembly binary /// let key = WasmHash::generate(&[]); /// // Store a module into the cache given a key -/// fs_cache.store(key, module)?; +/// fs_cache.store(key, module.clone())?; +/// module /// } /// ``` pub struct FileSystemCache { From 16d62ace52dadf719d56b6c4582c533b56d2e284 Mon Sep 17 00:00:00 2001 From: Syrus Date: Fri, 22 Feb 2019 12:17:49 -0800 Subject: [PATCH 5/5] Fixed cache example (again) --- lib/runtime/src/cache.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/runtime/src/cache.rs b/lib/runtime/src/cache.rs index c174af4ac..6ebbf1017 100644 --- a/lib/runtime/src/cache.rs +++ b/lib/runtime/src/cache.rs @@ -31,7 +31,7 @@ pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash}; /// let key = WasmHash::generate(&[]); /// // Store a module into the cache given a key /// fs_cache.store(key, module.clone())?; -/// module +/// Ok(module) /// } /// ``` pub struct FileSystemCache {