Merge remote-tracking branch 'origin/master' into command/pyodide

lib/emscripten/src/lib.rs

@@ -3,8 +3,10 @@
 #[macro_use]
 extern crate wasmer_runtime_core;
 
+use hashbrown::HashMap;
 use lazy_static::lazy_static;
 use std::cell::UnsafeCell;
+use std::path::PathBuf;
 use std::{f64, ffi::c_void};
 use wasmer_runtime_core::{
     error::CallResult,
@@ -144,10 +146,14 @@ pub struct EmscriptenData<'a> {
     pub stack_save: Option<Func<'a, (), i32>>,
     pub stack_restore: Option<Func<'a, (i32)>>,
     pub set_threw: Option<Func<'a, (i32, i32)>>,
+    pub mapped_dirs: HashMap<String, PathBuf>,
 }
 
 impl<'a> EmscriptenData<'a> {
-    pub fn new(instance: &'a mut Instance) -> EmscriptenData<'a> {
+    pub fn new(
+        instance: &'a mut Instance,
+        mapped_dirs: HashMap<String, PathBuf>,
+    ) -> EmscriptenData<'a> {
         let malloc = instance.func("_malloc").unwrap();
         let free = instance.func("_free").unwrap();
         let memalign = instance.func("_memalign").ok();
@@ -275,6 +281,7 @@ impl<'a> EmscriptenData<'a> {
             stack_save,
             stack_restore,
             set_threw,
+            mapped_dirs,
         }
     }
 }
@@ -285,8 +292,9 @@ pub fn run_emscripten_instance(
     path: &str,
     args: Vec<&str>,
     entrypoint: Option<String>,
+    mapped_dirs: Vec<(String, PathBuf)>,
 ) -> CallResult<()> {
-    let mut data = EmscriptenData::new(instance);
+    let mut data = EmscriptenData::new(instance, mapped_dirs.into_iter().collect());
     let data_ptr = &mut data as *mut _ as *mut c_void;
     instance.context_mut().data = data_ptr;
 

lib/emscripten/src/syscalls/mod.rs

@@ -10,7 +10,8 @@ pub use self::unix::*;
 #[cfg(windows)]
 pub use self::windows::*;
 
-use super::utils::copy_stat_into_wasm;
+use crate::utils::{copy_stat_into_wasm, get_cstr_path, get_current_directory};
+
 use super::varargs::VarArgs;
 use byteorder::{ByteOrder, LittleEndian};
 /// NOTE: TODO: These syscalls only support wasm_32 for now because they assume offsets are u32
@@ -95,12 +96,19 @@ pub fn ___syscall6(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_int
 pub fn ___syscall12(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_int {
     debug!("emscripten::___syscall12 (chdir) {}", _which);
     let path_ptr = varargs.get_str(ctx);
-    unsafe {
-        let _path = std::ffi::CStr::from_ptr(path_ptr);
-        let ret = chdir(path_ptr);
-        debug!("=> path: {:?}, ret: {}", _path, ret);
+    let real_path_owned = get_cstr_path(ctx, path_ptr);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        path_ptr
+    };
+    let ret = unsafe { chdir(real_path) };
+    debug!(
+        "=> path: {:?}, ret: {}",
+        unsafe { std::ffi::CStr::from_ptr(real_path) },
         ret
-    }
+    );
+    ret
 }
 
 pub fn ___syscall10(_ctx: &mut Ctx, _one: i32, _two: i32) -> i32 {
@@ -159,11 +167,23 @@ pub fn ___syscall38(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> i32 {
     debug!("emscripten::___syscall38 (rename)");
     let old_path = varargs.get_str(ctx);
     let new_path = varargs.get_str(ctx);
-    let result = unsafe { rename(old_path, new_path) };
+    let real_old_path_owned = get_cstr_path(ctx, old_path);
+    let real_old_path = if let Some(ref rp) = real_old_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        old_path
+    };
+    let real_new_path_owned = get_cstr_path(ctx, new_path);
+    let real_new_path = if let Some(ref rp) = real_new_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        new_path
+    };
+    let result = unsafe { rename(real_old_path, real_new_path) };
     debug!(
         "=> old_path: {}, new_path: {}, result: {}",
-        unsafe { std::ffi::CStr::from_ptr(old_path).to_str().unwrap() },
-        unsafe { std::ffi::CStr::from_ptr(new_path).to_str().unwrap() },
+        unsafe { std::ffi::CStr::from_ptr(real_old_path).to_str().unwrap() },
+        unsafe { std::ffi::CStr::from_ptr(real_new_path).to_str().unwrap() },
         result
     );
     result
@@ -173,7 +193,13 @@ pub fn ___syscall38(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> i32 {
 pub fn ___syscall40(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_int {
     debug!("emscripten::___syscall40 (rmdir)");
     let pathname_addr = varargs.get_str(ctx);
-    unsafe { rmdir(pathname_addr) }
+    let real_path_owned = get_cstr_path(ctx, pathname_addr);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        pathname_addr
+    };
+    unsafe { rmdir(real_path) }
 }
 
 // pipe
@@ -354,10 +380,9 @@ pub fn ___syscall163(_ctx: &mut Ctx, _one: i32, _two: i32) -> i32 {
 // getcwd
 pub fn ___syscall183(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> i32 {
     debug!("emscripten::___syscall183");
-    use std::env;
     let buf_offset: c_int = varargs.get(ctx);
     let _size: c_int = varargs.get(ctx);
-    let path = env::current_dir();
+    let path = get_current_directory(ctx);
     let path_string = path.unwrap().display().to_string();
     let len = path_string.len();
     unsafe {
@@ -530,12 +555,19 @@ pub fn ___syscall195(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_in
     let pathname_addr = varargs.get_str(ctx);
     let buf: u32 = varargs.get(ctx);
 
+    let real_path_owned = get_cstr_path(ctx, pathname_addr);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        pathname_addr
+    };
+
     unsafe {
         let mut _stat: stat = std::mem::zeroed();
-        let ret = stat(pathname_addr, &mut _stat);
+        let ret = stat(real_path, &mut _stat);
         debug!(
             "=> pathname: {}, buf: {} = {}, last os error: {}",
-            std::ffi::CStr::from_ptr(pathname_addr).to_str().unwrap(),
+            std::ffi::CStr::from_ptr(real_path).to_str().unwrap(),
             buf,
             ret,
             Error::last_os_error()
@@ -604,7 +636,7 @@ pub fn ___syscall221(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_in
     // |FNONBLOCK - 0x04
     debug!("=> fd: {}, cmd: {}", _fd, cmd);
     match cmd {
-        2 => 0,
+        1 | 2 => 0,
         13 | 14 => 0, // pretend file locking worked
         _ => -1,
     }

lib/emscripten/src/syscalls/unix.rs

@@ -43,6 +43,7 @@ use libc::{
     pid_t,
     pread,
     pwrite,
+    readdir,
     // readv,
     recvfrom,
     recvmsg,
@@ -107,8 +108,14 @@ pub fn ___syscall5(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_int
     let pathname_addr = varargs.get_str(ctx);
     let flags: i32 = varargs.get(ctx);
     let mode: u32 = varargs.get(ctx);
-    let _path_str = unsafe { std::ffi::CStr::from_ptr(pathname_addr).to_str().unwrap() };
-    let fd = unsafe { open(pathname_addr, flags, mode) };
+    let real_path_owned = utils::get_cstr_path(ctx, pathname_addr);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        pathname_addr
+    };
+    let _path_str = unsafe { std::ffi::CStr::from_ptr(real_path).to_str().unwrap() };
+    let fd = unsafe { open(real_path, flags, mode) };
     debug!(
         "=> path: {}, flags: {}, mode: {} = fd: {}, last os error: {}",
         _path_str,
@@ -154,11 +161,23 @@ pub fn ___syscall83(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_int
 
     let path1 = varargs.get_str(ctx);
     let path2 = varargs.get_str(ctx);
-    let result = unsafe { symlink(path1, path2) };
+    let real_path1_owned = utils::get_cstr_path(ctx, path1);
+    let real_path1 = if let Some(ref rp) = real_path1_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        path1
+    };
+    let real_path2_owned = utils::get_cstr_path(ctx, path2);
+    let real_path2 = if let Some(ref rp) = real_path2_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        path2
+    };
+    let result = unsafe { symlink(real_path1, real_path2) };
     debug!(
         "=> path1: {}, path2: {}, result: {}",
-        unsafe { std::ffi::CStr::from_ptr(path1).to_str().unwrap() },
-        unsafe { std::ffi::CStr::from_ptr(path2).to_str().unwrap() },
+        unsafe { std::ffi::CStr::from_ptr(real_path1).to_str().unwrap() },
+        unsafe { std::ffi::CStr::from_ptr(real_path2).to_str().unwrap() },
         result,
     );
     result
@@ -181,12 +200,18 @@ pub fn ___syscall194(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_in
 pub fn ___syscall198(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_int {
     debug!("emscripten::___syscall198 (lchown) {}", _which);
     let path_ptr = varargs.get_str(ctx);
+    let real_path_owned = utils::get_cstr_path(ctx, path_ptr);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        path_ptr
+    };
     let uid: uid_t = varargs.get(ctx);
     let gid: gid_t = varargs.get(ctx);
-    let result = unsafe { lchown(path_ptr, uid, gid) };
+    let result = unsafe { lchown(real_path, uid, gid) };
     debug!(
         "=> path: {}, uid: {}, gid: {}, result: {}",
-        unsafe { std::ffi::CStr::from_ptr(path_ptr).to_str().unwrap() },
+        unsafe { std::ffi::CStr::from_ptr(real_path).to_str().unwrap() },
         uid,
         gid,
         result,
@@ -216,10 +241,16 @@ pub fn ___syscall212(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_in
     debug!("emscripten::___syscall212 (chown) {}", _which);
 
     let pathname_addr = varargs.get_str(ctx);
+    let real_path_owned = utils::get_cstr_path(ctx, pathname_addr);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        pathname_addr
+    };
     let owner: u32 = varargs.get(ctx);
     let group: u32 = varargs.get(ctx);
 
-    unsafe { chown(pathname_addr, owner, group) }
+    unsafe { chown(real_path, owner, group) }
 }
 
 /// madvise
@@ -239,11 +270,17 @@ pub fn ___syscall219(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_in
 pub fn ___syscall33(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_int {
     debug!("emscripten::___syscall33 (access) {}", _which);
     let path = varargs.get_str(ctx);
+    let real_path_owned = utils::get_cstr_path(ctx, path);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        path
+    };
     let amode: c_int = varargs.get(ctx);
-    let result = unsafe { access(path, amode) };
+    let result = unsafe { access(real_path, amode) };
     debug!(
         "=> path: {}, amode: {}, result: {}",
-        unsafe { std::ffi::CStr::from_ptr(path).to_str().unwrap() },
+        unsafe { std::ffi::CStr::from_ptr(real_path).to_str().unwrap() },
         amode,
         result
     );
@@ -261,8 +298,14 @@ pub fn ___syscall34(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_int
 pub fn ___syscall39(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_int {
     debug!("emscripten::___syscall39 (mkdir) {}", _which);
     let pathname_addr = varargs.get_str(ctx);
+    let real_path_owned = utils::get_cstr_path(ctx, pathname_addr);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        pathname_addr
+    };
     let mode: u32 = varargs.get(ctx);
-    unsafe { mkdir(pathname_addr, mode as _) }
+    unsafe { mkdir(real_path, mode as _) }
 }
 
 /// dup
@@ -761,6 +804,12 @@ pub fn ___syscall122(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_in
 pub fn ___syscall196(ctx: &mut Ctx, _which: i32, mut varargs: VarArgs) -> i32 {
     debug!("emscripten::___syscall196 (lstat64) {}", _which);
     let path = varargs.get_str(ctx);
+    let real_path_owned = utils::get_cstr_path(ctx, path);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        path
+    };
     let buf_ptr: u32 = varargs.get(ctx);
     unsafe {
         let mut stat: stat = std::mem::zeroed();
@@ -771,9 +820,9 @@ pub fn ___syscall196(ctx: &mut Ctx, _which: i32, mut varargs: VarArgs) -> i32 {
         let stat_ptr = &mut stat as *mut stat;
 
         #[cfg(target_os = "macos")]
-        let ret = lstat64(path, stat_ptr);
+        let ret = lstat64(real_path, stat_ptr);
         #[cfg(not(target_os = "macos"))]
-        let ret = lstat(path, stat_ptr);
+        let ret = lstat(real_path, stat_ptr);
 
         debug!("ret: {}", ret);
         if ret != 0 {
@@ -784,6 +833,45 @@ pub fn ___syscall196(ctx: &mut Ctx, _which: i32, mut varargs: VarArgs) -> i32 {
     0
 }
 
+// getdents
+// dirent structure is
+// i64, i64, u16 (280), i8, [i8; 256]
+pub fn ___syscall220(ctx: &mut Ctx, _which: i32, mut varargs: VarArgs) -> i32 {
+    debug!("emscripten::___syscall220");
+    let fd: i32 = varargs.get(ctx);
+    let dirp_addr: i32 = varargs.get(ctx);
+    let count: u32 = varargs.get(ctx);
+
+    let dirp = emscripten_memory_pointer!(ctx.memory(0), dirp_addr) as *mut u8;
+    // need to persist stream across calls?
+    let dir: *mut libc::DIR = unsafe { libc::fdopendir(fd) };
+
+    let mut pos = 0;
+    let offset = 280;
+    while pos + offset <= count as usize {
+        let dirent = unsafe { readdir(dir) };
+        if dirent.is_null() {
+            break;
+        }
+        #[allow(clippy::cast_ptr_alignment)]
+        unsafe {
+            *(dirp.add(pos) as *mut u64) = (*dirent).d_ino;
+            *(dirp.add(pos + 8) as *mut u64) = pos as u64 + offset as u64;
+            *(dirp.add(pos + 16) as *mut u16) = offset as u16;
+            *(dirp.add(pos + 18) as *mut u8) = (*dirent).d_type;
+            let upper_bound = std::cmp::min((*dirent).d_reclen, 254) as usize;
+            let mut i = 0;
+            while i < upper_bound {
+                *(dirp.add(pos + 19 + i) as *mut i8) = (*dirent).d_name[i];
+                i += 1;
+            }
+            *(dirp.add(pos + 19 + i) as *mut i8) = 0 as i8;
+        }
+        pos += offset;
+    }
+    pos as i32
+}
+
 /// fallocate
 pub fn ___syscall324(ctx: &mut Ctx, _which: c_int, mut varargs: VarArgs) -> c_int {
     debug!("emscripten::___syscall324 (fallocate) {}", _which);

lib/emscripten/src/syscalls/windows.rs

@@ -1,4 +1,4 @@
-use crate::utils::copy_cstr_into_wasm;
+use crate::utils::{copy_cstr_into_wasm, get_cstr_path};
 use crate::varargs::VarArgs;
 use libc::mkdir;
 use libc::open;
@@ -19,9 +19,15 @@ pub fn ___syscall5(ctx: &mut Ctx, which: c_int, mut varargs: VarArgs) -> c_int {
     #[cfg(not(feature = "debug"))]
     let _ = which;
     let pathname_addr = varargs.get_str(ctx);
+    let real_path_owned = get_cstr_path(ctx, pathname_addr);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        pathname_addr
+    };
     let flags: i32 = varargs.get(ctx);
     let mode: u32 = varargs.get(ctx);
-    let path_str = unsafe { std::ffi::CStr::from_ptr(pathname_addr).to_str().unwrap() };
+    let path_str = unsafe { std::ffi::CStr::from_ptr(real_path).to_str().unwrap() };
     match path_str {
         "/dev/urandom" => {
             // create a fake urandom file for windows, super hacky
@@ -47,7 +53,7 @@ pub fn ___syscall5(ctx: &mut Ctx, which: c_int, mut varargs: VarArgs) -> c_int {
             fd
         }
         _ => {
-            let fd = unsafe { open(pathname_addr, flags, mode) };
+            let fd = unsafe { open(real_path, flags, mode) };
             debug!(
                 "=> pathname: {}, flags: {}, mode: {} = fd: {}\npath: {}",
                 path_str, flags, mode, fd, path_str
@@ -95,7 +101,13 @@ pub fn ___syscall39(ctx: &mut Ctx, which: c_int, mut varargs: VarArgs) -> c_int
     #[cfg(not(feature = "debug"))]
     let _ = which;
     let pathname_addr = varargs.get_str(ctx);
-    unsafe { mkdir(pathname_addr) }
+    let real_path_owned = get_cstr_path(ctx, pathname_addr);
+    let real_path = if let Some(ref rp) = real_path_owned {
+        rp.as_c_str().as_ptr()
+    } else {
+        pathname_addr
+    };
+    unsafe { mkdir(real_path) }
 }
 
 /// dup
@@ -249,6 +261,12 @@ pub fn ___syscall196(_ctx: &mut Ctx, _one: i32, _two: i32) -> i32 {
     -1
 }
 
+// getdents
+pub fn ___syscall220(_ctx: &mut Ctx, _one: i32, _two: i32) -> i32 {
+    debug!("emscripten::___syscall220");
+    -1
+}
+
 /// fchown
 pub fn ___syscall207(_ctx: &mut Ctx, _which: c_int, _varargs: VarArgs) -> c_int {
     debug!("emscripten::___syscall207 (fchown) {}", _which);

lib/emscripten/src/utils.rs

@@ -5,6 +5,7 @@ use libc::stat;
 use std::ffi::CStr;
 use std::mem::size_of;
 use std::os::raw::c_char;
+use std::path::PathBuf;
 use std::slice;
 use wasmer_runtime_core::memory::Memory;
 use wasmer_runtime_core::{
@@ -204,6 +205,61 @@ pub fn read_string_from_wasm(memory: &Memory, offset: u32) -> String {
     String::from_utf8_lossy(&v).to_owned().to_string()
 }
 
+/// This function trys to find an entry in mapdir
+/// translating paths into their correct value
+pub fn get_cstr_path(ctx: &mut Ctx, path: *const i8) -> Option<std::ffi::CString> {
+    use std::collections::VecDeque;
+
+    let path_str = unsafe { std::ffi::CStr::from_ptr(path).to_str().unwrap() }.to_string();
+    let data = get_emscripten_data(ctx);
+    let path = PathBuf::from(path_str);
+    let mut prefix_added = false;
+    let mut components = path.components().collect::<VecDeque<_>>();
+    // TODO(mark): handle absolute/non-canonical/non-relative paths too (this
+    // functionality should be shared among the abis)
+    if components.len() == 1 {
+        components.push_front(std::path::Component::CurDir);
+        prefix_added = true;
+    }
+    let mut cumulative_path = PathBuf::new();
+    for c in components.into_iter() {
+        cumulative_path.push(c);
+        if let Some(val) = data
+            .mapped_dirs
+            .get(&cumulative_path.to_string_lossy().to_string())
+        {
+            let rest_of_path = if !prefix_added {
+                path.strip_prefix(cumulative_path).ok()?
+            } else {
+                &path
+            };
+            let rebased_path = val.join(rest_of_path);
+            return std::ffi::CString::new(rebased_path.to_string_lossy().as_bytes()).ok();
+        }
+    }
+    None
+}
+
+/// gets the current directory
+/// handles mapdir logic
+pub fn get_current_directory(ctx: &mut Ctx) -> Option<PathBuf> {
+    if let Some(val) = get_emscripten_data(ctx).mapped_dirs.get(".") {
+        return Some(val.clone());
+    }
+    std::env::current_dir()
+        .map(|cwd| {
+            if let Some(val) = get_emscripten_data(ctx)
+                .mapped_dirs
+                .get(&cwd.to_string_lossy().to_string())
+            {
+                val.clone()
+            } else {
+                cwd
+            }
+        })
+        .ok()
+}
+
 #[cfg(test)]
 mod tests {
     use super::is_emscripten_module;