diff --git a/lib/llvm-backend/src/intrinsics.rs b/lib/llvm-backend/src/intrinsics.rs index 0e00ff421..4dedfd5bc 100644 --- a/lib/llvm-backend/src/intrinsics.rs +++ b/lib/llvm-backend/src/intrinsics.rs @@ -164,6 +164,7 @@ impl Intrinsics { let memory_base_ty = i8_ty; let memory_bound_ty = void_ty; let internals_ty = i64_ty; + let interrupt_signal_mem_ty = i8_ty; let local_function_ty = i8_ptr_ty; let anyfunc_ty = context.struct_type( @@ -222,6 +223,9 @@ impl Intrinsics { internals_ty .ptr_type(AddressSpace::Generic) .as_basic_type_enum(), + interrupt_signal_mem_ty + .ptr_type(AddressSpace::Generic) + .as_basic_type_enum(), local_function_ty .ptr_type(AddressSpace::Generic) .as_basic_type_enum(), diff --git a/lib/middleware-common/src/call_trace.rs b/lib/middleware-common/src/call_trace.rs index 9c47d7d7b..04a763abc 100644 --- a/lib/middleware-common/src/call_trace.rs +++ b/lib/middleware-common/src/call_trace.rs @@ -17,6 +17,7 @@ impl FunctionMiddleware for CallTrace { Event::Internal(InternalEvent::FunctionBegin(id)) => sink.push(Event::Internal( InternalEvent::Breakpoint(Box::new(move |_| { eprintln!("func ({})", id); + Ok(()) })), )), _ => {} diff --git a/lib/middleware-common/src/metering.rs b/lib/middleware-common/src/metering.rs index dd5b856b5..8546be47c 100644 --- a/lib/middleware-common/src/metering.rs +++ b/lib/middleware-common/src/metering.rs @@ -94,11 +94,9 @@ impl FunctionMiddleware for Metering { sink.push(Event::WasmOwned(Operator::If { ty: WpType::EmptyBlockType, })); - sink.push(Event::Internal(InternalEvent::Breakpoint(Box::new( - move |ctx| unsafe { - (ctx.throw)(Box::new(ExecutionLimitExceededError)); - }, - )))); + sink.push(Event::Internal(InternalEvent::Breakpoint(Box::new(|_| { + Err(Box::new(ExecutionLimitExceededError)) + })))); sink.push(Event::WasmOwned(Operator::End)); } _ => {} diff --git a/lib/runtime-core/image-loading-linux-x86-64.s b/lib/runtime-core/image-loading-linux-x86-64.s index c357d8354..a1d8aef03 100644 --- a/lib/runtime-core/image-loading-linux-x86-64.s +++ b/lib/runtime-core/image-loading-linux-x86-64.s @@ -16,8 +16,42 @@ leaq run_on_alternative_stack.returning(%rip), %rax movq %rax, -24(%rdi) movq %rsi, %rsp + +movq (%rsp), %xmm0 +add $8, %rsp + +movq (%rsp), %xmm1 +add $8, %rsp + +movq (%rsp), %xmm2 +add $8, %rsp + +movq (%rsp), %xmm3 +add $8, %rsp + +movq (%rsp), %xmm4 +add $8, %rsp + +movq (%rsp), %xmm5 +add $8, %rsp + +movq (%rsp), %xmm6 +add $8, %rsp + +movq (%rsp), %xmm7 +add $8, %rsp + popq %rbp +popq %rax popq %rbx +popq %rcx +popq %rdx +popq %rdi +popq %rsi +popq %r8 +popq %r9 +popq %r10 +popq %r11 popq %r12 popq %r13 popq %r14 diff --git a/lib/runtime-core/image-loading-macos-x86-64.s b/lib/runtime-core/image-loading-macos-x86-64.s index 64e279f1f..416896f1d 100644 --- a/lib/runtime-core/image-loading-macos-x86-64.s +++ b/lib/runtime-core/image-loading-macos-x86-64.s @@ -16,8 +16,42 @@ leaq _run_on_alternative_stack.returning(%rip), %rax movq %rax, -24(%rdi) movq %rsi, %rsp + +movq (%rsp), %xmm0 +add $8, %rsp + +movq (%rsp), %xmm1 +add $8, %rsp + +movq (%rsp), %xmm2 +add $8, %rsp + +movq (%rsp), %xmm3 +add $8, %rsp + +movq (%rsp), %xmm4 +add $8, %rsp + +movq (%rsp), %xmm5 +add $8, %rsp + +movq (%rsp), %xmm6 +add $8, %rsp + +movq (%rsp), %xmm7 +add $8, %rsp + popq %rbp +popq %rax popq %rbx +popq %rcx +popq %rdx +popq %rdi +popq %rsi +popq %r8 +popq %r9 +popq %r10 +popq %r11 popq %r12 popq %r13 popq %r14 diff --git a/lib/runtime-core/src/alternative_stack.rs b/lib/runtime-core/src/alternative_stack.rs index aa44eb1de..9ca655d29 100644 --- a/lib/runtime-core/src/alternative_stack.rs +++ b/lib/runtime-core/src/alternative_stack.rs @@ -2,49 +2,103 @@ mod raw { use std::ffi::c_void; extern "C" { - pub fn run_on_alternative_stack( - stack_end: *mut u64, - stack_begin: *mut u64, - userdata_arg2: *mut u8, - ) -> u64; + pub fn run_on_alternative_stack(stack_end: *mut u64, stack_begin: *mut u64) -> u64; pub fn setjmp(env: *mut c_void) -> i32; pub fn longjmp(env: *mut c_void, val: i32) -> !; } } -use crate::state::x64::{read_stack, X64Register, GPR}; -use crate::suspend; +use crate::codegen::{BkptInfo, BkptMap}; +use crate::state::x64::{build_instance_image, read_stack, X64Register, GPR, XMM}; use crate::vm; -use libc::siginfo_t; +use libc::{mmap, mprotect, siginfo_t, MAP_ANON, MAP_PRIVATE, PROT_NONE, PROT_READ, PROT_WRITE}; use nix::sys::signal::{ - sigaction, SaFlags, SigAction, SigHandler, SigSet, SIGBUS, SIGFPE, SIGILL, SIGINT, SIGSEGV, - SIGTRAP, + sigaction, SaFlags, SigAction, SigHandler, SigSet, Signal, SIGBUS, SIGFPE, SIGILL, SIGINT, + SIGSEGV, SIGTRAP, }; use std::any::Any; use std::cell::UnsafeCell; use std::ffi::c_void; use std::process; +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Once; pub(crate) unsafe fn run_on_alternative_stack(stack_end: *mut u64, stack_begin: *mut u64) -> u64 { - raw::run_on_alternative_stack(stack_end, stack_begin, ::std::ptr::null_mut()) + raw::run_on_alternative_stack(stack_end, stack_begin) } const SETJMP_BUFFER_LEN: usize = 27; type SetJmpBuffer = [i32; SETJMP_BUFFER_LEN]; -thread_local! { - static UNWIND: UnsafeCell>)>> = UnsafeCell::new(None); +struct UnwindInfo { + jmpbuf: SetJmpBuffer, // in + breakpoints: Option, + payload: Option>, // out } -pub unsafe fn catch_unsafe_unwind R>(f: F) -> Result> { +thread_local! { + static UNWIND: UnsafeCell> = UnsafeCell::new(None); +} + +struct InterruptSignalMem(*mut u8); +unsafe impl Send for InterruptSignalMem {} +unsafe impl Sync for InterruptSignalMem {} + +const INTERRUPT_SIGNAL_MEM_SIZE: usize = 4096; + +lazy_static! { + static ref INTERRUPT_SIGNAL_MEM: InterruptSignalMem = { + let ptr = unsafe { + mmap( + ::std::ptr::null_mut(), + INTERRUPT_SIGNAL_MEM_SIZE, + PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANON, + -1, + 0, + ) + }; + if ptr as isize == -1 { + panic!("cannot allocate code memory"); + } + InterruptSignalMem(ptr as _) + }; +} +static INTERRUPT_SIGNAL_DELIVERED: AtomicBool = AtomicBool::new(false); + +pub unsafe fn get_wasm_interrupt_signal_mem() -> *mut u8 { + INTERRUPT_SIGNAL_MEM.0 +} + +pub unsafe fn set_wasm_interrupt() { + let mem: *mut u8 = INTERRUPT_SIGNAL_MEM.0; + if mprotect(mem as _, INTERRUPT_SIGNAL_MEM_SIZE, PROT_NONE) < 0 { + panic!("cannot set PROT_NONE on signal mem"); + } +} + +pub unsafe fn clear_wasm_interrupt() { + let mem: *mut u8 = INTERRUPT_SIGNAL_MEM.0; + if mprotect(mem as _, INTERRUPT_SIGNAL_MEM_SIZE, PROT_READ | PROT_WRITE) < 0 { + panic!("cannot set PROT_READ | PROT_WRITE on signal mem"); + } +} + +pub unsafe fn catch_unsafe_unwind R>( + f: F, + breakpoints: Option, +) -> Result> { let unwind = UNWIND.with(|x| x.get()); let old = (*unwind).take(); - *unwind = Some(([0; SETJMP_BUFFER_LEN], None)); + *unwind = Some(UnwindInfo { + jmpbuf: [0; SETJMP_BUFFER_LEN], + breakpoints: breakpoints, + payload: None, + }); - if raw::setjmp(&mut (*unwind).as_mut().unwrap().0 as *mut SetJmpBuffer as *mut _) != 0 { + if raw::setjmp(&mut (*unwind).as_mut().unwrap().jmpbuf as *mut SetJmpBuffer as *mut _) != 0 { // error - let ret = (*unwind).as_mut().unwrap().1.take().unwrap(); + let ret = (*unwind).as_mut().unwrap().payload.take().unwrap(); *unwind = old; Err(ret) } else { @@ -60,8 +114,16 @@ pub unsafe fn begin_unsafe_unwind(e: Box) -> ! { let inner = (*unwind) .as_mut() .expect("not within a catch_unsafe_unwind scope"); - inner.1 = Some(e); - raw::longjmp(&mut inner.0 as *mut SetJmpBuffer as *mut _, 0xffff); + inner.payload = Some(e); + raw::longjmp(&mut inner.jmpbuf as *mut SetJmpBuffer as *mut _, 0xffff); +} + +unsafe fn with_breakpoint_map) -> R>(f: F) -> R { + let unwind = UNWIND.with(|x| x.get()); + let inner = (*unwind) + .as_mut() + .expect("not within a catch_unsafe_unwind scope"); + f(inner.breakpoints.as_ref()) } pub fn allocate_and_run R>(size: usize, f: F) -> R { @@ -70,7 +132,7 @@ pub fn allocate_and_run R>(size: usize, f: F) -> R { ret: Option, } - extern "C" fn invoke R, R>(_: u64, _: u64, ctx: &mut Context) { + extern "C" fn invoke R, R>(ctx: &mut Context) { let f = ctx.f.take().unwrap(); ctx.ret = Some(f()); } @@ -89,29 +151,65 @@ pub fn allocate_and_run R>(size: usize, f: F) -> R { stack[end_offset - 4] = invoke:: as usize as u64; // NOTE: Keep this consistent with `image-loading-*.s`. - let stack_begin = stack.as_mut_ptr().offset((end_offset - 4 - 6) as isize); + stack[end_offset - 4 - 10] = &mut ctx as *mut Context as usize as u64; // rdi + const NUM_SAVED_REGISTERS: usize = 23; + let stack_begin = stack + .as_mut_ptr() + .offset((end_offset - 4 - NUM_SAVED_REGISTERS) as isize); let stack_end = stack.as_mut_ptr().offset(end_offset as isize); - raw::run_on_alternative_stack( - stack_end, - stack_begin, - &mut ctx as *mut Context as *mut u8, - ); + raw::run_on_alternative_stack(stack_end, stack_begin); ctx.ret.take().unwrap() } } extern "C" fn signal_trap_handler( - _signum: ::nix::libc::c_int, + signum: ::nix::libc::c_int, siginfo: *mut siginfo_t, ucontext: *mut c_void, ) { unsafe { let fault = get_fault_info(siginfo as _, ucontext); - allocate_and_run(65536, || { + let mut unwind_result: Box = Box::new(()); + + let should_unwind = allocate_and_run(1048576, || { + let mut is_suspend_signal = false; + + match Signal::from_c_int(signum) { + Ok(SIGTRAP) => { + // breakpoint + let out: Option>> = with_breakpoint_map(|bkpt_map| { + bkpt_map.and_then(|x| x.get(&(fault.ip as usize))).map(|x| { + x(BkptInfo { + fault: Some(&fault), + }) + }) + }); + match out { + Some(Ok(())) => { + return false; + } + Some(Err(e)) => { + unwind_result = e; + return true; + } + None => {} + } + } + Ok(SIGSEGV) | Ok(SIGBUS) => { + println!("SIGSEGV/SIGBUS on addr {:?}", fault.faulting_addr); + if fault.faulting_addr as usize == get_wasm_interrupt_signal_mem() as usize { + is_suspend_signal = true; + clear_wasm_interrupt(); + INTERRUPT_SIGNAL_DELIVERED.store(false, Ordering::SeqCst); + } + } + _ => {} + } + // TODO: make this safer - let ctx = &*(fault.known_registers[X64Register::GPR(GPR::R15).to_index().0].unwrap() + let ctx = &mut *(fault.known_registers[X64Register::GPR(GPR::R15).to_index().0].unwrap() as *mut vm::Ctx); let rsp = fault.known_registers[X64Register::GPR(GPR::RSP).to_index().0].unwrap(); @@ -120,7 +218,7 @@ extern "C" fn signal_trap_handler( .get_module_state_map() .unwrap(); let code_base = (*ctx.module).runnable_module.get_code().unwrap().as_ptr() as usize; - let image = read_stack( + let es_image = read_stack( &msm, code_base, rsp as usize as *const u64, @@ -128,17 +226,26 @@ extern "C" fn signal_trap_handler( Some(fault.ip as usize as u64), ); - use colored::*; - eprintln!( - "\n{}", - "Wasmer encountered an error while running your WebAssembly program." - .bold() - .red() - ); - image.print_backtrace_if_needed(); + if is_suspend_signal { + let image = build_instance_image(ctx, es_image); + unwind_result = Box::new(image); + } else { + use colored::*; + eprintln!( + "\n{}", + "Wasmer encountered an error while running your WebAssembly program." + .bold() + .red() + ); + es_image.print_backtrace_if_needed(); + } + + true }); - begin_unsafe_unwind(Box::new(())); + if should_unwind { + begin_unsafe_unwind(unwind_result); + } } } @@ -147,14 +254,13 @@ extern "C" fn sigint_handler( _siginfo: *mut siginfo_t, _ucontext: *mut c_void, ) { - if suspend::get_interrupted() { - eprintln!( - "Got another SIGINT before interrupt is handled by WebAssembly program, aborting" - ); + if INTERRUPT_SIGNAL_DELIVERED.swap(true, Ordering::SeqCst) { + eprintln!("Got another SIGINT before trap is triggered on WebAssembly side, aborting"); process::abort(); } - suspend::set_interrupted(true); - eprintln!("Notified WebAssembly program to exit"); + unsafe { + set_wasm_interrupt(); + } } pub fn ensure_sighandler() { @@ -285,12 +391,17 @@ pub unsafe fn get_fault_info(siginfo: *const c_void, ucontext: *const c_void) -> fs: u64, gs: u64, } + #[repr(C)] + struct fpstate { + _unused: [u8; 168], + xmm: [[u64; 2]; 8], + } #[allow(dead_code)] #[repr(C)] struct mcontext_t { es: exception_state, ss: regs, - // ... + fs: fpstate, } let siginfo = siginfo as *const siginfo_t; @@ -298,6 +409,7 @@ pub unsafe fn get_fault_info(siginfo: *const c_void, ucontext: *const c_void) -> let ucontext = ucontext as *const ucontext_t; let ss = &(*(*ucontext).uc_mcontext).ss; + let fs = &(*(*ucontext).uc_mcontext).fs; let mut known_registers: [Option; 24] = [None; 24]; @@ -319,7 +431,14 @@ pub unsafe fn get_fault_info(siginfo: *const c_void, ucontext: *const c_void) -> known_registers[X64Register::GPR(GPR::RBP).to_index().0] = Some(ss.rbp); known_registers[X64Register::GPR(GPR::RSP).to_index().0] = Some(ss.rsp); - // TODO: XMM registers + known_registers[X64Register::XMM(XMM::XMM0).to_index().0] = Some(fs.xmm[0][0]); + known_registers[X64Register::XMM(XMM::XMM1).to_index().0] = Some(fs.xmm[1][0]); + known_registers[X64Register::XMM(XMM::XMM2).to_index().0] = Some(fs.xmm[2][0]); + known_registers[X64Register::XMM(XMM::XMM3).to_index().0] = Some(fs.xmm[3][0]); + known_registers[X64Register::XMM(XMM::XMM4).to_index().0] = Some(fs.xmm[4][0]); + known_registers[X64Register::XMM(XMM::XMM5).to_index().0] = Some(fs.xmm[5][0]); + known_registers[X64Register::XMM(XMM::XMM6).to_index().0] = Some(fs.xmm[6][0]); + known_registers[X64Register::XMM(XMM::XMM7).to_index().0] = Some(fs.xmm[7][0]); FaultInfo { faulting_addr: si_addr, diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index 95451a106..d1fce60fc 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -9,6 +9,7 @@ use crate::{ use crate::{ cache::{Artifact, Error as CacheError}, + codegen::BkptMap, module::ModuleInfo, sys::Memory, }; @@ -89,6 +90,10 @@ pub trait RunnableModule: Send + Sync { None } + fn get_breakpoints(&self) -> Option { + None + } + /// A wasm trampoline contains the necessary data to dynamically call an exported wasm function. /// Given a particular signature index, we are returned a trampoline that is matched with that /// signature and an invoke function that can call the trampoline. diff --git a/lib/runtime-core/src/codegen.rs b/lib/runtime-core/src/codegen.rs index 4ff0c25cb..20f84ef6e 100644 --- a/lib/runtime-core/src/codegen.rs +++ b/lib/runtime-core/src/codegen.rs @@ -9,6 +9,7 @@ use crate::{ }; use smallvec::SmallVec; use std::any::Any; +use std::collections::HashMap; use std::fmt; use std::fmt::Debug; use std::marker::PhantomData; @@ -16,6 +17,9 @@ use std::sync::{Arc, RwLock}; use wasmparser::{self, WasmDecoder}; use wasmparser::{Operator, Type as WpType}; +pub type BkptHandler = Box Result<(), Box> + Send + Sync + 'static>; +pub type BkptMap = Arc>; + #[derive(Debug)] pub enum Event<'a, 'b> { Internal(InternalEvent), @@ -26,7 +30,7 @@ pub enum Event<'a, 'b> { pub enum InternalEvent { FunctionBegin(u32), FunctionEnd, - Breakpoint(Box), + Breakpoint(BkptHandler), SetInternal(u32), GetInternal(u32), } @@ -43,8 +47,8 @@ impl fmt::Debug for InternalEvent { } } -pub struct BkptInfo { - pub throw: unsafe fn(Box) -> !, +pub struct BkptInfo<'a> { + pub fault: Option<&'a dyn Any>, } pub trait ModuleCodeGenerator, RM: RunnableModule, E: Debug> { diff --git a/lib/runtime-core/src/lib.rs b/lib/runtime-core/src/lib.rs index 8b7a29156..c8da29509 100644 --- a/lib/runtime-core/src/lib.rs +++ b/lib/runtime-core/src/lib.rs @@ -46,8 +46,6 @@ pub use trampoline_x64 as trampoline; #[cfg(all(unix, target_arch = "x86_64"))] pub mod alternative_stack; pub mod state; -#[cfg(all(unix, target_arch = "x86_64"))] -pub mod suspend; use self::error::CompileResult; #[doc(inline)] diff --git a/lib/runtime-core/src/state.rs b/lib/runtime-core/src/state.rs index c9b2d8b98..e94daca00 100644 --- a/lib/runtime-core/src/state.rs +++ b/lib/runtime-core/src/state.rs @@ -53,10 +53,17 @@ pub struct FunctionStateMap { pub locals: Vec, pub shadow_size: usize, // for single-pass backend, 32 bytes on x86-64 pub diffs: Vec, + pub wasm_function_header_target_offset: Option, pub wasm_offset_to_target_offset: Vec, - pub loop_offsets: BTreeMap, /* offset -> diff_id */ - pub call_offsets: BTreeMap, /* offset -> diff_id */ - pub trappable_offsets: BTreeMap, /* offset -> diff_id */ + pub loop_offsets: BTreeMap, /* suspend_offset -> info */ + pub call_offsets: BTreeMap, /* suspend_offset -> info */ + pub trappable_offsets: BTreeMap, /* suspend_offset -> info */ +} + +#[derive(Clone, Debug)] +pub struct OffsetInfo { + pub diff_id: usize, + pub activate_offset: usize, } #[derive(Clone, Debug)] @@ -98,7 +105,7 @@ impl ModuleStateMap { .unwrap(); match fsm.call_offsets.get(&(ip - base)) { - Some(x) => Some((fsm, fsm.diffs[*x].build_state(fsm))), + Some(x) => Some((fsm, fsm.diffs[x.diff_id].build_state(fsm))), None => None, } } @@ -120,7 +127,25 @@ impl ModuleStateMap { .unwrap(); match fsm.trappable_offsets.get(&(ip - base)) { - Some(x) => Some((fsm, fsm.diffs[*x].build_state(fsm))), + Some(x) => Some((fsm, fsm.diffs[x.diff_id].build_state(fsm))), + None => None, + } + } + } + + fn lookup_loop_ip(&self, ip: usize, base: usize) -> Option<(&FunctionStateMap, MachineState)> { + if ip < base || ip - base >= self.total_size { + None + } else { + //println!("lookup ip: {} in {:?}", ip - base, self.local_functions); + let (_, fsm) = self + .local_functions + .range((Unbounded, Included(&(ip - base)))) + .last() + .unwrap(); + + match fsm.loop_offsets.get(&(ip - base)) { + Some(x) => Some((fsm, fsm.diffs[x.diff_id].build_state(fsm))), None => None, } } @@ -140,6 +165,7 @@ impl FunctionStateMap { shadow_size, locals, diffs: vec![], + wasm_function_header_target_offset: None, wasm_offset_to_target_offset: Vec::new(), loop_offsets: BTreeMap::new(), call_offsets: BTreeMap::new(), @@ -330,6 +356,7 @@ impl InstanceImage { pub mod x64 { use super::*; use crate::alternative_stack::{catch_unsafe_unwind, run_on_alternative_stack}; + use crate::codegen::BkptMap; use crate::structures::TypedIndex; use crate::types::LocalGlobalIndex; use crate::vm::Ctx; @@ -352,6 +379,7 @@ pub mod x64 { code_base: usize, image: InstanceImage, vmctx: &mut Ctx, + breakpoints: Option, ) -> Result> { let mut stack: Vec = vec![0; 1048576 * 8 / 8]; // 8MB stack let mut stack_offset: usize = stack.len(); @@ -368,15 +396,31 @@ pub mod x64 { // Bottom to top for f in image.execution_state.frames.iter().rev() { let fsm = local_functions_vec[f.local_function_id]; - let call_begin_offset = fsm.wasm_offset_to_target_offset[f.wasm_inst_offset]; + let begin_offset = if f.wasm_inst_offset == ::std::usize::MAX { + fsm.wasm_function_header_target_offset.unwrap() + } else { + fsm.wasm_offset_to_target_offset[f.wasm_inst_offset] + }; - // Left bound must be Excluded because it's possible that the previous instruction's (after-)call offset == call_begin_offset. - let (after_call_inst, diff_id) = fsm - .call_offsets - .range((Excluded(&call_begin_offset), Unbounded)) - .next() - .map(|(k, v)| (*k, *v)) - .expect("instruction offset not found in call offsets"); + let (target_inst_offset, diff_id) = fsm + .loop_offsets + .get(&begin_offset) + .map(|v| (v.activate_offset, v.diff_id)) + .or_else(|| { + fsm.trappable_offsets + .get(&begin_offset) + .map(|v| (v.activate_offset, v.diff_id)) + }) + .or_else(|| { + // Left bound must be Excluded because it's possible that the previous instruction's (after-)call offset == call_begin_offset. + // This might not be the correct offset if begin_offset itself does not correspond to a call(_indirect) instruction, + // but anyway safety isn't broken because diff_id always corresponds to target_inst_offset. + fsm.call_offsets + .range((Excluded(&begin_offset), Unbounded)) + .next() + .map(|(_, v)| (v.activate_offset, v.diff_id)) + }) + .expect("instruction offset not found in any offset type"); let diff = &fsm.diffs[diff_id]; let state = diff.build_state(fsm); @@ -434,7 +478,10 @@ pub mod x64 { } } } - assert!(got_explicit_shadow); + if !got_explicit_shadow { + assert!(fsm.shadow_size % 8 == 0); + stack_offset -= fsm.shadow_size / 8; + } for (i, v) in state.register_values.iter().enumerate() { match *v { MachineValue::Undefined => {} @@ -460,9 +507,11 @@ pub mod x64 { _ => unreachable!(), } } - assert!((stack.len() - stack_offset) % 2 == 0); // 16-byte alignment + + // no need to check 16-byte alignment here because it's possible that we're not at a call entry. + stack_offset -= 1; - stack[stack_offset] = (code_base + after_call_inst) as u64; // return address + stack[stack_offset] = (code_base + target_inst_offset) as u64; // return address } stack_offset -= 1; @@ -477,12 +526,71 @@ pub mod x64 { stack_offset -= 1; stack[stack_offset] = known_registers[X64Register::GPR(GPR::R12).to_index().0].unwrap_or(0); + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R11).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R10).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R9).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::R8).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RSI).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RDI).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RDX).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RCX).to_index().0].unwrap_or(0); + stack_offset -= 1; stack[stack_offset] = known_registers[X64Register::GPR(GPR::RBX).to_index().0].unwrap_or(0); + stack_offset -= 1; + stack[stack_offset] = known_registers[X64Register::GPR(GPR::RAX).to_index().0].unwrap_or(0); + stack_offset -= 1; stack[stack_offset] = stack.as_ptr().offset(last_stack_offset as isize) as usize as u64; // rbp + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM7).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM6).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM5).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM4).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM3).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM2).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM1).to_index().0].unwrap_or(0); + + stack_offset -= 1; + stack[stack_offset] = + known_registers[X64Register::XMM(XMM::XMM0).to_index().0].unwrap_or(0); + if let Some(ref memory) = image.memory { assert!(vmctx.internal.memory_bound <= memory.len()); @@ -512,12 +620,15 @@ pub mod x64 { drop(image); // free up host memory - catch_unsafe_unwind(|| { - run_on_alternative_stack( - stack.as_mut_ptr().offset(stack.len() as isize), - stack.as_mut_ptr().offset(stack_offset as isize), - ) - }) + catch_unsafe_unwind( + || { + run_on_alternative_stack( + stack.as_mut_ptr().offset(stack.len() as isize), + stack.as_mut_ptr().offset(stack_offset as isize), + ) + }, + breakpoints, + ) } pub fn build_instance_image( @@ -575,6 +686,7 @@ pub mod x64 { let (fsm, state) = match msm .lookup_call_ip(ret_addr as usize, code_base) .or_else(|| msm.lookup_trappable_ip(ret_addr as usize, code_base)) + .or_else(|| msm.lookup_loop_ip(ret_addr as usize, code_base)) { Some(x) => x, _ => return ExecutionStateImage { frames: results }, diff --git a/lib/runtime-core/src/suspend.rs b/lib/runtime-core/src/suspend.rs deleted file mode 100644 index e73695dc5..000000000 --- a/lib/runtime-core/src/suspend.rs +++ /dev/null @@ -1,104 +0,0 @@ -use crate::alternative_stack::begin_unsafe_unwind; -use crate::import::{ImportObject, Namespace}; -use crate::trampoline::{CallContext, TrampolineBufferBuilder}; -use crate::vm::Ctx; -use std::sync::atomic::{AtomicBool, Ordering}; - -static INTERRUPTED: AtomicBool = AtomicBool::new(false); - -pub fn set_interrupted(x: bool) { - INTERRUPTED.store(x, Ordering::SeqCst); -} - -pub fn get_interrupted() -> bool { - INTERRUPTED.load(Ordering::SeqCst) -} - -pub fn get_and_reset_interrupted() -> bool { - INTERRUPTED.swap(false, Ordering::SeqCst) -} - -pub fn patch_import_object(x: &mut ImportObject) { - struct Intrinsics { - suspend: fn(&mut Ctx), - check_interrupt: fn(&mut Ctx), - } - - lazy_static! { - static ref INTRINSICS: Intrinsics = { - let mut builder = TrampolineBufferBuilder::new(); - let idx_suspend = - builder.add_context_rsp_state_preserving_trampoline(suspend, ::std::ptr::null()); - let idx_check_interrupt = builder - .add_context_rsp_state_preserving_trampoline(check_interrupt, ::std::ptr::null()); - let trampolines = builder.build(); - - let ret = Intrinsics { - suspend: unsafe { ::std::mem::transmute(trampolines.get_trampoline(idx_suspend)) }, - check_interrupt: unsafe { - ::std::mem::transmute(trampolines.get_trampoline(idx_check_interrupt)) - }, - }; - ::std::mem::forget(trampolines); - ret - }; - } - - let mut ns = Namespace::new(); - - let suspend_fn = INTRINSICS.suspend; - let check_interrupt_fn = INTRINSICS.check_interrupt; - - ns.insert("suspend", func!(suspend_fn)); - ns.insert("check_interrupt", func!(check_interrupt_fn)); - x.register("wasmer_suspend", ns); -} - -#[allow(clippy::cast_ptr_alignment)] -unsafe extern "C" fn check_interrupt(ctx: &mut Ctx, _: *const CallContext, stack: *const u64) { - if get_and_reset_interrupted() { - do_suspend(ctx, stack); - } -} - -#[allow(clippy::cast_ptr_alignment)] -unsafe extern "C" fn suspend(ctx: &mut Ctx, _: *const CallContext, stack: *const u64) { - do_suspend(ctx, stack); -} - -unsafe fn do_suspend(ctx: &mut Ctx, mut stack: *const u64) -> ! { - use crate::state::x64::{build_instance_image, read_stack, X64Register, GPR}; - - let image = { - let msm = (*ctx.module) - .runnable_module - .get_module_state_map() - .unwrap(); - let code_base = (*ctx.module).runnable_module.get_code().unwrap().as_ptr() as usize; - - let mut known_registers: [Option; 24] = [None; 24]; - - let r15 = *stack; - let r14 = *stack.offset(1); - let r13 = *stack.offset(2); - let r12 = *stack.offset(3); - let rbx = *stack.offset(4); - stack = stack.offset(5); - - known_registers[X64Register::GPR(GPR::R15).to_index().0] = Some(r15); - known_registers[X64Register::GPR(GPR::R14).to_index().0] = Some(r14); - known_registers[X64Register::GPR(GPR::R13).to_index().0] = Some(r13); - known_registers[X64Register::GPR(GPR::R12).to_index().0] = Some(r12); - known_registers[X64Register::GPR(GPR::RBX).to_index().0] = Some(rbx); - - let es_image = read_stack(&msm, code_base, stack, known_registers, None); - - { - use colored::*; - eprintln!("{}", "Suspending instance.".green().bold()); - } - build_instance_image(ctx, es_image) - }; - - begin_unsafe_unwind(Box::new(image)); -} diff --git a/lib/runtime-core/src/vm.rs b/lib/runtime-core/src/vm.rs index 1bba38801..6518e4577 100644 --- a/lib/runtime-core/src/vm.rs +++ b/lib/runtime-core/src/vm.rs @@ -100,6 +100,8 @@ pub struct InternalCtx { pub memory_bound: usize, pub internals: *mut [u64; INTERNALS_SIZE], // TODO: Make this dynamic? + + pub interrupt_signal_mem: *mut u8, } static INTERNAL_FIELDS: AtomicUsize = AtomicUsize::new(0); @@ -207,6 +209,17 @@ fn get_intrinsics_for_module(m: &ModuleInfo) -> *const Intrinsics { } } +#[cfg(all(unix, target_arch = "x86_64"))] +fn get_interrupt_signal_mem() -> *mut u8 { + unsafe { crate::alternative_stack::get_wasm_interrupt_signal_mem() } +} + +#[cfg(not(all(unix, target_arch = "x86_64")))] +fn get_interrupt_signal_mem() -> *mut u8 { + static mut REGION: u64 = 0; + unsafe { &mut REGION as *mut u64 as *mut u8 } +} + impl Ctx { #[doc(hidden)] pub unsafe fn new( @@ -245,6 +258,8 @@ impl Ctx { memory_bound: mem_bound, internals: &mut local_backing.internals.0, + + interrupt_signal_mem: get_interrupt_signal_mem(), }, local_functions: local_backing.local_functions.as_ptr(), @@ -296,6 +311,8 @@ impl Ctx { memory_bound: mem_bound, internals: &mut local_backing.internals.0, + + interrupt_signal_mem: get_interrupt_signal_mem(), }, local_functions: local_backing.local_functions.as_ptr(), @@ -419,9 +436,13 @@ impl Ctx { 12 * (mem::size_of::() as u8) } - pub fn offset_local_functions() -> u8 { + pub fn offset_interrupt_signal_mem() -> u8 { 13 * (mem::size_of::() as u8) } + + pub fn offset_local_functions() -> u8 { + 14 * (mem::size_of::() as u8) + } } enum InnerFunc {} @@ -640,6 +661,11 @@ mod vm_offset_tests { offset_of!(InternalCtx => internals).get_byte_offset(), ); + assert_eq!( + Ctx::offset_interrupt_signal_mem() as usize, + offset_of!(InternalCtx => interrupt_signal_mem).get_byte_offset(), + ); + assert_eq!( Ctx::offset_local_functions() as usize, offset_of!(Ctx => local_functions).get_byte_offset(), diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index 41922746a..987dfc293 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -24,7 +24,7 @@ use wasmer_runtime_core::{ module::{ModuleInfo, ModuleInner}, state::{ x64::new_machine_state, x64::X64Register, FunctionStateMap, MachineState, MachineValue, - ModuleStateMap, WasmAbstractValue, + ModuleStateMap, OffsetInfo, WasmAbstractValue, }, structures::{Map, TypedIndex}, typed_func::Wasm, @@ -151,7 +151,12 @@ pub struct X64FunctionCode { assembler: Option, function_labels: Option)>>, - breakpoints: Option>>, + breakpoints: Option< + HashMap< + AssemblyOffset, + Box Result<(), Box> + Send + Sync + 'static>, + >, + >, returns: SmallVec<[WpType; 1]>, locals: Vec, num_params: usize, @@ -179,7 +184,7 @@ pub struct X64ExecutionContext { function_pointers: Vec, function_offsets: Vec, signatures: Arc>, - breakpoints: Arc>>, + breakpoints: BkptMap, func_import_count: usize, msm: ModuleStateMap, } @@ -217,6 +222,10 @@ impl RunnableModule for X64ExecutionContext { Some(self.msm.clone()) } + fn get_breakpoints(&self) -> Option { + Some(self.breakpoints.clone()) + } + fn get_trampoline(&self, _: &ModuleInfo, sig_index: SigIndex) -> Option { use std::ffi::c_void; use wasmer_runtime_core::typed_func::WasmTrapInfo; @@ -245,16 +254,17 @@ impl RunnableModule for X64ExecutionContext { num_params_plus_one.unwrap().as_ptr() as usize - 1, ); let args_reverse: SmallVec<[u64; 8]> = args.iter().cloned().rev().collect(); - protect_unix::BKPT_MAP - .with(|x| x.borrow_mut().push(execution_context.breakpoints.clone())); - let ret = match protect_unix::call_protected(|| { - CONSTRUCT_STACK_AND_CALL_WASM( - args_reverse.as_ptr(), - args_reverse.as_ptr().offset(args_reverse.len() as isize), - ctx, - func.as_ptr(), - ) - }) { + let ret = match protect_unix::call_protected( + || { + CONSTRUCT_STACK_AND_CALL_WASM( + args_reverse.as_ptr(), + args_reverse.as_ptr().offset(args_reverse.len() as isize), + ctx, + func.as_ptr(), + ) + }, + Some(execution_context.breakpoints.clone()), + ) { Ok(x) => { if !rets.is_null() { *rets = x; @@ -269,7 +279,6 @@ impl RunnableModule for X64ExecutionContext { false } }; - protect_unix::BKPT_MAP.with(|x| x.borrow_mut().pop().unwrap()); ret } @@ -548,7 +557,13 @@ impl X64FunctionCode { ) { let state_diff_id = Self::get_state_diff(m, fsm, control_stack); let offset = a.get_offset().0; - fsm.trappable_offsets.insert(offset, state_diff_id); + fsm.trappable_offsets.insert( + offset, + OffsetInfo { + activate_offset: offset, + diff_id: state_diff_id, + }, + ); } /// Moves `loc` to a valid location for `div`/`idiv`. @@ -1204,7 +1219,9 @@ impl X64FunctionCode { } // Align stack to 16 bytes. - if (m.get_stack_offset() + used_gprs.len() * 8 + stack_offset) % 16 != 0 { + if (m.get_stack_offset() + used_gprs.len() * 8 + used_xmms.len() * 8 + stack_offset) % 16 + != 0 + { a.emit_sub(Size::S64, Location::Imm32(8), Location::GPR(GPR::RSP)); stack_offset += 8; m.state.stack_values.push(MachineValue::Undefined); @@ -1299,13 +1316,21 @@ impl X64FunctionCode { Machine::get_param_location(0), ); // vmctx + assert!(m.state.stack_values.len() % 2 == 1); // explicit shadow takes one slot + cb(a); // Offset needs to be after the 'call' instruction. if let Some((fsm, control_stack)) = state_context { let state_diff_id = Self::get_state_diff(m, fsm, control_stack); let offset = a.get_offset().0; - fsm.call_offsets.insert(offset, state_diff_id); + fsm.call_offsets.insert( + offset, + OffsetInfo { + activate_offset: offset, + diff_id: state_diff_id, + }, + ); } // Restore stack. @@ -1642,6 +1667,31 @@ impl FunctionCodeGenerator for X64FunctionCode { state_diff_id, }); + // Check interrupt signal without branching + let activate_offset = a.get_offset().0; + + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_interrupt_signal_mem() as i32, + ), + Location::GPR(GPR::RAX), + ); + self.fsm.loop_offsets.insert( + a.get_offset().0, + OffsetInfo { + activate_offset, + diff_id: state_diff_id, + }, + ); + self.fsm.wasm_function_header_target_offset = Some(a.get_offset().0); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, 0), + Location::GPR(GPR::RAX), + ); + assert_eq!(self.machine.state.wasm_inst_offset, ::std::usize::MAX); Ok(()) @@ -3863,6 +3913,8 @@ impl FunctionCodeGenerator for X64FunctionCode { let label = a.get_label(); let state_diff_id = Self::get_state_diff(&self.machine, &mut self.fsm, &mut self.control_stack); + let activate_offset = a.get_offset().0; + self.control_stack.push(ControlFrame { label: label, loop_like: true, @@ -3875,10 +3927,29 @@ impl FunctionCodeGenerator for X64FunctionCode { state: self.machine.state.clone(), state_diff_id, }); - self.fsm - .loop_offsets - .insert(a.get_offset().0, state_diff_id); a.emit_label(label); + + // Check interrupt signal without branching + a.emit_mov( + Size::S64, + Location::Memory( + Machine::get_vmctx_reg(), + vm::Ctx::offset_interrupt_signal_mem() as i32, + ), + Location::GPR(GPR::RAX), + ); + self.fsm.loop_offsets.insert( + a.get_offset().0, + OffsetInfo { + activate_offset, + diff_id: state_diff_id, + }, + ); + a.emit_mov( + Size::S64, + Location::Memory(GPR::RAX, 0), + Location::GPR(GPR::RAX), + ); } Operator::Nop => {} Operator::MemorySize { reserved } => { diff --git a/lib/singlepass-backend/src/protect_unix.rs b/lib/singlepass-backend/src/protect_unix.rs index 9854ef458..8160e7891 100644 --- a/lib/singlepass-backend/src/protect_unix.rs +++ b/lib/singlepass-backend/src/protect_unix.rs @@ -10,18 +10,15 @@ //! unless you have memory unsafety elsewhere in your code. //! use std::any::Any; -use std::cell::{Cell, RefCell}; -use std::collections::HashMap; -use std::sync::Arc; +use std::cell::Cell; use wasmer_runtime_core::alternative_stack::{ begin_unsafe_unwind, catch_unsafe_unwind, ensure_sighandler, }; -use wasmer_runtime_core::codegen::BkptInfo; +use wasmer_runtime_core::codegen::BkptMap; use wasmer_runtime_core::typed_func::WasmTrapInfo; thread_local! { pub static TRAP_EARLY_DATA: Cell>> = Cell::new(None); - pub static BKPT_MAP: RefCell>>>> = RefCell::new(Vec::new()); } pub unsafe fn trigger_trap() -> ! { @@ -33,17 +30,20 @@ pub enum CallProtError { Error(Box), } -pub fn call_protected(f: impl FnOnce() -> T) -> Result { +pub fn call_protected( + f: impl FnOnce() -> T, + breakpoints: Option, +) -> Result { ensure_sighandler(); unsafe { - let ret = catch_unsafe_unwind(|| f()); + let ret = catch_unsafe_unwind(|| f(), breakpoints); match ret { Ok(x) => Ok(x), - Err(_) => { + Err(e) => { if let Some(data) = TRAP_EARLY_DATA.with(|cell| cell.replace(None)) { Err(CallProtError::Error(data)) } else { - Err(CallProtError::Trap(WasmTrapInfo::Unknown)) + Err(CallProtError::Error(e)) } } } diff --git a/src/bin/wasmer.rs b/src/bin/wasmer.rs index 865514554..e7cd7cd27 100644 --- a/src/bin/wasmer.rs +++ b/src/bin/wasmer.rs @@ -505,14 +505,6 @@ fn execute_wasm(options: &Run) -> Result<(), String> { mapped_dirs, ); - #[cfg(feature = "backend:singlepass")] - { - if options.backend == Backend::Singlepass { - use wasmer_runtime_core::suspend::patch_import_object; - patch_import_object(&mut import_object); - } - } - let mut instance = module .instantiate(&import_object) .map_err(|e| format!("Can't instantiate module: {:?}", e))?; @@ -543,6 +535,8 @@ fn execute_wasm(options: &Run) -> Result<(), String> { } else { None }; + let breakpoints = instance.module.runnable_module.get_breakpoints(); + loop { let ret = if let Some(image) = image.take() { let msm = instance @@ -558,10 +552,14 @@ fn execute_wasm(options: &Run) -> Result<(), String> { code_base, image, instance.context_mut(), + breakpoints.clone(), ) .map(|_| ()) } else { - catch_unsafe_unwind(|| start_raw(instance.context_mut())) + catch_unsafe_unwind( + || start_raw(instance.context_mut()), + breakpoints.clone(), + ) }; if let Err(e) = ret { if let Some(new_image) = e.downcast_ref::() { @@ -709,7 +707,7 @@ fn run(options: Run) { match execute_wasm(&options) { Ok(()) => {} Err(message) => { - eprintln!("{:?}", message); + eprintln!("execute_wasm: {:?}", message); exit(1); } }