diff --git a/examples/single_pass_tests/read_stack.wat b/examples/single_pass_tests/read_stack.wat index c82162688..b4b906cb9 100644 --- a/examples/single_pass_tests/read_stack.wat +++ b/examples/single_pass_tests/read_stack.wat @@ -3,7 +3,7 @@ (func $stack_read (import "wasi_unstable" "stack_read") (type $t1)) (func $_start (export "_start") - (if (i32.ne (call $fib (i32.const 10)) (i32.const 55)) + (if (i32.ne (call $fib (i32.const 1)) (i32.const 1)) (then unreachable) ) ) diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index 67c3e281b..af5cb9ab1 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -1,10 +1,10 @@ use crate::{ error::CompileResult, module::ModuleInner, + state::ModuleStateMap, typed_func::Wasm, types::{LocalFuncIndex, SigIndex}, vm, - state::ModuleStateMap, }; use crate::{ @@ -85,9 +85,9 @@ pub trait RunnableModule: Send + Sync { local_func_index: LocalFuncIndex, ) -> Option>; - fn get_module_state_map( - &self, - ) -> Option { None } + fn get_module_state_map(&self) -> Option { + None + } /// A wasm trampoline contains the necesarry data to dynamically call an exported wasm function. /// Given a particular signature index, we are returned a trampoline that is matched with that diff --git a/lib/runtime-core/src/state.rs b/lib/runtime-core/src/state.rs index cb8d4d436..e1c2d866a 100644 --- a/lib/runtime-core/src/state.rs +++ b/lib/runtime-core/src/state.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::fmt::Debug; use std::ops::Bound::{Included, Unbounded}; #[derive(Copy, Clone, Debug, Eq, PartialEq, Hash)] @@ -23,7 +24,7 @@ pub enum MachineValue { Undefined, PreserveRegister(RegisterIndex), CopyStackBPRelative(i32), // relative to Base Pointer, in byte offset - ExplicitShadow, // indicates that all values above this are above the shadow region + ExplicitShadow, // indicates that all values above this are above the shadow region WasmStack(usize), WasmLocal(usize), } @@ -43,14 +44,64 @@ pub struct ModuleStateMap { pub total_size: usize, } +#[derive(Clone, Debug)] +pub struct DenseArrayMap { + pub elements: Vec>, +} + +impl DenseArrayMap { + pub fn new() -> DenseArrayMap { + DenseArrayMap { elements: vec![] } + } + + pub fn set(&mut self, idx: usize, elem: T) { + while self.elements.len() < idx + 1 { + self.elements.push(None); + } + self.elements[idx] = Some(elem); + } + + pub fn into_vec(self) -> Option> { + let mut ret: Vec = Vec::with_capacity(self.elements.len()); + for elem in self.elements { + if elem.is_none() { + return None; + } + ret.push(elem.unwrap()); + } + Some(ret) + } +} + +#[derive(Clone, Debug)] +pub struct WasmFunctionState { + stack: Vec>, + locals: Vec, +} + impl ModuleStateMap { - pub fn lookup_call_ip(&self, ip: usize, base: usize) -> Option<(&FunctionStateMap, MachineState)> { + pub fn lookup_call_ip( + &self, + ip: usize, + base: usize, + ) -> Option<(&FunctionStateMap, MachineState)> { if ip < base || ip - base >= self.total_size { None } else { //println!("lookup ip: {} in {:?}", ip - base, self.local_functions); - let fsm = self.local_functions.range((Unbounded, Included(&(ip - base)))).last().map(|x| x.1).unwrap(); - Some((fsm, fsm.call_offsets.get(&(ip - base)).map(|x| fsm.diffs[*x].build_state(fsm)).unwrap())) + let fsm = self + .local_functions + .range((Unbounded, Included(&(ip - base)))) + .last() + .map(|x| x.1) + .unwrap(); + Some(( + fsm, + fsm.call_offsets + .get(&(ip - base)) + .map(|x| fsm.diffs[*x].build_state(fsm)) + .unwrap(), + )) } } } @@ -69,11 +120,20 @@ impl FunctionStateMap { impl MachineState { pub fn diff(&self, old: &MachineState) -> MachineStateDiff { - let first_diff_stack_depth: usize = self.stack_values.iter().zip(old.stack_values.iter()).enumerate() - .find(|&(_, (&a, &b))| a != b).map(|x| x.0) + let first_diff_stack_depth: usize = self + .stack_values + .iter() + .zip(old.stack_values.iter()) + .enumerate() + .find(|&(_, (&a, &b))| a != b) + .map(|x| x.0) .unwrap_or(old.stack_values.len().min(self.stack_values.len())); assert_eq!(self.register_values.len(), old.register_values.len()); - let reg_diff: Vec<_> = self.register_values.iter().zip(old.register_values.iter()).enumerate() + let reg_diff: Vec<_> = self + .register_values + .iter() + .zip(old.register_values.iter()) + .enumerate() .filter(|&(_, (&a, &b))| a != b) .map(|(i, (&a, _))| (RegisterIndex(i), a)) .collect(); @@ -124,31 +184,83 @@ pub mod x64 { } } - pub fn read_stack(msm: &ModuleStateMap, code_base: usize, mut stack: *const u64) { + #[warn(unused_variables)] + pub unsafe fn read_stack(msm: &ModuleStateMap, code_base: usize, mut stack: *const u64) { + let r15 = *stack; + let r14 = *stack.offset(1); + let r13 = *stack.offset(2); + let r12 = *stack.offset(3); + let rbx = *stack.offset(4); + stack = stack.offset(5); + + let mut next_known_registers: [Option; 24] = [None; 24]; + next_known_registers[X64Register::GPR(GPR::R15).to_index().0] = Some(r15); + next_known_registers[X64Register::GPR(GPR::R14).to_index().0] = Some(r14); + next_known_registers[X64Register::GPR(GPR::R13).to_index().0] = Some(r13); + next_known_registers[X64Register::GPR(GPR::R12).to_index().0] = Some(r12); + next_known_registers[X64Register::GPR(GPR::RBX).to_index().0] = Some(rbx); + for i in 0.. { - unsafe { - let ret_addr = *stack; - stack = stack.offset(1); - let (fsm, state) = match msm.lookup_call_ip(ret_addr as usize, code_base) { - Some(x) => x, - _ => break - }; - let mut found_shadow = false; - for v in &state.stack_values { - match *v { - MachineValue::ExplicitShadow => { - stack = stack.offset((fsm.shadow_size / 8) as isize); - found_shadow = true; - } - _ => { - stack = stack.offset(1); - } + let known_registers = ::std::mem::replace(&mut next_known_registers, [None; 24]); + let mut wasm_stack: DenseArrayMap = DenseArrayMap::new(); + let mut wasm_locals: DenseArrayMap = DenseArrayMap::new(); + let ret_addr = *stack; + stack = stack.offset(1); + let (fsm, state) = match msm.lookup_call_ip(ret_addr as usize, code_base) { + Some(x) => x, + _ => break, + }; + let mut found_shadow = false; + for v in state.stack_values.iter().rev() { + match *v { + MachineValue::ExplicitShadow => { + stack = stack.offset((fsm.shadow_size / 8) as isize); + found_shadow = true; + } + MachineValue::Undefined => { + stack = stack.offset(1); + } + MachineValue::PreserveRegister(idx) => { + next_known_registers[idx.0] = Some(*stack); + stack = stack.offset(1); + } + MachineValue::CopyStackBPRelative(offset) => { + stack = stack.offset(1); + } + MachineValue::WasmStack(idx) => { + wasm_stack.set(idx, *stack); + stack = stack.offset(1); + } + MachineValue::WasmLocal(idx) => { + wasm_locals.set(idx, *stack); + stack = stack.offset(1); } } - assert_eq!(found_shadow, true); - stack = stack.offset(1); // RBP - println!("Frame #{}: {:p} {:?}", i, ret_addr as *const u8, state); } + for (i, v) in state.register_values.iter().enumerate() { + match *v { + MachineValue::Undefined => {} + MachineValue::WasmStack(idx) => { + if let Some(v) = known_registers[i] { + wasm_stack.set(idx, v); + } + } + MachineValue::WasmLocal(idx) => { + if let Some(v) = known_registers[i] { + wasm_locals.set(idx, v); + } + } + _ => unreachable!(), + } + } + assert_eq!(found_shadow, true); + stack = stack.offset(1); // RBP + + let wfs = WasmFunctionState { + stack: wasm_stack.elements, + locals: wasm_locals.into_vec().unwrap(), + }; + println!("Frame #{}: {:p} {:?}", i, ret_addr as *const u8, wfs); } } diff --git a/lib/runtime-core/src/trampoline_x64.rs b/lib/runtime-core/src/trampoline_x64.rs index b12ee67bf..80de3e0d6 100644 --- a/lib/runtime-core/src/trampoline_x64.rs +++ b/lib/runtime-core/src/trampoline_x64.rs @@ -7,9 +7,9 @@ //! Variadic functions are not supported because `rax` is used by the trampoline code. use crate::loader::CodeMemory; -use std::{mem, slice}; use crate::vm::Ctx; use std::fmt; +use std::{mem, slice}; lazy_static! { /// Reads the context pointer from `mm0`. @@ -100,13 +100,21 @@ impl TrampolineBufferBuilder { idx } - pub fn add_context_rsp_trampoline( + pub fn add_context_rsp_state_preserving_trampoline( &mut self, - target: unsafe extern "C" fn (&mut Ctx, *const CallContext, *const u64), + target: unsafe extern "C" fn(&mut Ctx, *const CallContext, *const u64), context: *const CallContext, ) -> usize { let idx = self.offsets.len(); self.offsets.push(self.code.len()); + + self.code.extend_from_slice(&[ + 0x53, // push %rbx + 0x41, 0x54, // push %r12 + 0x41, 0x55, // push %r13 + 0x41, 0x56, // push %r14 + 0x41, 0x57, // push %r15 + ]); self.code.extend_from_slice(&[ 0x48, 0xbe, // movabsq ?, %rsi ]); @@ -120,7 +128,14 @@ impl TrampolineBufferBuilder { ]); self.code.extend_from_slice(value_to_bytes(&target)); self.code.extend_from_slice(&[ - 0xff, 0xe0, // jmpq *%rax + 0xff, 0xd0, // callq *%rax + ]); + self.code.extend_from_slice(&[ + 0x48, 0x81, 0xc4, // add ?, %rsp + ]); + self.code.extend_from_slice(value_to_bytes(&40i32)); // 5 * 8 + self.code.extend_from_slice(&[ + 0xc3, //retq ]); idx } diff --git a/lib/singlepass-backend/src/codegen_x64.rs b/lib/singlepass-backend/src/codegen_x64.rs index d553ad1b3..d414e570b 100644 --- a/lib/singlepass-backend/src/codegen_x64.rs +++ b/lib/singlepass-backend/src/codegen_x64.rs @@ -11,7 +11,7 @@ use smallvec::SmallVec; use std::ptr::NonNull; use std::{ any::Any, - collections::{HashMap, BTreeMap}, + collections::{BTreeMap, HashMap}, sync::{Arc, RwLock}, }; use wasmer_runtime_core::{ @@ -22,6 +22,10 @@ use wasmer_runtime_core::{ codegen::*, memory::MemoryType, module::{ModuleInfo, ModuleInner}, + state::{ + x64::new_machine_state, x64::X64Register, FunctionStateMap, MachineState, MachineStateDiff, + MachineValue, ModuleStateMap, + }, structures::{Map, TypedIndex}, typed_func::Wasm, types::{ @@ -29,7 +33,6 @@ use wasmer_runtime_core::{ TableIndex, Type, }, vm::{self, LocalGlobal, LocalTable, INTERNALS_SIZE}, - state::{ModuleStateMap, FunctionStateMap, x64::X64Register, MachineState, MachineValue, MachineStateDiff, x64::new_machine_state}, }; use wasmparser::{Operator, Type as WpType}; @@ -146,7 +149,7 @@ pub struct InstMetadata { #[derive(Copy, Clone, Debug)] pub enum SpecialInst { - Loop, /* header state */ + Loop, /* header state */ Call { mid_offset: usize }, /* inside state */ } @@ -453,9 +456,11 @@ impl ModuleCodeGenerator .collect(), ); - let local_function_maps: BTreeMap = self.functions.iter().map(|x| { - (x.offset, x.fsm.clone()) - }).collect(); + let local_function_maps: BTreeMap = self + .functions + .iter() + .map(|x| (x.offset, x.fsm.clone())) + .collect(); struct Placeholder; impl CacheGen for Placeholder { @@ -767,7 +772,11 @@ impl X64FunctionCode { // Using Red Zone here. let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; if loc_a != ret { let tmp = m.acquire_temp_gpr().unwrap(); @@ -806,7 +815,11 @@ impl X64FunctionCode { // Using Red Zone here. let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; if loc_a != ret { let tmp = m.acquire_temp_gpr().unwrap(); @@ -846,7 +859,11 @@ impl X64FunctionCode { // Using Red Zone here. let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; match ret { Location::GPR(x) => { Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S32, loc_b, loc_a); @@ -888,7 +905,11 @@ impl X64FunctionCode { // Using Red Zone here. let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; match ret { Location::GPR(x) => { Self::emit_relaxed_binop(a, m, Assembler::emit_cmp, Size::S64, loc_b, loc_a); @@ -927,7 +948,11 @@ impl X64FunctionCode { f: fn(&mut Assembler, Size, Location, Location), ) { let loc = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; match loc { Location::Imm32(_) => { @@ -966,7 +991,11 @@ impl X64FunctionCode { f: fn(&mut Assembler, Size, Location, Location), ) { let loc = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; match loc { Location::Imm64(_) | Location::Imm32(_) => { @@ -1006,7 +1035,11 @@ impl X64FunctionCode { ) { let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S32, loc_b, Location::GPR(GPR::RCX)); @@ -1027,7 +1060,11 @@ impl X64FunctionCode { ) { let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S64, loc_b, Location::GPR(GPR::RCX)); @@ -1048,7 +1085,11 @@ impl X64FunctionCode { ) { let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_relaxed_avx(a, m, f, loc_a, loc_b, ret); @@ -1063,7 +1104,11 @@ impl X64FunctionCode { ) { let loc_b = get_location_released(a, m, value_stack.pop().unwrap()); let loc_a = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_relaxed_avx(a, m, f, loc_a, loc_b, ret); @@ -1078,7 +1123,11 @@ impl X64FunctionCode { f: fn(&mut Assembler, XMM, XMMOrMemory, XMM), ) { let loc = get_location_released(a, m, value_stack.pop().unwrap()); - let ret = m.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(value_stack.len()))], false)[0]; + let ret = m.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(value_stack.len()))], + false, + )[0]; value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_relaxed_avx(a, m, f, loc, loc, ret); @@ -1179,7 +1228,9 @@ impl X64FunctionCode { if reg != GPR::RBP { unreachable!(); } - m.state.stack_values.push(MachineValue::CopyStackBPRelative(offset)); // TODO: Read value at this offset + m.state + .stack_values + .push(MachineValue::CopyStackBPRelative(offset)); // TODO: Read value at this offset } _ => { m.state.stack_values.push(MachineValue::Undefined); @@ -1291,7 +1342,10 @@ impl X64FunctionCode { m.state.stack_values.pop().unwrap(); } - assert_eq!(m.state.stack_values.pop().unwrap(), MachineValue::ExplicitShadow); + assert_eq!( + m.state.stack_values.pop().unwrap(), + MachineValue::ExplicitShadow + ); } /// Emits a System V call sequence, specialized for labels as the call target. @@ -1497,7 +1551,11 @@ impl X64FunctionCode { m.release_temp_gpr(tmp); } - pub fn get_state_diff(m: &Machine, fsm: &mut FunctionStateMap, control_stack: &[ControlFrame]) -> usize { + pub fn get_state_diff( + m: &Machine, + fsm: &mut FunctionStateMap, + control_stack: &[ControlFrame], + ) -> usize { let last_frame = control_stack.last().unwrap(); let mut diff = m.state.diff(&last_frame.state); diff.last = Some(last_frame.state_diff_id); @@ -1718,7 +1776,10 @@ impl FunctionCodeGenerator for X64FunctionCode { ); self.machine.acquire_locations( a, - &[(type_to_wp_type(module_info.globals[local_index].desc.ty), MachineValue::WasmStack(self.value_stack.len()))], + &[( + type_to_wp_type(module_info.globals[local_index].desc.ty), + MachineValue::WasmStack(self.value_stack.len()), + )], false, )[0] } @@ -1738,9 +1799,10 @@ impl FunctionCodeGenerator for X64FunctionCode { ); self.machine.acquire_locations( a, - &[(type_to_wp_type( - module_info.imported_globals[import_index].1.ty, - ), MachineValue::WasmStack(self.value_stack.len()))], + &[( + type_to_wp_type(module_info.imported_globals[import_index].1.ty), + MachineValue::WasmStack(self.value_stack.len()), + )], false, )[0] } @@ -1861,7 +1923,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); a.emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); Self::emit_relaxed_xdiv( @@ -1880,7 +1946,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); a.emit_cdq(); Self::emit_relaxed_xdiv( @@ -1899,7 +1969,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S32, loc_a, Location::GPR(GPR::RAX)); a.emit_xor(Size::S32, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); Self::emit_relaxed_xdiv( @@ -1918,7 +1992,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; let normal_path = a.get_label(); let end = a.get_label(); @@ -2118,7 +2196,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); a.emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); Self::emit_relaxed_xdiv( @@ -2137,7 +2219,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); a.emit_cqo(); Self::emit_relaxed_xdiv( @@ -2156,7 +2242,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; a.emit_mov(Size::S64, loc_a, Location::GPR(GPR::RAX)); a.emit_xor(Size::S64, Location::GPR(GPR::RDX), Location::GPR(GPR::RDX)); Self::emit_relaxed_xdiv( @@ -2175,7 +2265,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; let normal_path = a.get_label(); let end = a.get_label(); @@ -2349,7 +2443,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64ExtendUI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_relaxed_binop( a, @@ -2363,7 +2461,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64ExtendSI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_relaxed_zx_sx( a, @@ -2378,7 +2480,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32WrapI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_relaxed_binop( a, @@ -2501,7 +2607,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp1 = self.machine.acquire_temp_gpr().unwrap(); @@ -2527,7 +2637,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32Abs => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp = self.machine.acquire_temp_gpr().unwrap(); a.emit_mov(Size::S32, loc, Location::GPR(tmp)); @@ -2543,7 +2657,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32Neg => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp = self.machine.acquire_temp_gpr().unwrap(); a.emit_mov(Size::S32, loc, Location::GPR(tmp)); @@ -2663,7 +2781,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let loc_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp1 = self.machine.acquire_temp_gpr().unwrap(); @@ -2698,7 +2820,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64Abs => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp = self.machine.acquire_temp_gpr().unwrap(); @@ -2720,7 +2846,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64Neg => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp = self.machine.acquire_temp_gpr().unwrap(); a.emit_mov(Size::S64, loc, Location::GPR(tmp)); @@ -2745,7 +2875,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32ReinterpretF32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); if loc != ret { @@ -2762,7 +2896,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32ReinterpretI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); if loc != ret { @@ -2780,7 +2918,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64ReinterpretF64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); if loc != ret { @@ -2797,7 +2939,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64ReinterpretI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); if loc != ret { @@ -2815,7 +2961,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32TruncUF32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -2840,7 +2990,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32TruncSF32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -2871,7 +3025,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64TruncSF32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -2916,7 +3074,11 @@ impl FunctionCodeGenerator for X64FunctionCode { */ let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); // xmm2 @@ -2971,7 +3133,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32TruncUF64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -2996,7 +3162,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32TruncSF64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -3032,7 +3202,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64TruncSF64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); @@ -3063,7 +3237,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64TruncUF64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_gpr().unwrap(); let tmp_in = self.machine.acquire_temp_xmm().unwrap(); // xmm2 @@ -3118,7 +3296,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32ConvertSI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3133,7 +3315,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32ConvertUI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3148,7 +3334,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32ConvertSI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3163,7 +3353,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32ConvertUI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3195,7 +3389,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64ConvertSI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3210,7 +3408,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64ConvertUI32 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3225,7 +3427,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64ConvertSI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3240,7 +3446,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64ConvertUI64 => { let loc = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let tmp_out = self.machine.acquire_temp_xmm().unwrap(); let tmp_in = self.machine.acquire_temp_gpr().unwrap(); @@ -3311,7 +3521,14 @@ impl FunctionCodeGenerator for X64FunctionCode { self.machine.release_locations_only_stack(a, &released); if return_types.len() > 0 { - let ret = self.machine.acquire_locations(a, &[(return_types[0], MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[( + return_types[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); } @@ -3426,7 +3643,14 @@ impl FunctionCodeGenerator for X64FunctionCode { self.machine.release_locations_only_stack(a, &released); if return_types.len() > 0 { - let ret = self.machine.acquire_locations(a, &[(return_types[0], MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[( + return_types[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); } @@ -3448,7 +3672,11 @@ impl FunctionCodeGenerator for X64FunctionCode { }, value_stack_depth: self.value_stack.len(), state: self.machine.state.clone(), - state_diff_id: Self::get_state_diff(&self.machine, &mut self.fsm, &self.control_stack), + state_diff_id: Self::get_state_diff( + &self.machine, + &mut self.fsm, + &self.control_stack, + ), }); Self::emit_relaxed_binop( a, @@ -3499,7 +3727,11 @@ impl FunctionCodeGenerator for X64FunctionCode { get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); let v_a = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); let end_label = a.get_label(); @@ -3549,12 +3781,17 @@ impl FunctionCodeGenerator for X64FunctionCode { }, value_stack_depth: self.value_stack.len(), state: self.machine.state.clone(), - state_diff_id: Self::get_state_diff(&self.machine, &mut self.fsm, &self.control_stack), + state_diff_id: Self::get_state_diff( + &self.machine, + &mut self.fsm, + &self.control_stack, + ), }); } Operator::Loop { ty } => { let label = a.get_label(); - let state_diff_id = Self::get_state_diff(&self.machine, &mut self.fsm, &self.control_stack); + let state_diff_id = + Self::get_state_diff(&self.machine, &mut self.fsm, &self.control_stack); self.control_stack.push(ControlFrame { label: label, loop_like: true, @@ -3568,7 +3805,9 @@ impl FunctionCodeGenerator for X64FunctionCode { state_diff_id, }); inst_metadata.special = Some((SpecialInst::Loop, state_diff_id)); - self.fsm.loop_offsets.insert(a.get_offset().0, state_diff_id); + self.fsm + .loop_offsets + .insert(a.get_offset().0, state_diff_id); a.emit_label(label); } Operator::Nop => {} @@ -3596,7 +3835,11 @@ impl FunctionCodeGenerator for X64FunctionCode { ::std::iter::once(Location::Imm32(memory_index.index() as u32)), None, ); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); } @@ -3637,14 +3880,22 @@ impl FunctionCodeGenerator for X64FunctionCode { self.machine.release_locations_only_stack(a, &[param_pages]); } - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); a.emit_mov(Size::S64, Location::GPR(GPR::RAX), ret); } Operator::I32Load { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -3670,7 +3921,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F32Load { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -3696,7 +3951,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32Load8U { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -3723,7 +3982,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32Load8S { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -3750,7 +4013,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32Load16U { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -3777,7 +4044,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I32Load16S { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I32, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -3908,7 +4179,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -3934,7 +4209,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::F64Load { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::F64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -3960,7 +4239,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load8U { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -3987,7 +4270,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load8S { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -4014,7 +4301,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load16U { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -4041,7 +4332,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load16S { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -4068,7 +4363,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load32U { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -4100,7 +4399,11 @@ impl FunctionCodeGenerator for X64FunctionCode { Operator::I64Load32S { ref memarg } => { let target = get_location_released(a, &mut self.machine, self.value_stack.pop().unwrap()); - let ret = self.machine.acquire_locations(a, &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], false)[0]; + let ret = self.machine.acquire_locations( + a, + &[(WpType::I64, MachineValue::WasmStack(self.value_stack.len()))], + false, + )[0]; self.value_stack.push((ret, LocalOrTemp::Temp)); Self::emit_memory_op( @@ -4439,9 +4742,14 @@ impl FunctionCodeGenerator for X64FunctionCode { if frame.returns.len() > 0 { assert_eq!(frame.returns.len(), 1); - let loc = self.machine.acquire_locations(a, &[ - (frame.returns[0], MachineValue::WasmStack(self.value_stack.len())) - ], false)[0]; + let loc = self.machine.acquire_locations( + a, + &[( + frame.returns[0], + MachineValue::WasmStack(self.value_stack.len()), + )], + false, + )[0]; a.emit_mov(Size::S64, Location::GPR(GPR::RAX), loc); self.value_stack.push((loc, LocalOrTemp::Temp)); } diff --git a/lib/singlepass-backend/src/machine.rs b/lib/singlepass-backend/src/machine.rs index 7504e8ea3..2f861751c 100644 --- a/lib/singlepass-backend/src/machine.rs +++ b/lib/singlepass-backend/src/machine.rs @@ -1,9 +1,9 @@ use crate::emitter_x64::*; use smallvec::SmallVec; use std::collections::HashSet; -use wasmparser::Type as WpType; -use wasmer_runtime_core::state::*; use wasmer_runtime_core::state::x64::X64Register; +use wasmer_runtime_core::state::*; +use wasmparser::Type as WpType; struct MachineStackOffset(usize); @@ -188,11 +188,13 @@ impl Machine { match *loc { Location::GPR(ref x) => { assert_eq!(self.used_gprs.remove(x), true); - self.state.register_values[X64Register::GPR(*x).to_index().0] = MachineValue::Undefined; + self.state.register_values[X64Register::GPR(*x).to_index().0] = + MachineValue::Undefined; } Location::XMM(ref x) => { assert_eq!(self.used_xmms.remove(x), true); - self.state.register_values[X64Register::XMM(*x).to_index().0] = MachineValue::Undefined; + self.state.register_values[X64Register::XMM(*x).to_index().0] = + MachineValue::Undefined; } Location::Memory(GPR::RBP, x) => { if x >= 0 { @@ -224,11 +226,13 @@ impl Machine { match *loc { Location::GPR(ref x) => { assert_eq!(self.used_gprs.remove(x), true); - self.state.register_values[X64Register::GPR(*x).to_index().0] = MachineValue::Undefined; + self.state.register_values[X64Register::GPR(*x).to_index().0] = + MachineValue::Undefined; } Location::XMM(ref x) => { assert_eq!(self.used_xmms.remove(x), true); - self.state.register_values[X64Register::XMM(*x).to_index().0] = MachineValue::Undefined; + self.state.register_values[X64Register::XMM(*x).to_index().0] = + MachineValue::Undefined; } _ => {} } @@ -342,7 +346,8 @@ impl Machine { for (i, loc) in locations.iter().enumerate() { match *loc { Location::GPR(x) => { - self.state.register_values[X64Register::GPR(x).to_index().0] = MachineValue::WasmLocal(i); + self.state.register_values[X64Register::GPR(x).to_index().0] = + MachineValue::WasmLocal(i); } Location::Memory(_, _) => { self.state.stack_values.push(MachineValue::WasmLocal(i)); @@ -375,14 +380,18 @@ impl Machine { if let Location::GPR(x) = *loc { a.emit_push(Size::S64, *loc); self.stack_offset.0 += 8; - self.state.stack_values.push(MachineValue::PreserveRegister(X64Register::GPR(x).to_index())); + self.state.stack_values.push(MachineValue::PreserveRegister( + X64Register::GPR(x).to_index(), + )); } } // Save R15 for vmctx use. a.emit_push(Size::S64, Location::GPR(GPR::R15)); self.stack_offset.0 += 8; - self.state.stack_values.push(MachineValue::PreserveRegister(X64Register::GPR(GPR::R15).to_index())); + self.state.stack_values.push(MachineValue::PreserveRegister( + X64Register::GPR(GPR::R15).to_index(), + )); // Save the offset of static area. self.save_area_offset = Some(MachineStackOffset(self.stack_offset.0)); @@ -457,7 +466,11 @@ mod test { fn test_release_locations_keep_state_nopanic() { let mut machine = Machine::new(); let mut assembler = Assembler::new().unwrap(); - let locs = machine.acquire_locations(&mut assembler, &[(WpType::I32, MachineValue::Undefined); 10], false); + let locs = machine.acquire_locations( + &mut assembler, + &[(WpType::I32, MachineValue::Undefined); 10], + false, + ); machine.release_locations_keep_state(&mut assembler, &locs); } diff --git a/lib/wasi/src/lib.rs b/lib/wasi/src/lib.rs index c882ceacf..8e0ed26a7 100644 --- a/lib/wasi/src/lib.rs +++ b/lib/wasi/src/lib.rs @@ -20,10 +20,15 @@ use std::path::PathBuf; pub use self::utils::is_wasi_module; -use wasmer_runtime_core::{func, import::ImportObject, imports, trampoline::{TrampolineBufferBuilder, CallContext}}; -use wasmer_runtime_core::state::{x64::read_stack}; -use wasmer_runtime_core::vm::Ctx; use std::rc::Rc; +use wasmer_runtime_core::state::x64::read_stack; +use wasmer_runtime_core::vm::Ctx; +use wasmer_runtime_core::{ + func, + import::ImportObject, + imports, + trampoline::{CallContext, TrampolineBufferBuilder}, +}; /// This is returned in the Box RuntimeError::Error variant. /// Use `downcast` or `downcast_ref` to retrieve the `ExitCode`. @@ -38,18 +43,20 @@ pub fn generate_import_object( mapped_dirs: Vec<(String, PathBuf)>, ) -> ImportObject { unsafe extern "C" fn read_stack(ctx: &mut Ctx, _: *const CallContext, stack: *const u64) { - let msm = (*ctx.module).runnable_module.get_module_state_map().unwrap(); + let msm = (*ctx.module) + .runnable_module + .get_module_state_map() + .unwrap(); let code_base = (*ctx.module).runnable_module.get_code().unwrap().as_ptr() as usize; self::read_stack(&msm, code_base, stack); } let mut builder = TrampolineBufferBuilder::new(); - let idx = builder.add_context_rsp_trampoline(read_stack, ::std::ptr::null()); + let idx = builder.add_context_rsp_state_preserving_trampoline(read_stack, ::std::ptr::null()); let trampolines = builder.build(); - let read_stack_indirect: fn (&mut Ctx) = unsafe { - ::std::mem::transmute(trampolines.get_trampoline(idx)) - }; + let read_stack_indirect: fn(&mut Ctx) = + unsafe { ::std::mem::transmute(trampolines.get_trampoline(idx)) }; let trampolines = Rc::new(trampolines); diff --git a/lib/wasi/src/state.rs b/lib/wasi/src/state.rs index 1df588e22..79d51f9a7 100644 --- a/lib/wasi/src/state.rs +++ b/lib/wasi/src/state.rs @@ -10,11 +10,11 @@ use std::{ fs, io::{self, Read, Seek, Write}, path::PathBuf, - time::SystemTime, rc::Rc, + time::SystemTime, }; use wasmer_runtime_core::debug; -use wasmer_runtime_core::trampoline::{TrampolineBuffer}; +use wasmer_runtime_core::trampoline::TrampolineBuffer; pub const MAX_SYMLINKS: usize = 100;