Restructure to use external crate, add integration

This commit is contained in:
Mark McCaskey 2020-02-03 16:01:23 -08:00
parent 800b2a42cc
commit 42132c42b6
13 changed files with 256 additions and 191 deletions

61
Cargo.lock generated
View File

@ -456,6 +456,22 @@ dependencies = [
"libc",
]
[[package]]
name = "faerie"
version = "0.13.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f902f2af041f6c7177a2a04f805687cdc71e69c7cbef059a2755d8923f4cd7a8"
dependencies = [
"anyhow",
"goblin 0.1.3",
"indexmap",
"log",
"scroll 0.10.1",
"string-interner",
"target-lexicon 0.9.0",
"thiserror",
]
[[package]]
name = "faerie"
version = "0.14.0"
@ -786,6 +802,12 @@ dependencies = [
"x11-dl",
]
[[package]]
name = "more-asserts"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0debeb9fcf88823ea64d64e4a815ab1643f33127d995978e099942ce38f25238"
[[package]]
name = "nix"
version = "0.15.0"
@ -1724,6 +1746,20 @@ version = "0.9.0+wasi-snapshot-preview1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
[[package]]
name = "wasm-debug"
version = "0.1.0"
dependencies = [
"anyhow",
"cranelift-entity",
"faerie 0.13.0",
"gimli",
"more-asserts",
"target-lexicon 0.9.0",
"thiserror",
"wasmparser 0.39.3",
]
[[package]]
name = "wasmer"
version = "0.13.1"
@ -1772,11 +1808,12 @@ dependencies = [
"serde_bytes",
"serde_derive",
"target-lexicon 0.9.0",
"wasm-debug",
"wasmer-clif-fork-frontend",
"wasmer-clif-fork-wasm",
"wasmer-runtime-core",
"wasmer-win-exception-handler",
"wasmparser",
"wasmparser 0.45.0",
"winapi",
]
@ -1803,18 +1840,19 @@ dependencies = [
"log",
"thiserror",
"wasmer-clif-fork-frontend",
"wasmparser",
"wasmparser 0.45.0",
]
[[package]]
name = "wasmer-debug-writer"
version = "0.13.1"
dependencies = [
"faerie",
"faerie 0.14.0",
"gimli",
"target-lexicon 0.10.0",
"target-lexicon 0.9.0",
"wasm-debug",
"wasmer-runtime-core",
"wasmparser",
"wasmparser 0.45.0",
]
[[package]]
@ -1876,7 +1914,7 @@ dependencies = [
"smallvec 0.6.13",
"wabt",
"wasmer-runtime-core",
"wasmparser",
"wasmparser 0.45.0",
"winapi",
]
@ -1946,6 +1984,7 @@ dependencies = [
"bincode",
"blake3",
"cc",
"cranelift-entity",
"digest",
"errno",
"hex",
@ -1961,7 +2000,9 @@ dependencies = [
"serde_bytes",
"serde_derive",
"smallvec 0.6.13",
"wasmparser",
"target-lexicon 0.9.0",
"wasm-debug",
"wasmparser 0.45.0",
"winapi",
]
@ -2058,6 +2099,12 @@ dependencies = [
"winapi",
]
[[package]]
name = "wasmparser"
version = "0.39.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c702914acda5feeeffbc29e4d953e5b9ce79d8b98da4dbf18a77086e116c5470"
[[package]]
name = "wasmparser"
version = "0.45.0"

View File

@ -23,6 +23,7 @@ byteorder = "1.3.2"
nix = "0.15.0"
libc = "0.2.60"
rayon = "1.1"
wasm-debug = { version = "0.1", path = "../../../Dev/wasm-debug" }
# Dependencies for caching.
[dependencies.serde]

View File

@ -6,7 +6,7 @@ use crate::{
resolver::FuncResolverBuilder, signal::Caller, trampoline::Trampolines,
};
use cranelift_codegen::entity::EntityRef;
use cranelift_codegen::entity::{EntityRef, PrimaryMap};
use cranelift_codegen::ir::{self, Ebb, Function, InstBuilder};
use cranelift_codegen::isa::CallConv;
use cranelift_codegen::{cursor::FuncCursor, isa};
@ -141,9 +141,33 @@ impl ModuleCodeGenerator<CraneliftFunctionCodeGenerator, Caller, CodegenError>
fn finalize(
self,
module_info: &ModuleInfo,
) -> Result<(Caller, Box<dyn CacheGen>), CodegenError> {
) -> Result<((Caller, Option<wasmer_runtime_core::codegen::DebugMetadata>), Box<dyn CacheGen>), CodegenError> {
use wasm_debug::types::{CompiledFunctionData};
let mut debug_metadata = wasmer_runtime_core::codegen::DebugMetadata {
func_info: PrimaryMap::new(),
inst_info: PrimaryMap::new(),
};
let mut func_bodies: Map<LocalFuncIndex, ir::Function> = Map::new();
for f in self.functions.into_iter() {
// TODO: review
// if container is stable, then first and last probably makes more sense here,
let min_srcloc = f.func.srclocs.iter().map(|x| x.1.bits()).min().unwrap_or_default();
let max_srcloc = f.func.srclocs.iter().map(|x| x.1.bits()).max().unwrap_or_default();
let entry = CompiledFunctionData {
instructions: vec![],
start: wasm_debug::types::SourceLoc::new(min_srcloc),
end: wasm_debug::types::SourceLoc::new(max_srcloc),
compiled_offset: 0,
compiled_size: 0,
};
debug_metadata.func_info.push(entry);
/*let mut map = std::collections::HashMap::new();
for (k, v) in f.func.srclocs {
map.
}
debug_metadata.inst_info.push(map);*/
func_bodies.push(f.func);
}
@ -171,7 +195,7 @@ impl ModuleCodeGenerator<CraneliftFunctionCodeGenerator, Caller, CodegenError>
));
Ok((
Caller::new(handler_data, trampolines, func_resolver),
(Caller::new(handler_data, trampolines, func_resolver), Some(debug_metadata)),
cache_gen,
))
}

View File

@ -56,7 +56,7 @@ macro_rules! convert_clif_to_runtime_index {
($clif_index:ident, $runtime_index:ident) => {
impl From<Converter<cranelift_wasm::$clif_index>> for $runtime_index {
fn from(clif_index: Converter<cranelift_wasm::$clif_index>) -> Self {
$runtime_index::new(clif_index.0.index())
<$runtime_index as TypedIndex>::new(clif_index.0.index())
}
}

View File

@ -11,6 +11,7 @@ license = "MIT"
[dependencies]
faerie = "0.14"
gimli = "0.20"
target-lexicon = "0.10"
target-lexicon = "0.9"
wasm-debug = { version = "0.1.0", path = "../../../Dev/wasm-debug" }
wasmer-runtime-core = { path = "../runtime-core", version = "0.13.1" }
wasmparser = "0.45"

View File

@ -3,14 +3,8 @@ use std::str::FromStr;
use std::ptr;
use std::ffi::c_void;
mod read_debug_info;
mod write_debug_info;
mod gc;
mod transform;
pub use crate::read_debug_info::{read_debug_info, DebugInfoData, WasmFileInfo};
pub use crate::write_debug_info::{emit_dwarf, ResolvedSymbol, SymbolResolver};
use crate::transform::WasmTypesDieRefs;
pub use wasm_debug::{read_debuginfo, DebugInfoData, WasmFileInfo};
pub use wasm_debug::{emit_dwarf, ResolvedSymbol, SymbolResolver};
use target_lexicon::{Triple, Architecture, Vendor, OperatingSystem, Environment, BinaryFormat};
use gimli::write::{self, DwarfUnit, Sections, Address, RangeList, EndianVec, AttributeValue, Range};
@ -59,6 +53,7 @@ impl<'a> SymbolResolver for ImageRelocResolver<'a> {
}
}
/*
// the structure of this function and some of its details come from WasmTime
// TODO: attribute
pub fn generate_dwarf(module_info: &ModuleInfo, debug_info_data: &DebugInfoData, code_version: &CodeVersion, platform: Triple) -> Result<Vec<u8>, String> {
@ -153,6 +148,7 @@ pub fn generate_dwarf(module_info: &ModuleInfo, debug_info_data: &DebugInfoData,
}
Ok(vec![])
}
*/
// converts existing dwarf into a usable form with metadata from the JIT
fn reprocess_dwarf(module_info: &ModuleInfo, debug_info_data: &DebugInfoData, code_version: &CodeVersion, platform: Triple) -> Option<write::Dwarf> {
@ -165,174 +161,10 @@ fn reprocess_dwarf(module_info: &ModuleInfo, debug_info_data: &DebugInfoData, co
// where is this documented?
// we need to pass in target triple, isa config, memories/pointers to memories, ranges of where things are,
// and info like function names
pub fn generate_debug_sections_image() -> Option<Vec<u8>> {
pub fn generate_debug_sections_image(bytes: &[u8]) -> Option<Vec<u8>> {
let debug_info = read_debuginfo(bytes);
dbg!(debug_info);
//emit_debugsections_image(X86_64_OSX, 8, debug_info, )
None
}
// do it
// this code copied from WasmTime, TODO: give attribution
// The `emit_wasm_types` function is a derative work of code in WasmTime:
// TODO: update attributions file and/or do clean reimplementation of this logic
//
// Copyright 2019 WasmTime Project Developers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
fn emit_wasm_types(unit: &mut write::Unit, root_id: write::UnitEntryId, string_table: &mut write::StringTable) -> WasmTypesDieRefs {
macro_rules! def_type {
($id:literal, $size:literal, $enc:path) => {{
let die_id = unit.add(root_id, gimli::DW_TAG_base_type);
let die = unit.get_mut(die_id);
die.set(
gimli::DW_AT_name,
write::AttributeValue::StringRef(string_table.add($id)),
);
die.set(gimli::DW_AT_byte_size, write::AttributeValue::Data1($size));
die.set(gimli::DW_AT_encoding, write::AttributeValue::Encoding($enc));
die_id
}};
}
let vmctx_id = {
// TODO: get memory_offset
let memory_offset = 0;
let vmctx_die_id = unit.add(root_id, gimli::DW_TAG_structure_type);
let vmctx_die = unit.get_mut(vmctx_die_id);
vmctx_die.set(
gimli::DW_AT_name,
write::AttributeValue::StringRef(string_table.add("WasmerVMContext")),
);
vmctx_die.set(
gimli::DW_AT_byte_size,
write::AttributeValue::Data4(memory_offset as u32 + 8),
);
let vmctx_ptr_id = unit.add(root_id, gimli::DW_TAG_pointer_type);
let vmctx_ptr_die = unit.get_mut(vmctx_ptr_id);
vmctx_ptr_die.set(
gimli::DW_AT_name,
write::AttributeValue::StringRef(string_table.add("WasmerVMContext*")),
);
vmctx_ptr_die.set(
gimli::DW_AT_type,
write::AttributeValue::ThisUnitEntryRef(vmctx_die_id),
);
vmctx_ptr_id
};
let i32_id = def_type!("i32", 4, gimli::DW_ATE_signed);
let i64_id = def_type!("i64", 8, gimli::DW_ATE_signed);
let i128_id = def_type!("i128", 16, gimli::DW_ATE_signed);
let f32_id = def_type!("f32", 4, gimli::DW_ATE_float);
let f64_id = def_type!("f64", 8, gimli::DW_ATE_float);
WasmTypesDieRefs {
vmctx: vmctx_id,
i32: i32_id,
i64: i64_id,
i128: i128_id,
f32: f32_id,
f64: f64_id,
}
}
// =============================================================================
// LLDB hook magic:
// see lldb/packages/Python/lldbsuite/test/functionalities/jitloader_gdb in
// llvm repo for example
//
// see also https://sourceware.org/gdb/current/onlinedocs/gdb.html#JIT-Interface
#[inline(never)]
pub extern "C" fn __jit_debug_register_code() {
}
#[allow(non_camel_case_types)]
#[derive(Debug)]
#[repr(u32)]
pub enum JITAction { JIT_NOACTION = 0, JIT_REGISTER_FN = 1, JIT_UNREGISTER_FN = 2 }
#[no_mangle]
#[repr(C)]
pub struct JITCodeEntry {
next: *mut JITCodeEntry,
prev: *mut JITCodeEntry,
// TODO: use CStr here?
symfile_addr: *const u8,
symfile_size: u64,
}
impl Default for JITCodeEntry {
fn default() -> Self {
Self {
next: ptr::null_mut(),
prev: ptr::null_mut(),
symfile_addr: ptr::null(),
symfile_size: 0,
}
}
}
#[no_mangle]
#[repr(C)]
pub struct JitDebugDescriptor {
version: u32,
action_flag: u32,
relevant_entry: *mut JITCodeEntry,
first_entry: *mut JITCodeEntry,
}
#[no_mangle]
#[allow(non_upper_case_globals)]
pub static mut __jit_debug_descriptor: JitDebugDescriptor = JitDebugDescriptor {
version: 1,
action_flag: JITAction::JIT_NOACTION as _,
relevant_entry: ptr::null_mut(),
first_entry: ptr::null_mut(),
};
/// Prepend an item to the front of the `__jit_debug_descriptor` entry list
///
/// # Safety
/// - Pointer to [`JITCodeEntry`] should point to a valid entry and stay alive
/// for the 'static lifetime
unsafe fn push_front(jce: *mut JITCodeEntry) {
if __jit_debug_descriptor.first_entry.is_null() {
__jit_debug_descriptor.first_entry = jce;
} else {
let old_first = __jit_debug_descriptor.first_entry;
debug_assert!((*old_first).prev.is_null());
(*jce).next = old_first;
(*old_first).prev = jce;
__jit_debug_descriptor.first_entry = jce;
}
}
pub fn register_new_jit_code_entry(bytes: &'static [u8], action: JITAction) -> *mut JITCodeEntry {
let entry: *mut JITCodeEntry = Box::into_raw(Box::new(JITCodeEntry {
symfile_addr: bytes.as_ptr(),
symfile_size: bytes.len() as _,
..JITCodeEntry::default()
}));
unsafe {
push_front(entry);
__jit_debug_descriptor.relevant_entry = entry;
__jit_debug_descriptor.action_flag = action as u32;
}
entry
}

View File

@ -20,6 +20,11 @@ libc = "0.2.60"
hex = "0.4"
smallvec = "0.6"
bincode = "1.1"
# todo: review
cranelift-entity = "0.52"
# todo, make optional, etc
wasm-debug = { version = "0.1.0", path = "../../../Dev/wasm-debug" }
target-lexicon = "0.9"
[dependencies.indexmap]
version = "1.2"

View File

@ -120,6 +120,8 @@ pub struct CompilerConfig {
pub cpu_features: Option<String>,
pub backend_specific_config: Option<BackendCompilerConfig>,
pub generate_debug_info: bool,
}
/// An exception table for a `RunnableModule`.
@ -210,6 +212,11 @@ pub trait RunnableModule: Send + Sync {
None
}
/// TODO: document before shipppping
fn get_local_function_pointers_and_lengths(&self) -> Option<Vec<(*const u8, usize)>> {
None
}
/// Returns the inline breakpoint size corresponding to an Architecture (None in case is not implemented)
fn get_inline_breakpoint_size(&self, _arch: Architecture) -> Option<usize> {
None

View File

@ -114,12 +114,21 @@ pub trait ModuleCodeGenerator<FCG: FunctionCodeGenerator<E>, RM: RunnableModule,
/// Creates a new function and returns the function-scope code generator for it.
fn next_function(&mut self, module_info: Arc<RwLock<ModuleInfo>>) -> Result<&mut FCG, E>;
/// Finalizes this module.
fn finalize(self, module_info: &ModuleInfo) -> Result<(RM, Box<dyn CacheGen>), E>;
fn finalize(self, module_info: &ModuleInfo) -> Result<((RM, Option<DebugMetadata>), Box<dyn CacheGen>), E>;
/// Creates a module from cache.
unsafe fn from_cache(cache: Artifact, _: Token) -> Result<ModuleInner, CacheError>;
}
use cranelift_entity::PrimaryMap;
/// missing documentation!
pub struct DebugMetadata {
///f unc info
pub func_info: PrimaryMap<FuncIndex, wasm_debug::types::CompiledFunctionData>,
/// inst_info
pub inst_info: PrimaryMap<FuncIndex, wasm_debug::types::ValueLabelsRangesInner>,
}
/// A streaming compiler which is designed to generated code for a module based on a stream
/// of wasm parser events.
pub struct StreamingCompiler<
@ -221,6 +230,7 @@ impl<
CGEN: Fn() -> MiddlewareChain,
> Compiler for StreamingCompiler<MCG, FCG, RM, E, CGEN>
{
#[allow(unused_variables)]
fn compile(
&self,
wasm: &[u8],
@ -231,6 +241,7 @@ impl<
validate_with_features(wasm, &compiler_config.features)?;
}
let mut mcg = match MCG::backend_id() {
"llvm" => MCG::new_with_target(
compiler_config.triple.clone(),
@ -241,11 +252,43 @@ impl<
};
let mut chain = (self.middleware_chain_generator)();
let info = crate::parse::read_module(wasm, &mut mcg, &mut chain, &compiler_config)?;
let (exec_context, cache_gen) =
let ((exec_context, debug_metadata), cache_gen) =
mcg.finalize(&info.read().unwrap())
.map_err(|x| CompileError::InternalError {
msg: format!("{:?}", x),
})?;
use target_lexicon::{Triple, Architecture, Vendor, OperatingSystem, Environment, BinaryFormat};
const X86_64_OSX: Triple = Triple {
architecture: Architecture::X86_64,
vendor: Vendor::Apple,
operating_system: OperatingSystem::Darwin,
environment: Environment::Unknown,
binary_format: BinaryFormat::Macho,
};
if compiler_config.generate_debug_info {
let debug_metadata = debug_metadata.expect("debug metadata");
let debug_info = wasm_debug::read_debuginfo(wasm);
let extra_info = wasm_debug::types::ModuleVmctxInfo {
memory_offset: 0,
stack_slot_offsets: cranelift_entity::PrimaryMap::new(),
};
// lazy type hack (TODO:)
let compiled_fn_map = unsafe { std::mem::transmute(debug_metadata.func_info) };
let range_map = unsafe { std::mem::transmute(debug_metadata.inst_info) };
let raw_func_slice = vec![];//exec_context.get_local_function_pointers_and_lengths().expect("raw func slice");
dbg!("DEBUG INFO GENERATED");
let debug_image = wasm_debug::emit_debugsections_image(X86_64_OSX,
std::mem::size_of::<usize>() as u8,
&debug_info,
&extra_info,
&compiled_fn_map,
&range_map,
&raw_func_slice).expect("make debug image");
crate::jit_debug::register_new_jit_code_entry(&debug_image, crate::jit_debug::JITAction::JIT_REGISTER_FN);
}
Ok(ModuleInner {
cache_gen,
runnable_module: Arc::new(Box::new(exec_context)),

View File

@ -0,0 +1,92 @@
#![allow(missing_docs)]
use std::ptr;
// =============================================================================
// LLDB hook magic:
// see lldb/packages/Python/lldbsuite/test/functionalities/jitloader_gdb in
// llvm repo for example
//
// see also https://sourceware.org/gdb/current/onlinedocs/gdb.html#JIT-Interface
#[inline(never)]
pub extern "C" fn __jit_debug_register_code() {
}
#[allow(non_camel_case_types)]
#[derive(Debug)]
#[repr(u32)]
pub enum JITAction { JIT_NOACTION = 0, JIT_REGISTER_FN = 1, JIT_UNREGISTER_FN = 2 }
#[no_mangle]
#[repr(C)]
pub struct JITCodeEntry {
next: *mut JITCodeEntry,
prev: *mut JITCodeEntry,
// TODO: use CStr here?
symfile_addr: *const u8,
symfile_size: u64,
}
impl Default for JITCodeEntry {
fn default() -> Self {
Self {
next: ptr::null_mut(),
prev: ptr::null_mut(),
symfile_addr: ptr::null(),
symfile_size: 0,
}
}
}
#[no_mangle]
#[repr(C)]
pub struct JitDebugDescriptor {
version: u32,
action_flag: u32,
relevant_entry: *mut JITCodeEntry,
first_entry: *mut JITCodeEntry,
}
#[no_mangle]
#[allow(non_upper_case_globals)]
pub static mut __jit_debug_descriptor: JitDebugDescriptor = JitDebugDescriptor {
version: 1,
action_flag: JITAction::JIT_NOACTION as _,
relevant_entry: ptr::null_mut(),
first_entry: ptr::null_mut(),
};
/// Prepend an item to the front of the `__jit_debug_descriptor` entry list
///
/// # Safety
/// - Pointer to [`JITCodeEntry`] should point to a valid entry and stay alive
/// for the 'static lifetime
unsafe fn push_front(jce: *mut JITCodeEntry) {
if __jit_debug_descriptor.first_entry.is_null() {
__jit_debug_descriptor.first_entry = jce;
} else {
let old_first = __jit_debug_descriptor.first_entry;
debug_assert!((*old_first).prev.is_null());
(*jce).next = old_first;
(*old_first).prev = jce;
__jit_debug_descriptor.first_entry = jce;
}
}
// deleted static (added and deleted by Mark): TODO:
pub fn register_new_jit_code_entry(bytes: &[u8], action: JITAction) -> *mut JITCodeEntry {
let entry: *mut JITCodeEntry = Box::into_raw(Box::new(JITCodeEntry {
symfile_addr: bytes.as_ptr(),
symfile_size: bytes.len() as _,
..JITCodeEntry::default()
}));
unsafe {
push_front(entry);
__jit_debug_descriptor.relevant_entry = entry;
__jit_debug_descriptor.action_flag = action as u32;
}
entry
}

View File

@ -69,6 +69,7 @@ pub mod fault;
pub mod state;
#[cfg(feature = "managed")]
pub mod tiering;
pub mod jit_debug;
use self::error::CompileResult;
#[doc(inline)]

View File

@ -502,6 +502,17 @@ define_map_index![
| imported: ImportedFuncIndex, ImportedMemoryIndex, ImportedTableIndex, ImportedGlobalIndex,
];
// lol
impl cranelift_entity::EntityRef for FuncIndex {
fn index(self) -> usize {
self.0 as usize
}
fn new(x: usize) -> Self {
Self(x as u32)
}
}
#[rustfmt::skip]
macro_rules! define_local_or_import {
($ty:ident, $local_ty:ident, $imported_ty:ident, $imports:ident) => {

View File

@ -719,6 +719,7 @@ fn execute_wasm(options: &Run) -> Result<(), String> {
track_state,
features: options.features.into_backend_features(),
backend_specific_config,
generate_debug_info: true,
..Default::default()
},
&*compiler,