Rework memory access interface to support non-wasmer environments (#14)

This commit is contained in:
Valery Antopol
2022-01-28 22:34:17 +07:00
committed by GitHub
parent 94023f668f
commit 0c5145aa37
31 changed files with 524 additions and 413 deletions

View File

@ -15,18 +15,15 @@
*/
use crate::traits::RecordResolvableError;
use it_memory_traits::MemoryAccessError;
use thiserror::Error as ThisError;
#[derive(Debug, ThisError)]
pub enum LiError {
#[error(
"Out-of-bound Wasm memory access: offset {offset}, size {size}, while memory_size {memory_size}"
)]
InvalidAccess {
offset: usize,
size: usize,
memory_size: usize,
},
#[error("{0}")]
MemoryAccessError(#[from] MemoryAccessError),
#[error("{0}")]
RecordResolvableError(#[from] RecordResolvableError),

View File

@ -22,8 +22,10 @@ use crate::utils::ser_type_size;
use crate::IType;
use crate::IValue;
pub fn array_lift_memory<R: RecordResolvable>(
lifter: &ILifter<'_, '_, R>,
use it_memory_traits::{SequentialMemoryView, SequentialReader};
pub fn array_lift_memory<R: RecordResolvable, MV: for<'a> SequentialMemoryView<'a>>(
lifter: &ILifter<'_, R, MV>,
value_type: &IType,
offset: usize,
elements_count: usize,
@ -59,8 +61,8 @@ pub fn array_lift_memory<R: RecordResolvable>(
Ok(IValue::Array(ivalues))
}
fn read_string_array<R: RecordResolvable>(
lifter: &ILifter<'_, '_, R>,
fn read_string_array<R: RecordResolvable, MV: for<'a> SequentialMemoryView<'a>>(
lifter: &ILifter<'_, R, MV>,
offset: usize,
elements_count: usize,
) -> LiResult<Vec<IValue>> {
@ -81,8 +83,8 @@ fn read_string_array<R: RecordResolvable>(
Ok(result)
}
fn read_array_array<R: RecordResolvable>(
lifter: &ILifter<'_, '_, R>,
fn read_array_array<R: RecordResolvable, MV: for<'a> SequentialMemoryView<'a>>(
lifter: &ILifter<'_, R, MV>,
ty: &IType,
offset: usize,
elements_count: usize,
@ -103,8 +105,8 @@ fn read_array_array<R: RecordResolvable>(
Ok(result)
}
fn read_record_array<R: RecordResolvable>(
lifter: &ILifter<'_, '_, R>,
fn read_record_array<R: RecordResolvable, MV: for<'a> SequentialMemoryView<'a>>(
lifter: &ILifter<'_, R, MV>,
record_type_id: u64,
offset: usize,
elements_count: usize,

View File

@ -18,7 +18,6 @@ use super::ILifter;
use super::LiError;
use super::LiResult;
use super::MemoryReader;
use super::SequentialReader;
use crate::traits::RecordResolvable;
use crate::utils::record_size;
use crate::IRecordType;
@ -26,8 +25,10 @@ use crate::IType;
use crate::IValue;
use crate::NEVec;
pub fn record_lift_memory<R: RecordResolvable>(
lifter: &ILifter<'_, '_, R>,
use it_memory_traits::{SequentialMemoryView, SequentialReader};
pub fn record_lift_memory<R: RecordResolvable, MV: for<'a> SequentialMemoryView<'a>>(
lifter: &ILifter<'_, R, MV>,
record_type: &IRecordType,
offset: usize,
) -> LiResult<IValue> {
@ -67,9 +68,9 @@ pub fn record_lift_memory<R: RecordResolvable>(
Ok(IValue::Record(record))
}
fn read_string(
reader: &MemoryReader<'_>,
seq_reader: &SequentialReader<'_, '_>,
fn read_string<MV: for<'a> SequentialMemoryView<'a>>(
reader: &MemoryReader<MV>,
seq_reader: &<MV as SequentialMemoryView<'_>>::SR,
) -> LiResult<String> {
let offset = seq_reader.read_u32();
let size = seq_reader.read_u32();
@ -80,9 +81,9 @@ fn read_string(
Ok(string)
}
fn read_byte_array(
reader: &MemoryReader<'_>,
seq_reader: &SequentialReader<'_, '_>,
fn read_byte_array<MV: for<'a> SequentialMemoryView<'a>>(
reader: &MemoryReader<MV>,
seq_reader: &<MV as SequentialMemoryView<'_>>::SR,
) -> LiResult<IValue> {
let offset = seq_reader.read_u32();
let size = seq_reader.read_u32();
@ -92,9 +93,9 @@ fn read_byte_array(
Ok(IValue::ByteArray(array))
}
fn read_array<R: RecordResolvable>(
lifter: &ILifter<'_, '_, R>,
seq_reader: &SequentialReader<'_, '_>,
fn read_array<R: RecordResolvable, MV: for<'a> SequentialMemoryView<'a>>(
lifter: &ILifter<'_, R, MV>,
seq_reader: &<MV as SequentialMemoryView<'_>>::SR,
value_type: &IType,
) -> LiResult<IValue> {
let offset = seq_reader.read_u32();
@ -103,9 +104,9 @@ fn read_array<R: RecordResolvable>(
super::array_lift_memory(lifter, value_type, offset as _, size as _)
}
fn read_record<R: RecordResolvable>(
lifter: &ILifter<'_, '_, R>,
seq_reader: &SequentialReader<'_, '_>,
fn read_record<R: RecordResolvable, MV: for<'a> SequentialMemoryView<'a>>(
lifter: &ILifter<'_, R, MV>,
seq_reader: &<MV as SequentialMemoryView<'_>>::SR,
record_type_id: u64,
) -> LiResult<IValue> {
let offset = seq_reader.read_u32();

View File

@ -17,7 +17,7 @@
#[macro_export]
macro_rules! value_der {
($self:expr, $offset:expr, @seq_start $($ids:tt),* @seq_end) => {
[$($self.reader.memory[$offset + $ids].get()),+]
[$($self.memory.get($offset + $ids)),+]
};
($self:expr, $offset:expr, 1) => {
@ -44,7 +44,7 @@ macro_rules! value_der {
#[macro_export]
macro_rules! read_ty {
($func_name:ident, $ty:ty, 1) => {
pub fn $func_name(&self) -> $ty {
fn $func_name(&self) -> $ty {
let offset = self.offset.get();
let result = <$ty>::from_le_bytes(crate::value_der!(self, offset, 1));
@ -54,7 +54,7 @@ macro_rules! read_ty {
};
($func_name:ident, $ty:ty, 2) => {
pub fn $func_name(&self) -> $ty {
fn $func_name(&self) -> $ty {
let offset = self.offset.get();
let result = <$ty>::from_le_bytes(crate::value_der!(self, offset, 2));
@ -64,7 +64,7 @@ macro_rules! read_ty {
};
($func_name:ident, $ty:ty, 4) => {
pub fn $func_name(&self) -> $ty {
fn $func_name(&self) -> $ty {
let offset = self.offset.get();
let result = <$ty>::from_le_bytes(crate::value_der!(self, offset, 4));
@ -74,7 +74,7 @@ macro_rules! read_ty {
};
($func_name:ident, $ty:ty, 8) => {
pub fn $func_name(&self) -> $ty {
fn $func_name(&self) -> $ty {
let offset = self.offset.get();
let result = <$ty>::from_le_bytes(crate::value_der!(self, offset, 8));
@ -84,7 +84,7 @@ macro_rules! read_ty {
};
($func_name:ident, $ty:ty, 16) => {
pub fn $func_name(&self) -> $ty {
fn $func_name(&self) -> $ty {
let offset = self.offset.get();
let result = <$ty>::from_le_bytes(crate::value_der!(self, offset, 16));
@ -94,6 +94,29 @@ macro_rules! read_ty {
};
}
#[macro_export]
macro_rules! read_ty_decl {
($func_name:ident, $ty:ty, 1) => {
fn $func_name(&self) -> $ty;
};
($func_name:ident, $ty:ty, 2) => {
fn $func_name(&self) -> $ty;
};
($func_name:ident, $ty:ty, 4) => {
fn $func_name(&self) -> $ty;
};
($func_name:ident, $ty:ty, 8) => {
fn $func_name(&self) -> $ty;
};
($func_name:ident, $ty:ty, 16) => {
fn $func_name(&self) -> $ty;
};
}
#[macro_export]
macro_rules! read_array_ty {
($func_name:ident, $ty:ident, $ity:ident) => {

View File

@ -14,29 +14,19 @@
* limitations under the License.
*/
use super::LiError;
use super::LiResult;
use crate::read_array_ty;
use crate::read_ty;
use crate::IValue;
use std::cell::Cell;
use it_memory_traits::{SequentialMemoryView, SequentialReader};
pub struct MemoryReader<'m> {
pub(self) memory: &'m [Cell<u8>],
pub struct MemoryReader<MV> {
pub(self) view: MV,
}
/// Reads values of basic types sequentially from the provided reader.
/// It doesn't check memory limits for the optimization purposes,
/// so it could be created only by the MemoryReader::sequential_reader method.
pub struct SequentialReader<'r, 'm> {
reader: &'r MemoryReader<'m>,
offset: Cell<usize>,
}
impl<'m> MemoryReader<'m> {
pub fn new(memory: &'m [Cell<u8>]) -> Self {
Self { memory }
impl<MV: for<'a> SequentialMemoryView<'a>> MemoryReader<MV> {
pub fn new(view: MV) -> Self {
Self { view }
}
/// Returns reader that allows read sequentially. It's important that memory limit is checked
@ -46,10 +36,9 @@ impl<'m> MemoryReader<'m> {
&self,
offset: usize,
size: usize,
) -> LiResult<SequentialReader<'_, '_>> {
self.check_access(offset, size)?;
Ok(SequentialReader::new(&self, offset))
) -> LiResult<<MV as SequentialMemoryView<'_>>::SR> {
let seq_reader = self.view.sequential_reader(offset, size)?;
Ok(seq_reader)
}
pub fn read_raw_u8_array(&self, offset: usize, elements_count: usize) -> LiResult<Vec<u8>> {
@ -76,21 +65,6 @@ impl<'m> MemoryReader<'m> {
Ok(result)
}
pub fn check_access(&self, offset: usize, size: usize) -> LiResult<()> {
let right = offset + size;
// the first condition is a check for overflow
if right < offset || right >= self.memory.len() {
return Err(LiError::InvalidAccess {
offset,
size,
memory_size: self.memory.len(),
});
}
Ok(())
}
read_array_ty!(read_u8_array, u8, U8);
read_array_ty!(read_s8_array, i8, S8);
read_array_ty!(read_u16_array, u16, U16);
@ -104,29 +78,3 @@ impl<'m> MemoryReader<'m> {
read_array_ty!(read_i64_array, i64, I64);
read_array_ty!(read_f64_array, f64, F64);
}
impl<'r, 'm> SequentialReader<'r, 'm> {
pub(self) fn new(reader: &'r MemoryReader<'m>, offset: usize) -> Self {
let offset = Cell::new(offset);
Self { reader, offset }
}
pub fn read_bool(&self) -> bool {
let offset = self.offset.get();
let result = self.reader.memory[offset].get() != 0;
self.offset.set(offset + 1);
result
}
read_ty!(read_u8, u8, 1);
read_ty!(read_i8, i8, 1);
read_ty!(read_u16, u16, 2);
read_ty!(read_i16, i16, 2);
read_ty!(read_u32, u32, 4);
read_ty!(read_i32, i32, 4);
read_ty!(read_f32, f32, 4);
read_ty!(read_u64, u64, 8);
read_ty!(read_i64, i64, 8);
read_ty!(read_f64, f64, 8);
}

View File

@ -24,22 +24,21 @@ pub use error::LiError;
pub use lift_array::array_lift_memory;
pub use lift_record::record_lift_memory;
pub use memory_reader::MemoryReader;
pub use memory_reader::SequentialReader;
use super::traits::RecordResolvable;
use std::cell::Cell;
pub use it_memory_traits::SequentialMemoryView;
pub type LiResult<T> = std::result::Result<T, error::LiError>;
pub struct ILifter<'m, 'r, R: RecordResolvable> {
pub reader: MemoryReader<'m>,
pub struct ILifter<'r, R: RecordResolvable, MV> {
pub reader: MemoryReader<MV>,
pub resolver: &'r R,
}
impl<'m, 'r, R: RecordResolvable> ILifter<'m, 'r, R> {
pub fn new(memory: &'m [Cell<u8>], resolver: &'r R) -> Self {
let reader = MemoryReader::new(memory);
impl<'r, R: RecordResolvable, MV: for<'a> SequentialMemoryView<'a>> ILifter<'r, R, MV> {
pub fn new(view: MV, resolver: &'r R) -> Self {
let reader = MemoryReader::new(view);
Self { reader, resolver }
}
}

View File

@ -16,6 +16,7 @@
use crate::traits::AllocatableError;
use crate::traits::RecordResolvableError;
use it_memory_traits::MemoryAccessError;
use thiserror::Error as ThisError;
#[derive(Debug, ThisError)]
@ -25,4 +26,7 @@ pub enum LoError {
#[error("{0}")]
RecordResolvableError(#[from] RecordResolvableError),
#[error("{0}")]
MemoryAccessError(#[from] MemoryAccessError),
}

View File

@ -21,6 +21,8 @@ use crate::utils::ser_value_size;
use crate::utils::type_tag_form_ivalue;
use crate::IValue;
use it_memory_traits::{SequentialMemoryView, SequentialWriter};
pub struct LoweredArray {
pub offset: usize,
pub size: usize,
@ -36,8 +38,8 @@ impl LoweredArray {
}
}
pub fn array_lower_memory<A: Allocatable>(
lowerer: &ILowerer<'_, A>,
pub fn array_lower_memory<A: Allocatable, MV: for<'a> SequentialMemoryView<'a>>(
lowerer: &ILowerer<'_, A, MV>,
array_values: Vec<IValue>,
) -> LoResult<LoweredArray> {
if array_values.is_empty() {
@ -52,40 +54,40 @@ pub fn array_lower_memory<A: Allocatable>(
// here it's known that all interface values have the same type
for value in array_values {
match value {
IValue::Boolean(value) => seq_writer.write_u8(&lowerer.writer, value as _),
IValue::S8(value) => seq_writer.write_u8(&lowerer.writer, value as _),
IValue::S16(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::S32(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::S64(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::U8(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::U16(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::U32(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::U64(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::I32(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::I64(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::F32(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::F64(value) => seq_writer.write_array(&lowerer.writer, value.to_le_bytes()),
IValue::Boolean(value) => seq_writer.write_u8(value as _),
IValue::S8(value) => seq_writer.write_u8(value as _),
IValue::S16(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::S32(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::S64(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::U8(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::U16(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::U32(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::U64(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::I32(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::I64(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::F32(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::F64(value) => seq_writer.write_bytes(&value.to_le_bytes()),
IValue::String(value) => {
let offset = lowerer.writer.write_bytes(value.as_bytes())? as u32;
let offset = lowerer.writer.write_bytes(&value.as_bytes())? as u32;
seq_writer.write_array(&lowerer.writer, offset.to_le_bytes());
seq_writer.write_array(&lowerer.writer, (value.len() as u32).to_le_bytes());
seq_writer.write_bytes(&offset.to_le_bytes());
seq_writer.write_bytes(&(value.len() as u32).to_le_bytes());
}
IValue::ByteArray(values) => {
let offset = lowerer.writer.write_bytes(&values)? as u32;
seq_writer.write_array(&lowerer.writer, offset.to_le_bytes());
seq_writer.write_array(&lowerer.writer, (values.len() as u32).to_le_bytes());
seq_writer.write_bytes(&offset.to_le_bytes());
seq_writer.write_bytes(&(values.len() as u32).to_le_bytes());
}
IValue::Array(values) => {
let LoweredArray { offset, size } = array_lower_memory(lowerer, values)?;
seq_writer.write_array(&lowerer.writer, (offset as u32).to_le_bytes());
seq_writer.write_array(&lowerer.writer, (size as u32).to_le_bytes());
seq_writer.write_bytes(&(offset as u32).to_le_bytes());
seq_writer.write_bytes(&(size as u32).to_le_bytes());
}
IValue::Record(values) => {
let offset = super::record_lower_memory(lowerer, values)? as u32;
seq_writer.write_array(&lowerer.writer, offset.to_le_bytes());
seq_writer.write_bytes(&offset.to_le_bytes());
}
}
}

View File

@ -21,8 +21,10 @@ use crate::traits::Allocatable;
use crate::IValue;
use crate::NEVec;
pub fn record_lower_memory<A: Allocatable>(
lowerer: &ILowerer<'_, A>,
use it_memory_traits::SequentialMemoryView;
pub fn record_lower_memory<A: Allocatable, MV: for<'a> SequentialMemoryView<'a>>(
lowerer: &ILowerer<'_, A, MV>,
values: NEVec<IValue>,
) -> LoResult<i32> {
let average_field_size = 4;

View File

@ -16,113 +16,36 @@
use super::LoResult;
use crate::traits::Allocatable;
use crate::traits::MemSlice;
use crate::traits::DEFAULT_MEMORY_INDEX;
use crate::utils::type_tag_form_itype;
use std::cell::Cell;
use it_memory_traits::{SequentialMemoryView, SequentialWriter};
pub struct MemoryWriter<'i, R: Allocatable> {
pub struct MemoryWriter<'i, R: Allocatable, MV> {
heap_manager: &'i R,
pub(self) memory: Cell<MemSlice<'i>>,
view: MV,
}
pub struct SequentialWriter {
start_offset: usize,
offset: Cell<usize>,
}
impl<'i, A: Allocatable> MemoryWriter<'i, A> {
pub fn new(heap_manager: &'i A) -> LoResult<Self> {
let mem_slice = heap_manager.memory_slice(DEFAULT_MEMORY_INDEX)?;
let memory = Cell::new(mem_slice);
let writer = Self {
heap_manager,
memory,
};
impl<'i, A: Allocatable, MV: for<'a> SequentialMemoryView<'a>> MemoryWriter<'i, A, MV> {
pub fn new(view: MV, heap_manager: &'i A) -> LoResult<Self> {
let writer = Self { heap_manager, view };
Ok(writer)
}
pub fn write_bytes(&self, bytes: &[u8]) -> LoResult<usize> {
let byte_type_tag = type_tag_form_itype(&crate::IType::U8);
let seq_writer = self.sequential_writer(bytes.len() as _, byte_type_tag)?;
seq_writer.write_bytes(self, bytes);
seq_writer.write_bytes(bytes);
Ok(seq_writer.start_offset())
}
pub fn sequential_writer(&self, size: u32, type_tag: u32) -> LoResult<SequentialWriter> {
let offset = self.heap_manager.allocate(size, type_tag)?;
let new_mem_slice = self.heap_manager.memory_slice(DEFAULT_MEMORY_INDEX)?;
self.memory.set(new_mem_slice);
Ok(SequentialWriter::new(offset))
}
}
impl SequentialWriter {
pub(self) fn new(offset: usize) -> Self {
Self {
offset: Cell::new(offset),
start_offset: offset,
}
}
pub fn start_offset(&self) -> usize {
self.start_offset
}
pub fn write_array<A: Allocatable, const N: usize>(
pub fn sequential_writer(
&self,
writer: &MemoryWriter<'_, A>,
values: [u8; N],
) {
let offset = self.offset.get();
writer.memory.get()[offset..offset + N]
.iter()
.zip(values.iter())
.for_each(|(cell, &byte)| cell.set(byte));
self.offset.set(offset + N);
}
// specialization of write_array for u8
pub fn write_u8<A: Allocatable>(&self, writer: &MemoryWriter<'_, A>, value: u8) {
let offset = self.offset.get();
writer.memory.get()[offset].set(value);
self.offset.set(offset + 1);
}
// specialization of write_array for u32
pub fn write_u32<A: Allocatable>(&self, writer: &MemoryWriter<'_, A>, value: u32) {
let offset = self.offset.get();
let value = value.to_le_bytes();
let memory = writer.memory.get();
memory[offset].set(value[0]);
memory[offset + 1].set(value[1]);
memory[offset + 2].set(value[2]);
memory[offset + 3].set(value[3]);
self.offset.set(offset + 4);
}
#[allow(dead_code)]
pub fn write_bytes<A: Allocatable>(&self, writer: &MemoryWriter<'_, A>, bytes: &[u8]) {
let offset = self.offset.get();
let memory = writer.memory.get();
memory[offset..offset + bytes.len()]
.iter()
.zip(bytes)
.for_each(|(cell, &byte)| cell.set(byte));
self.offset.set(offset + bytes.len());
size: u32,
type_tag: u32,
) -> LoResult<<MV as SequentialMemoryView<'_>>::SW> {
let offset = self.heap_manager.allocate(size, type_tag)?;
let seq_writer = self.view.sequential_writer(offset, size as usize)?;
Ok(seq_writer)
}
}

View File

@ -27,15 +27,17 @@ pub use lower_array::array_lower_memory;
pub use lower_array::LoweredArray;
pub use lower_record::record_lower_memory;
pub use it_memory_traits::SequentialMemoryView;
pub type LoResult<T> = std::result::Result<T, error::LoError>;
pub struct ILowerer<'m, A: Allocatable> {
pub writer: MemoryWriter<'m, A>,
pub struct ILowerer<'m, A: Allocatable, MV> {
pub writer: MemoryWriter<'m, A, MV>,
}
impl<'m, A: Allocatable> ILowerer<'m, A> {
pub fn new(allocatable: &'m A) -> LoResult<Self> {
let writer = MemoryWriter::new(allocatable)?;
impl<'m, A: Allocatable, MV: for<'a> SequentialMemoryView<'a>> ILowerer<'m, A, MV> {
pub fn new(view: MV, allocatable: &'m A) -> LoResult<Self> {
let writer = MemoryWriter::new(view, allocatable)?;
let lowerer = Self { writer };
Ok(lowerer)

View File

@ -14,17 +14,12 @@
* limitations under the License.
*/
use std::cell::Cell;
use thiserror::Error as ThisError;
pub const DEFAULT_MEMORY_INDEX: usize = 0;
pub type MemSlice<'m> = &'m [Cell<u8>];
pub trait Allocatable {
fn allocate(&self, size: u32, type_tag: u32) -> Result<usize, AllocatableError>;
fn memory_slice(&self, memory_index: usize) -> Result<MemSlice<'_>, AllocatableError>;
}
#[derive(Debug, ThisError)]

View File

@ -26,4 +26,7 @@ pub enum RecordResolvableError {
/// Record for such type is wasn't found.
#[error("Record with type id '{0}' not found")]
RecordNotFound(u64),
#[error("Memory with index '{memory_index}' not found")]
MemoryIsMissing { memory_index: usize },
}