Change configuration API for easier integration (#5)

This commit is contained in:
folex
2020-06-30 15:57:48 +03:00
committed by GitHub
parent c907865444
commit 6784d2ed08
17 changed files with 215 additions and 126 deletions

48
Cargo.lock generated
View File

@ -2,9 +2,9 @@
# It is not intended for manual editing.
[[package]]
name = "addr2line"
version = "0.12.1"
version = "0.12.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a49806b9dadc843c61e7c97e72490ad7f7220ae249012fbda9ad0609457c0543"
checksum = "602d785912f476e480434627e8732e6766b760c045bbf897d9dfaa9f4fbd399c"
dependencies = [
"gimli 0.21.0",
]
@ -75,9 +75,9 @@ dependencies = [
[[package]]
name = "bincode"
version = "1.2.1"
version = "1.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5753e2a71534719bf3f4e57006c3a4f0d2c672a4b676eec84161f763eca87dbf"
checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d"
dependencies = [
"byteorder",
"serde",
@ -303,9 +303,9 @@ checksum = "bb1f6b1ce1c140482ea30ddd3335fc0024ac7ee112895426e0a629a6c20adfe3"
[[package]]
name = "erased-serde"
version = "0.3.11"
version = "0.3.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d88b6d1705e16a4d62e05ea61cc0496c2bd190f4fa8e5c1f11ce747be6bcf3d1"
checksum = "6ca8b296792113e1500fd935ae487be6e00ce318952a6880555554824d6ebf38"
dependencies = [
"serde",
]
@ -375,6 +375,7 @@ dependencies = [
"multimap",
"parity-wasm",
"pwasm-utils",
"serde",
"wasmer-interface-types",
"wasmer-runtime",
"wasmer-runtime-core",
@ -444,9 +445,9 @@ dependencies = [
[[package]]
name = "ghost"
version = "0.1.1"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2a36606a68532b5640dc86bb1f33c64b45c4682aad4c50f3937b317ea387f3d6"
checksum = "1a5bcf1bbeab73aa4cf2fde60a846858dc036163c7c33bec309f8d17de785479"
dependencies = [
"proc-macro2",
"quote",
@ -511,9 +512,9 @@ dependencies = [
[[package]]
name = "inventory"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "82d3f4b90287725c97b17478c60dda0c6324e7c84ee1ed72fb9179d0fdf13956"
checksum = "621b50c176968fd3b0bd71f821a28a0ea98db2b5aea966b2fbb8bd1b7d310328"
dependencies = [
"ctor",
"ghost",
@ -522,9 +523,9 @@ dependencies = [
[[package]]
name = "inventory-impl"
version = "0.1.6"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9092a4fefc9d503e9287ef137f03180a6e7d1b04c419563171ee14947c5e80ec"
checksum = "f99a4111304bade76468d05beab3487c226e4fe4c4de1c4e8f006e815762db73"
dependencies = [
"proc-macro2",
"quote",
@ -535,14 +536,15 @@ dependencies = [
name = "ipfs_node"
version = "0.1.0"
dependencies = [
"anyhow",
"fluence-faas",
]
[[package]]
name = "itoa"
version = "0.4.5"
version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8b7a7c0c47db5545ed3fef7468ee7bb5b74691498139e4b3f6a20685dc6dd8e"
checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6"
[[package]]
name = "lazy_static"
@ -837,9 +839,9 @@ checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"
[[package]]
name = "serde"
version = "1.0.112"
version = "1.0.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "736aac72d1eafe8e5962d1d1c3d99b0df526015ba40915cb3c49d042e92ec243"
checksum = "5317f7588f0a5078ee60ef675ef96735a1442132dc645eb1d12c018620ed8cd3"
dependencies = [
"serde_derive",
]
@ -865,9 +867,9 @@ dependencies = [
[[package]]
name = "serde_derive"
version = "1.0.112"
version = "1.0.114"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf0343ce212ac0d3d6afd9391ac8e9c9efe06b533c8d33f660f6390cc4093f57"
checksum = "2a0be94b04690fbaed37cddffc5c134bf537c8e3329d53e982fe04c374978f8e"
dependencies = [
"proc-macro2",
"quote",
@ -911,9 +913,9 @@ checksum = "2d67a5a62ba6e01cb2192ff309324cb4875d0c451d55fe2319433abe7a05a8ee"
[[package]]
name = "syn"
version = "1.0.31"
version = "1.0.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b5304cfdf27365b7585c25d4af91b35016ed21ef88f17ced89c7093b43dba8b6"
checksum = "e8d5d96e8cbb005d6959f119f773bfaebb5684296108fb32600c00cde305b2cd"
dependencies = [
"proc-macro2",
"quote",
@ -1030,9 +1032,9 @@ checksum = "caaa9d531767d1ff2150b9332433f32a24622147e5ebb1f26409d5da67afd479"
[[package]]
name = "unicode-xid"
version = "0.2.0"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "826e7639553986605ec5979c7dd957c7895e93eabed50ab2ffa7f6128a75097c"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
[[package]]
name = "vec_map"
@ -1146,7 +1148,7 @@ dependencies = [
[[package]]
name = "wasmer-interface-types"
version = "0.17.0"
source = "git+https://github.com/fluencelabs/interface-types#82986d10358f02c8ae64850ed0b79efb4e823c54"
source = "git+https://github.com/fluencelabs/interface-types#689c6757fb5713495586085e53d51ecb2e8759b7"
dependencies = [
"nom",
"serde",

View File

@ -11,8 +11,9 @@ wit_parser = { path = "../crates/wit_parser", version = "0.1.0" }
wasmer-runtime = { git = "https://github.com/fluencelabs/wasmer", branch = "fluence" }
# dynamicfunc-fat-closures allows using state inside DynamicFunc
wasmer-core = { package = "wasmer-runtime-core", git = "http://github.com/fluencelabs/wasmer", branch = "fluence", features = ["dynamicfunc-fat-closures"] }
wasmer-wit = { package = "wasmer-interface-types", git = "http://github.com/fluencelabs/interface-types", branch = "master" }
wasmer-wit = { package = "wasmer-interface-types", git = "http://github.com/fluencelabs/interface-types", branch = "master", features = ["serde"] }
wasmer-wasi = { git = "https://github.com/fluencelabs/wasmer", branch = "fluence" }
serde = { version = "1.0.114", default-features = false, features = [ "derive" ] }
multimap = "0.8.1"
parity-wasm = "0.41.0"

View File

@ -27,7 +27,7 @@ pub struct FCE {
}
/// Represent a function type inside FCE.
#[derive(Debug)]
#[derive(Debug, serde::Serialize)]
pub struct FCEFunction<'a> {
pub name: &'a str,
pub inputs: &'a Vec<IType>,

View File

@ -39,3 +39,4 @@ pub use engine::FCEFunction;
pub use errors::FCEError;
pub use module::IValue;
pub use module::IType;
pub use module::{to_interface_value, from_interface_values};

View File

@ -25,6 +25,7 @@ pub(crate) use fce_module::FCEModule;
pub use wasmer_wit::types::InterfaceType as IType;
pub use wasmer_wit::values::InterfaceValue as IValue;
pub use wasmer_wit::values::{to_interface_value, from_interface_values};
pub(self) use wasmer_core::types::Type as WType;
pub(self) use wasmer_core::types::Value as WValue;

View File

@ -6,3 +6,4 @@ edition = "2018"
[dependencies]
fluence-faas = { path = "../../fluence-faas" }
anyhow = "1.0.31"

View File

@ -1,3 +1,5 @@
core_modules_dir = "wasm/artifacts/wasm_modules"
[[core_module]]
name = "ipfs_node.wasm"
mem_pages_count = 100

View File

@ -18,35 +18,29 @@ use fluence_faas::FluenceFaaS;
use fluence_faas::IValue;
use std::path::PathBuf;
use anyhow::Context;
const IPFS_MODULES_DIR: &str = "wasm/artifacts/wasm_modules";
const IPFS_MODULES_CONFIG_PATH: &str = "Config.toml";
const IPFS_RPC: &str = "wasm/artifacts/wasm_ipfs_rpc_wit.wasi.wasm";
fn main() {
let ipfs_rpc = std::fs::read(IPFS_RPC).unwrap();
fn main() -> Result<(), anyhow::Error> {
let ipfs_rpc = std::fs::read(IPFS_RPC).context(format!("{} wasn't found", IPFS_RPC))?;
let mut ipfs_node = FluenceFaaS::new(
PathBuf::from(IPFS_MODULES_DIR),
PathBuf::from(IPFS_MODULES_CONFIG_PATH),
)
.unwrap();
let mut ipfs_node = FluenceFaaS::new(PathBuf::from(IPFS_MODULES_CONFIG_PATH))?;
println!("ipfs node interface is\n{}", ipfs_node.get_interface());
let node_address = ipfs_node
.call_module("ipfs_node.wasm", "get_address", &[])
.unwrap();
let node_address = ipfs_node.call_module("ipfs_node.wasm", "get_address", &[])?;
println!("ipfs node address is:\n{:?}", node_address);
let result = ipfs_node
.call_code(
let result = ipfs_node.call_code(
&ipfs_rpc,
"put",
&[IValue::String("Hello, world".to_string())],
)
.unwrap();
)?;
println!("execution result {:?}", result);
Ok(())
}

View File

@ -99,13 +99,13 @@ pub unsafe fn get_address() {
let ipfs_address = match std::env::var(IPFS_ADDR_ENV_NAME) {
Ok(addr) => addr,
Err(e) => format!("getting {} env variable failed with error {:?}", IPFS_ADDR_ENV_NAME, e)
Err(e) => format!(
"getting {} env variable failed with error {:?}",
IPFS_ADDR_ENV_NAME, e
),
};
let msg = format!(
"ipfs_node.get_address: node address is {} \n",
ipfs_address
);
let msg = format!("ipfs_node.get_address: node address is {} \n", ipfs_address);
log_utf8_string(msg.as_ptr() as _, msg.len() as _);
*RESULT_PTR.get_mut() = ipfs_address.as_ptr() as _;

View File

@ -36,7 +36,7 @@ pub unsafe fn invoke(_ptr: *mut u8, _size: usize) {
let msg = "ipfs_rpc.invoke: invoke called\n";
log_utf8_string(msg.as_ptr() as _, msg.len() as _);
let result = "IFPFS_RPC wasm example, it allows to:\ninvoke\nput\nget".to_string();
let result = "IPFS_RPC wasm example, it allows to:\ninvoke\nput\nget".to_string();
*RESULT_PTR.get_mut() = result.as_ptr() as _;
*RESULT_SIZE.get_mut() = result.len();

View File

@ -18,6 +18,3 @@ serde_json = "1.0.53"
serde_derive = "1.0.111"
cmd_lib = "0.7.8"
log = "0.4.8"
[patch.'https://github.com/fluencelabs/wasmer']
wasmer-wasi = { path = "lib/wasi" }

View File

@ -54,3 +54,15 @@ impl From<FCEError> for FaaSError {
FaaSError::EngineError(err)
}
}
impl From<toml::de::Error> for FaaSError {
fn from(err: toml::de::Error) -> Self {
FaaSError::ConfigParseError(format!("{}", err))
}
}
impl From<std::convert::Infallible> for FaaSError {
fn from(inf: std::convert::Infallible) -> Self {
match inf {}
}
}

View File

@ -14,17 +14,25 @@
* limitations under the License.
*/
use super::FaaSError;
use crate::misc::{CoreModulesConfig, make_fce_config};
use crate::RawCoreModulesConfig;
use crate::Result;
use super::faas_interface::FaaSInterface;
use super::faas_interface::FaaSModuleInterface;
use super::FaaSError;
use super::IValue;
use fce::FCE;
use super::IValue;
use fce::FCEModuleConfig;
use std::convert::TryInto;
use std::fs;
use std::path::PathBuf;
// TODO: remove and use mutex instead
unsafe impl Send for FluenceFaaS {}
pub struct FluenceFaaS {
fce: FCE,
@ -36,60 +44,88 @@ pub struct FluenceFaaS {
}
impl FluenceFaaS {
pub fn new<P: Into<PathBuf>>(
core_modules_dir: P,
config_file_path: P,
) -> Result<Self, FaaSError> {
/// Creates FaaS from config on filesystem.
pub fn new<P: Into<PathBuf>>(config_file_path: P) -> Result<Self> {
let config = crate::misc::load_config(config_file_path.into())?;
Self::with_raw_config(config)
}
/// Creates FaaS from config deserialized from TOML.
pub fn with_raw_config(config: RawCoreModulesConfig) -> Result<Self> {
let config = crate::misc::from_raw_config(config)?;
let modules = config
.core_modules_dir
.as_ref()
.map_or(Ok(vec![]), |dir| Self::load_modules(dir))?;
Self::with_modules(modules, config)
}
/// Creates FaaS with given modules.
pub fn with_modules<I, C>(modules: I, config: C) -> Result<Self>
where
I: IntoIterator<Item = (String, Vec<u8>)>,
C: TryInto<CoreModulesConfig>,
FaaSError: From<C::Error>,
{
let mut fce = FCE::new();
let mut module_names = Vec::new();
let mut core_modules_config = crate::misc::parse_config_from_file(config_file_path.into())?;
let mut config = config.try_into()?;
for entry in fs::read_dir(core_modules_dir.into())? {
let path = entry?.path();
if path.is_dir() {
// just skip directories
continue;
for (name, bytes) in modules {
let module_config = crate::misc::make_fce_config(config.modules_config.remove(&name))?;
fce.load_module(name.clone(), &bytes, module_config)?;
module_names.push(name);
}
let module_name = path.file_name().unwrap();
let module_name = module_name
.to_os_string()
.into_string()
.map_err(|e| FaaSError::IOError(format!("failed to read from {:?} file", e)))?;
let module_bytes = fs::read(path.clone())?;
let core_module_config = crate::misc::make_fce_config(
core_modules_config.modules_config.remove(&module_name),
)?;
fce.load_module(module_name.clone(), &module_bytes, core_module_config)?;
module_names.push(module_name);
}
let rpc_module_config =
crate::misc::make_fce_config(core_modules_config.rpc_module_config)?;
let faas_code_config = make_fce_config(config.rpc_module_config)?;
Ok(Self {
fce,
module_names,
faas_code_config: rpc_module_config,
faas_code_config,
})
}
/// Loads modules from a directory at a given path. Non-recursive, ignores subdirectories.
fn load_modules(core_modules_dir: &str) -> Result<Vec<(String, Vec<u8>)>> {
use FaaSError::IOError;
let mut dir_entries = fs::read_dir(core_modules_dir)
.map_err(|e| IOError(format!("{}: {}", core_modules_dir, e)))?;
dir_entries.try_fold(vec![], |mut vec, entry| {
let entry = entry?;
let path = entry.path();
if !path.is_dir() {
let module_name = path
.file_name()
.ok_or(IOError(format!("No file name in path {:?}", path)))?
.to_os_string()
.into_string()
.map_err(|name| IOError(format!("invalid file name: {:?}", name)))?;
let module_bytes = fs::read(path)?;
vec.push((module_name, module_bytes))
}
Ok(vec)
})
}
/// Executes provided Wasm code in the internal environment (with access to module exports).
pub fn call_code(
&mut self,
wasm_rpc: &[u8],
wasm: &[u8],
func_name: &str,
args: &[IValue],
) -> Result<Vec<IValue>, FaaSError> {
let rpc_module_name = "ipfs_rpc";
) -> Result<Vec<IValue>> {
// We need this because every wasm code loaded into VM needs a module name
let anonymous_module = "anonymous_module_name";
self.fce
.load_module(rpc_module_name, wasm_rpc, self.faas_code_config.clone())?;
.load_module(anonymous_module, wasm, self.faas_code_config.clone())?;
let call_result = self.fce.call(rpc_module_name, func_name, args)?;
self.fce.unload_module(rpc_module_name)?;
let call_result = self.fce.call(anonymous_module, func_name, args)?;
self.fce.unload_module(anonymous_module)?;
Ok(call_result)
}
@ -100,7 +136,7 @@ impl FluenceFaaS {
module_name: &str,
func_name: &str,
args: &[IValue],
) -> Result<Vec<IValue>, FaaSError> {
) -> Result<Vec<IValue>> {
self.fce
.call(module_name, func_name, args)
.map_err(Into::into)

View File

@ -15,13 +15,14 @@
*/
use std::fmt;
use serde::Serialize;
#[derive(Debug)]
#[derive(Debug, Serialize)]
pub struct FaaSInterface<'a> {
pub modules: Vec<FaaSModuleInterface<'a>>,
}
#[derive(Debug)]
#[derive(Debug, Serialize)]
pub struct FaaSModuleInterface<'a> {
pub name: &'a str,
pub functions: Vec<fce::FCEFunction<'a>>,

View File

@ -19,10 +19,15 @@ mod faas;
mod faas_interface;
mod misc;
pub(crate) type Result<T> = std::result::Result<T, FaaSError>;
pub use fce::IValue;
pub use fce::{to_interface_value, from_interface_values};
pub use fce::IType;
pub use errors::FaaSError;
pub use faas::FluenceFaaS;
pub use faas_interface::FaaSInterface;
pub use faas_interface::FaaSModuleInterface;
pub use misc::{RawCoreModulesConfig, RawModuleConfig};

View File

@ -15,15 +15,19 @@
*/
use crate::FaaSError;
use crate::Result;
use serde_derive::Deserialize;
use serde_derive::{Serialize, Deserialize};
use toml::from_slice;
use std::collections::HashMap;
use std::convert::TryInto;
/*
An example of the config:
core_modules_dir = "wasm/artifacts/wasm_modules"
[[core_module]]
name = "ipfs_node.wasm"
mem_pages_count = 100
@ -48,14 +52,23 @@ An example of the config:
mapped_dirs = { "tmp" = "/Users/user/tmp" }
*/
#[derive(Deserialize, Debug)]
pub(crate) struct RawCoreModulesConfig {
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct RawCoreModulesConfig {
pub core_modules_dir: Option<String>,
pub core_module: Vec<RawModuleConfig>,
pub rpc_module: Option<RawRPCModuleConfig>,
}
#[derive(Deserialize, Debug)]
pub(crate) struct RawModuleConfig {
impl TryInto<CoreModulesConfig> for RawCoreModulesConfig {
type Error = FaaSError;
fn try_into(self) -> Result<CoreModulesConfig> {
from_raw_config(self)
}
}
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct RawModuleConfig {
pub name: String,
pub mem_pages_count: Option<u32>,
pub logger_enabled: Option<bool>,
@ -63,63 +76,75 @@ pub(crate) struct RawModuleConfig {
pub wasi: Option<RawWASIConfig>,
}
#[derive(Deserialize, Debug)]
pub(crate) struct RawWASIConfig {
impl RawModuleConfig {
pub fn new(name: String) -> Self {
Self {
name,
mem_pages_count: None,
logger_enabled: None,
imports: None,
wasi: None,
}
}
}
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct RawWASIConfig {
pub envs: Option<Vec<String>>,
pub preopened_files: Option<Vec<String>>,
pub mapped_dirs: Option<toml::value::Table>,
}
#[derive(Deserialize, Debug)]
pub(crate) struct RawRPCModuleConfig {
#[derive(Deserialize, Serialize, Debug, Clone, Default)]
pub struct RawRPCModuleConfig {
pub mem_pages_count: Option<u32>,
pub logger_enabled: Option<bool>,
pub wasi: Option<RawWASIConfig>,
}
#[derive(Debug)]
pub(crate) struct NodeConfig {
#[derive(Debug, Clone, Default)]
pub struct CoreModulesConfig {
pub core_modules_dir: Option<String>,
pub modules_config: HashMap<String, ModuleConfig>,
pub rpc_module_config: Option<ModuleConfig>,
}
#[derive(Debug)]
pub(crate) struct ModuleConfig {
#[derive(Debug, Clone, Default)]
pub struct ModuleConfig {
pub mem_pages_count: Option<u32>,
pub logger_enabled: Option<bool>,
pub imports: Option<Vec<(String, String)>>,
pub wasi: Option<WASIConfig>,
}
#[derive(Debug)]
pub(crate) struct WASIConfig {
#[derive(Debug, Clone, Default)]
pub struct WASIConfig {
pub envs: Option<Vec<Vec<u8>>>,
pub preopened_files: Option<Vec<String>>,
pub mapped_dirs: Option<Vec<(String, String)>>,
}
pub(crate) fn parse_config_from_file(
config_file_path: std::path::PathBuf,
) -> Result<NodeConfig, FaaSError> {
let file_content = std::fs::read(config_file_path)?;
let config: RawCoreModulesConfig =
from_slice(&file_content).map_err(|err| FaaSError::ConfigParseError(format!("{}", err)))?;
/// Prepare config after parsing it from TOML
pub(crate) fn from_raw_config(config: RawCoreModulesConfig) -> Result<CoreModulesConfig> {
let modules_config = config
.core_module
.into_iter()
.map(|module| {
let imports: Option<Vec<(String, String)>> = module.imports.map(|import| {
import
let imports = module
.imports
.map(|import| {
Ok(import
.into_iter()
.map(|(import_func_name, host_cmd)| {
(import_func_name, host_cmd.try_into::<String>().unwrap())
let host_cmd = host_cmd.try_into::<String>()?;
Ok((import_func_name, host_cmd))
})
.collect::<Vec<_>>()
});
.collect::<Result<Vec<_>>>()?) as Result<_>
})
.transpose()?;
let wasi = module.wasi.map(parse_raw_wasi);
(
Ok((
module.name,
ModuleConfig {
mem_pages_count: module.mem_pages_count,
@ -127,9 +152,9 @@ pub(crate) fn parse_config_from_file(
imports,
wasi,
},
)
))
})
.collect::<HashMap<_, _>>();
.collect::<Result<HashMap<_, _>>>()?;
let rpc_module_config = config.rpc_module.map(|rpc_module| {
let wasi = rpc_module.wasi.map(parse_raw_wasi);
@ -142,12 +167,19 @@ pub(crate) fn parse_config_from_file(
}
});
Ok(NodeConfig {
Ok(CoreModulesConfig {
core_modules_dir: config.core_modules_dir,
modules_config,
rpc_module_config,
})
}
/// Parse config from TOML
pub(crate) fn load_config(config_file_path: std::path::PathBuf) -> Result<RawCoreModulesConfig> {
let file_content = std::fs::read(config_file_path)?;
Ok(from_slice(&file_content)?)
}
fn parse_raw_wasi(wasi: RawWASIConfig) -> WASIConfig {
let envs = wasi
.envs

View File

@ -3,4 +3,8 @@ mod utils;
mod config;
pub(crate) use utils::make_fce_config;
pub(crate) use config::parse_config_from_file;
pub(crate) use config::load_config;
pub(crate) use config::from_raw_config;
pub(crate) use config::CoreModulesConfig;
pub use config::{RawCoreModulesConfig, RawModuleConfig};