diff --git a/url-downloader/README.md b/url-downloader/README.md new file mode 100644 index 0000000..12d1b45 --- /dev/null +++ b/url-downloader/README.md @@ -0,0 +1,13 @@ +# Download file to disk + +Example to show how to work with disk + link several .wasm modules into a service. + +# Build & deploy it +```shell +./deploy.sh +``` + +# Call it +```shell +fldist run_air -p download.air -d '{"service": "f7db6966-2d75-4424-812e-85e2ec5cb61b"}' +``` diff --git a/url-downloader/artifacts/curl_adapter.json b/url-downloader/artifacts/curl_adapter.json new file mode 100644 index 0000000..9de081f --- /dev/null +++ b/url-downloader/artifacts/curl_adapter.json @@ -0,0 +1,7 @@ +{ + "name": "curl_adapter", + "mountedBinaries": + { + "curl": "/usr/bin/curl" + } +} diff --git a/url-downloader/artifacts/facade.json b/url-downloader/artifacts/facade.json new file mode 100644 index 0000000..180ed7e --- /dev/null +++ b/url-downloader/artifacts/facade.json @@ -0,0 +1 @@ +{"name": "facade"} diff --git a/url-downloader/artifacts/local_storage.json b/url-downloader/artifacts/local_storage.json new file mode 100644 index 0000000..c4c3ad2 --- /dev/null +++ b/url-downloader/artifacts/local_storage.json @@ -0,0 +1,12 @@ +{ + "name": "local_storage", + "preopenedFiles": [ + "/tmp" + ], + "mappedDirs": { + "sites": "/tmp" + }, + "mountedBinaries": { + "curl": "/usr/bin/curl" + } +} diff --git a/url-downloader/build.sh b/url-downloader/build.sh index bacc9ad..fba8560 100755 --- a/url-downloader/build.sh +++ b/url-downloader/build.sh @@ -1,18 +1,24 @@ -#!/bin/sh +#!/bin/sh -euo pipefail # This script builds all subprojects and puts all created Wasm modules in one dir -cd local_storage -cargo update -fce build --release -cd ../curl_adapter -cargo update -fce build --release -cd ../facade -cargo update -fce build --release +( + cd local_storage + cargo update + fce build --release +) +( + cd curl_adapter + cargo update + fce build --release +) +( + cd facade + cargo update + fce build --release +) -cd .. -rm -f artifacts/* -cp ../../target/wasm32-wasi/release/local_storage.wasm artifacts/ -cp ../../target/wasm32-wasi/release/curl_adapter.wasm artifacts/ -cp ../../target/wasm32-wasi/release/facade.wasm artifacts/ +mkdir -p artifacts +rm -f artifacts/*.wasm +cp local_storage/target/wasm32-wasi/release/local_storage.wasm artifacts/ +cp curl_adapter/target/wasm32-wasi/release/curl_adapter.wasm artifacts/ +cp facade/target/wasm32-wasi/release/facade.wasm artifacts/ diff --git a/url-downloader/compose.air b/url-downloader/compose.air new file mode 100644 index 0000000..b568a53 --- /dev/null +++ b/url-downloader/compose.air @@ -0,0 +1,13 @@ +(xor + (seq + (seq + (seq + (call relay (curl "download") ["https://fluence.network/img/svg/logo_new.svg"] contents) + (call relay (storage "put") ["logo.svg" contents.$.stdout!] ret_code) + ) + (call relay (storage "get") ["logo.svg"] bytes) + ) + (call %init_peer_id% (returnService "run") [ret_code bytes]) + ) + (call %init_peer_id% (returnService "run") [%last_error%]) +) diff --git a/url-downloader/curl_adapter/src/main.rs b/url-downloader/curl_adapter/src/main.rs index c3e6acc..5903153 100644 --- a/url-downloader/curl_adapter/src/main.rs +++ b/url-downloader/curl_adapter/src/main.rs @@ -17,12 +17,10 @@ #![allow(improper_ctypes)] use fluence::fce; -use fluence::module_manifest; use fluence::WasmLoggerBuilder; -use fluence::MountedBinaryResult; - -module_manifest!(); +use fluence::MountedBinaryResult as Result; +use fluence::MountedBinaryStringResult as StringResult; /// Log level can be changed by `RUST_LOG` env as well. pub fn main() { @@ -30,16 +28,21 @@ pub fn main() { } #[fce] -pub fn download(url: String) -> String { - log::info!("get called with url {}", url); - - let result = unsafe { curl(vec![url]) }; - String::from_utf8(result.stdout).unwrap() +pub fn request(url: String) -> StringResult { + unsafe { curl(vec![url]) }.stringify().unwrap() } +#[fce] +pub fn download(url: String) -> Result { + log::info!("download called with url {}", url); + + unsafe { curl(vec![url]) } +} + + /// Permissions in `Config.toml` should exist to use host functions. #[fce] #[link(wasm_import_module = "host")] extern "C" { - fn curl(cmd: Vec) -> MountedBinaryResult; + fn curl(cmd: Vec) -> Result; } diff --git a/url-downloader/deploy.sh b/url-downloader/deploy.sh new file mode 100755 index 0000000..9b42425 --- /dev/null +++ b/url-downloader/deploy.sh @@ -0,0 +1,12 @@ +#!/bin/sh -euo pipefail + +# build wasms +./build.sh + +( + cd artifacts + fldist new_service --name "url_downloader" --modules \ + curl_adapter.wasm:curl_adapter.json \ + local_storage.wasm:local_storage.json \ + facade.wasm:facade.json +) diff --git a/url-downloader/deploy_separate.sh b/url-downloader/deploy_separate.sh new file mode 100755 index 0000000..9753639 --- /dev/null +++ b/url-downloader/deploy_separate.sh @@ -0,0 +1,15 @@ +#!/bin/sh -euo pipefail + +./build.sh + +echo "Deploying storage" +( + cd artifacts + fldist new_service --name "local_storage" --modules local_storage.wasm:local_storage.json +) + +echo "\n\nDeploying curl" +( + cd artifacts + fldist new_service --name "curl_adapter" --modules curl_adapter.wasm:curl_adapter.json +) diff --git a/url-downloader/download.air b/url-downloader/download.air new file mode 100644 index 0000000..d8a1334 --- /dev/null +++ b/url-downloader/download.air @@ -0,0 +1,10 @@ +(xor + (seq + (seq + (call relay (service "get_n_save") ["https://fluence.network/img/svg/logo_new.svg" "logo.svg"] ret_code) + (call relay (service "load_file") ["logo.svg"] bytes) + ) + (call %init_peer_id% (returnService "run") [ret_code bytes]) + ) + (call %init_peer_id% (returnService "run") [%last_error%]) +) diff --git a/url-downloader/facade/Cargo.lock b/url-downloader/facade/Cargo.lock new file mode 100644 index 0000000..d98a1f5 --- /dev/null +++ b/url-downloader/facade/Cargo.lock @@ -0,0 +1,192 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "anyhow" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" + +[[package]] +name = "base64" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "facade" +version = "0.1.0" +dependencies = [ + "anyhow", + "base64", + "fluence", + "log", +] + +[[package]] +name = "fluence" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "342831732a977f13220ecd6f25c7e6b6127af8d3794d42e3c880b05e457a0484" +dependencies = [ + "fluence-sdk-macro", + "fluence-sdk-main", +] + +[[package]] +name = "fluence-sdk-macro" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00c840552e5e58b62d4a272e3e4aabd548e1a4519df629a51d42157c7ecbe653" +dependencies = [ + "fluence-sdk-wit", +] + +[[package]] +name = "fluence-sdk-main" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f1d126f05a798c8f812fbe47a42145478ce8029e00411b04d1c5194ab368313" +dependencies = [ + "fluence-sdk-macro", + "log", + "serde", +] + +[[package]] +name = "fluence-sdk-wit" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ac71e8d48a8e2bdaccf59572dc7a8e8a1ad0de512fc452107756e37ac0bf752" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_json", + "syn", + "uuid", +] + +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "itoa" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" + +[[package]] +name = "libc" +version = "0.2.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "538c092e5586f4cdd7dd8078c4a79220e3e168880218124dcbce860f0ea938c6" + +[[package]] +name = "log" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "proc-macro2" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +dependencies = [ + "unicode-xid", +] + +[[package]] +name = "quote" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "ryu" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" + +[[package]] +name = "serde" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799e97dc9fdae36a5c8b8f2cae9ce2ee9fdce2058c57a93e6099d919fd982f79" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "syn" +version = "1.0.64" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fd9d1e9976102a03c542daa2eff1b43f9d72306342f3f8b3ed5fb8908195d6f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-xid", +] + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" + +[[package]] +name = "uuid" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom", +] + +[[package]] +name = "wasi" +version = "0.10.2+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" diff --git a/url-downloader/facade/Cargo.toml b/url-downloader/facade/Cargo.toml index d40090f..7c71f24 100644 --- a/url-downloader/facade/Cargo.toml +++ b/url-downloader/facade/Cargo.toml @@ -13,3 +13,4 @@ path = "src/main.rs" fluence = { version = "=0.5.0", features = ["logger"]} anyhow = "1.0.31" log = "0.4.8" +base64 = "0.13.0" diff --git a/url-downloader/facade/src/main.rs b/url-downloader/facade/src/main.rs index 8ff919b..4f2eb8c 100644 --- a/url-downloader/facade/src/main.rs +++ b/url-downloader/facade/src/main.rs @@ -18,6 +18,7 @@ use fluence::fce; use fluence::module_manifest; use fluence::WasmLoggerBuilder; +use fluence::MountedBinaryResult as Result; module_manifest!(); @@ -30,19 +31,30 @@ pub fn main() { #[fce] pub fn get_n_save(url: String, file_name: String) -> String { let result = unsafe { download(url) }; - unsafe { file_put(file_name, result.into_bytes()) }; - - String::from("Ok") + if result.is_success() { + log::info!("saving file {}", file_name); + unsafe { file_put(file_name, result.stdout) } + } else { + log::error!("download failed: {:#?}", result.as_std()); + format!("download failed: {:#?}", result.as_std()) + } } -/// Importing `curl` module +#[fce] +/// Loads file from disk and returns its content as base64 +pub fn load_file(file_name: String) -> String { + let bytes = unsafe { file_get(file_name) }; + base64::encode(bytes) +} + +/// Import `curl_adapter` module #[fce] #[link(wasm_import_module = "curl_adapter")] extern "C" { - pub fn download(url: String) -> String; + pub fn download(url: String) -> Result; } -/// Importing `local_storage` module +/// Import `local_storage` module #[fce] #[link(wasm_import_module = "local_storage")] extern "C" { diff --git a/url-downloader/local_storage/src/main.rs b/url-downloader/local_storage/src/main.rs index 8f6dabb..0f50966 100644 --- a/url-downloader/local_storage/src/main.rs +++ b/url-downloader/local_storage/src/main.rs @@ -14,14 +14,10 @@ * limitations under the License. */ -use fluence::fce; -use fluence::module_manifest; -use fluence::WasmLoggerBuilder; - use std::fs; -use std::path::PathBuf; - -module_manifest!(); +use fluence::fce; +use fluence::WasmLoggerBuilder; +use std::path::Path; const SITES_DIR: &str = "/sites/"; @@ -32,14 +28,14 @@ pub fn main() { /// You can read or write files from the file system if there is permission to use directories described in `Config.toml`. #[fce] -pub fn put(name: String, file_content: Vec) -> String { - log::info!("put called with file name {}", name); +pub fn put(file_name: String, file_content: Vec) -> String { + log::info!("put called with file name {}", file_name); - let rpc_tmp_filepath = format!("{}{}", SITES_DIR, name); + let path = Path::new(SITES_DIR).join(file_name); - let result = fs::write(PathBuf::from(rpc_tmp_filepath.clone()), file_content); + let result = fs::write(&path, file_content); if let Err(e) = result { - return format!("file can't be written: {}", e); + return format!("file {} can't be written: {}", path.to_string_lossy(), e); } String::from("Ok") @@ -49,7 +45,7 @@ pub fn put(name: String, file_content: Vec) -> String { pub fn get(file_name: String) -> Vec { log::info!("get called with file name: {}", file_name); - let tmp_filepath = format!("{}{}", SITES_DIR, file_name); + let path = Path::new(SITES_DIR).join(file_name); - fs::read(tmp_filepath).unwrap_or_else(|_| b"error while reading file".to_vec()) + fs::read(&path).unwrap_or_else(|err| format!("error while reading file {}: {}", path.to_string_lossy(), err).into_bytes()) } diff --git a/url-downloader/repl_config.toml b/url-downloader/repl_config.toml new file mode 100644 index 0000000..6701450 --- /dev/null +++ b/url-downloader/repl_config.toml @@ -0,0 +1,21 @@ +modules_dir = "artifacts/" + +[[module]] +name = "local_storage" +logger_enabled = true + +[module.wasi] +preopened_files = ["."] +# this is where files will be stored +mapped_dirs = { "sites" = "." } + +[[module]] +name = "curl_adapter" +logger_enabled = true + +[module.mounted_binaries] +curl = "/usr/bin/curl" + +[[module]] +name = "facade" +logger_enabled = true