Refactor Aqua & save file size to IPFS

This commit is contained in:
folex
2021-07-21 15:02:28 +03:00
parent e5e0d59835
commit 19cc6b8546
10 changed files with 422 additions and 99 deletions

View File

@ -8,9 +8,9 @@
"prebuild": "npm run compile-aqua",
"build": "tsc",
"prestart:local": "npm run build",
"start:local": "node dist/index.js local",
"start:local": "node dist/demo.js local",
"prestart:remote": "npm run build",
"start:remote": "node dist/index.js stage",
"start:remote": "node dist/demo.js stage",
"start": "npm run start:remote"
},
"keywords": [

View File

@ -1,7 +0,0 @@
data SizeResult:
size: u32
success: bool
error: string
service ProcessFiles:
file_size(file_path: string) -> SizeResult

View File

@ -19,7 +19,7 @@ import { set_timeout } from "@fluencelabs/aqua-ipfs";
import {createClient, FluenceClient, setLogLevel} from "@fluencelabs/fluence";
import {stage, krasnodar, Node, testNet} from "@fluencelabs/fluence-network-environment";
import { provideFile, globSource, urlSource } from "./provider";
import { deploy_service, get_file_size, remove_service } from "./process";
import { deploy_service, put_file_size, remove_service } from "./process";
async function main(environment: Node[]) {
// setLogLevel('DEBUG');
@ -38,28 +38,49 @@ async function main(environment: Node[]) {
await set_timeout(fluence, environment[2].peerId, 10);
console.log("\n\n📘 Will deploy ProcessFiles service");
let service_id = await deploy_service(
var service_id = await deploy_service(
fluence,
environment[2].peerId, file.cid.toString(), rpcAddr,
(msg, value) => console.log(msg, value),
(label, error) => { console.error("📕 deploy_service failed: ", label, error) },
{ ttl: 10000 }
)
service_id = from_option(service_id);
if (service_id === null) {
return;
}
console.log("📗 ProcessFiles service is now deployed and available as", service_id);
console.log("\n\n📘 Will upload file & calculate its size");
let { file: newFile } = await provideFile(urlSource("https://i.imgur.com/NZgK6DB.png"), providerClient);
let fileSize = await get_file_size(
var putResult = await put_file_size(
fluence,
environment[2].peerId, newFile.cid.toString(), rpcAddr, service_id,
fileSize => console.log("📗 Calculated file size:", fileSize),
(label, error) => { console.error("📕 put_file_size failed: ", label, error) },
{ ttl: 10000 }
)
console.log("📗 Calculated file size:", fileSize)
putResult = from_option(putResult);
if (putResult !== null) {
console.log("📗 File size is saved to IPFS:", putResult);
}
let result = await remove_service(fluence, environment[2].peerId, service_id);
console.log("📕 ProcessFiles service removed", result);
console.log("📗 ProcessFiles service removed", result);
return;
}
function from_option<T>(opt: T | T[] | null): T | null {
if (Array.isArray(opt)) {
if (opt.length === 0) { return null; }
opt = opt[0];
}
if (opt === null) { return null; }
return opt;
}
let args = process.argv.slice(2);
var environment: Node[];
if (args.length >= 1 && args[0] == "testnet") {

View File

@ -16,28 +16,65 @@ service NewOp("op"):
concat_strings(a: string, b: string) -> string
array(s: string) -> []string
func deploy_service(relay: PeerId, cid: CID, provider_ipfs: Multiaddr, log: string, u32 -> ()) -> ServiceID:
on relay:
get_result <- Ipfs.get_from(cid, provider_ipfs)
config <- NewDist.default_module_config("process_files")
module_hash <- NewDist.add_module_from_vault(get_result.path, config)
prefixed_hash <- NewOp.concat_strings("hash:", module_hash)
dependencies <- NewOp.array(prefixed_hash)
blueprint <- Dist.make_blueprint("process_files", dependencies)
blueprint_id <- Dist.add_blueprint(blueprint)
service_id <- Srv.create(blueprint_id)
-- Add module to node
func add_module(name: string, path: string) -> Hash:
config <- NewDist.default_module_config(name)
module_hash <- NewDist.add_module_from_vault(path, config)
<- module_hash
-- Add service blueprint to node
func add_blueprint(module_hash: Hash) -> string:
prefixed_hash <- NewOp.concat_strings("hash:", module_hash)
dependencies <- NewOp.array(prefixed_hash)
blueprint <- Dist.make_blueprint("process_files", dependencies)
blueprint_id <- Dist.add_blueprint(blueprint)
<- blueprint_id
-- Download single .wasm module from IPFS and create a service from it
func deploy_service(relay: PeerId, cid: CID, ipfs: Multiaddr, error: string, string -> ()) -> ?ServiceID:
service_id: *ServiceID
on relay:
-- Download .wasm from IPFS to node
get_result <- Ipfs.get_from(cid, ipfs)
if get_result.success:
module_hash <- add_module("process_files", get_result.path)
blueprint_id <- add_blueprint(module_hash)
service_id <- Srv.create(blueprint_id)
else:
co error("Ipfs.get_from failed", get_result.error)
ProcessFiles service_id
size <- ProcessFiles.file_size(get_result.path)
log("Size of the .wasm module is", size.size)
<- service_id
func get_file_size(relay: PeerId, cid: CID, provider_ipfs: Multiaddr, service_id: ServiceID) -> SizeResult:
-- Download file from IPFS, and write it's size to file in IPFS
func put_file_size(
relay: PeerId,
cid: CID,
ipfs: Multiaddr,
service_id: ServiceID,
logSize: u32 -> (),
error: string, string -> ()
) -> ?IpfsPutResult:
result: *IpfsPutResult
ProcessFiles service_id
on relay:
get_result <- Ipfs.get_from(cid, provider_ipfs)
size <- ProcessFiles.file_size(get_result.path)
<- size
get <- Ipfs.get_from(cid, ipfs)
if get.success:
size <- ProcessFiles.file_size(get.path)
if size.success:
-- report file size in background
co logSize(size.size)
-- write file size to disk
write <- ProcessFiles.write_file_size(size.size)
if write.success:
-- upload file to ipfs
result <- Ipfs.put(write.path)
else:
co error("ProcessFiles.write_file_size failed", write.error)
else:
co error("ProcessFiles.file_size failed", size.error)
else:
co error("Ipfs.get_from failed", get.error)
<- result
func remove_service(relay: PeerId, service_id: ServiceID) -> bool:
on relay:

View File

@ -12,7 +12,7 @@ import { RequestFlow } from '@fluencelabs/fluence/dist/internal/RequestFlow';
export async function deploy_service(client: FluenceClient, relay: string, cid: string, provider_ipfs: string, log: (arg0: string, arg1: number) => void, config?: {ttl?: number}): Promise<string> {
export async function add_blueprint(client: FluenceClient, module_hash: string, config?: {ttl?: number}): Promise<string> {
let request: RequestFlow;
const promise = new Promise<string>((resolve, reject) => {
const r = new RequestFlowBuilder()
@ -27,58 +27,22 @@ export async function deploy_service(client: FluenceClient, relay: string, cid:
(seq
(seq
(call %init_peer_id% ("getDataSrv" "-relay-") [] -relay-)
(call %init_peer_id% ("getDataSrv" "relay") [] relay)
(call %init_peer_id% ("getDataSrv" "module_hash") [] module_hash)
)
(call %init_peer_id% ("getDataSrv" "cid") [] cid)
(call %init_peer_id% ("op" "concat_strings") ["hash:" module_hash] prefixed_hash)
)
(call %init_peer_id% ("getDataSrv" "provider_ipfs") [] provider_ipfs)
(call %init_peer_id% ("op" "array") [prefixed_hash] dependencies)
)
(call -relay- ("op" "noop") [])
)
(xor
(seq
(seq
(seq
(seq
(seq
(seq
(seq
(seq
(seq
(seq
(call relay ("ipfs-adapter" "get_from") [cid provider_ipfs] get_result)
(call relay ("dist" "default_module_config") ["process_files"] config)
)
(call relay ("dist" "add_module_from_vault") [get_result.$.path! config] module_hash)
)
(call relay ("op" "concat_strings") ["hash:" module_hash] prefixed_hash)
)
(call relay ("op" "array") [prefixed_hash] dependencies)
)
(call relay ("dist" "make_blueprint") ["process_files" dependencies] blueprint)
)
(call relay ("dist" "add_blueprint") [blueprint] blueprint_id)
)
(call relay ("srv" "create") [blueprint_id] service_id)
)
(call relay (service_id "file_size") [get_result.$.path!] size)
)
(call -relay- ("op" "noop") [])
)
(xor
(call %init_peer_id% ("callbackSrv" "log") ["Size of the .wasm module is" size.$.size!])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 1])
)
)
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 2])
(call %init_peer_id% ("dist" "make_blueprint") ["process_files" dependencies] blueprint)
)
(call %init_peer_id% ("dist" "add_blueprint") [blueprint] blueprint_id)
)
(xor
(call %init_peer_id% ("callbackSrv" "response") [service_id])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 3])
(call %init_peer_id% ("callbackSrv" "response") [blueprint_id])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 1])
)
)
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 4])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 2])
)
`,
@ -87,10 +51,7 @@ export async function deploy_service(client: FluenceClient, relay: string, cid:
h.on('getDataSrv', '-relay-', () => {
return client.relayPeerId!;
});
h.on('getDataSrv', 'relay', () => {return relay;});
h.on('getDataSrv', 'cid', () => {return cid;});
h.on('getDataSrv', 'provider_ipfs', () => {return provider_ipfs;});
h.on('callbackSrv', 'log', (args) => {log(args[0], args[1]); return {};});
h.on('getDataSrv', 'module_hash', () => {return module_hash;});
h.onEvent('callbackSrv', 'response', (args) => {
const [res] = args;
resolve(res);
@ -104,7 +65,7 @@ h.on('callbackSrv', 'log', (args) => {log(args[0], args[1]); return {};});
})
.handleScriptError(reject)
.handleTimeout(() => {
reject('Request timed out for deploy_service');
reject('Request timed out for add_blueprint');
})
if(config && config.ttl) {
r.withTTL(config.ttl)
@ -117,9 +78,9 @@ h.on('callbackSrv', 'log', (args) => {log(args[0], args[1]); return {};});
export async function get_file_size(client: FluenceClient, relay: string, cid: string, provider_ipfs: string, service_id: string, config?: {ttl?: number}): Promise<{error:string;size:number;success:boolean}> {
export async function put_file_size(client: FluenceClient, relay: string, cid: string, ipfs: string, service_id: string, logSize: (arg0: number) => void, error: (arg0: string, arg1: string) => void, config?: {ttl?: number}): Promise<{error:string;hash:string;success:boolean} | null> {
let request: RequestFlow;
const promise = new Promise<{error:string;size:number;success:boolean}>((resolve, reject) => {
const promise = new Promise<{error:string;hash:string;success:boolean} | null>((resolve, reject) => {
const r = new RequestFlowBuilder()
.disableInjections()
.withRawScript(
@ -138,7 +99,7 @@ export async function get_file_size(client: FluenceClient, relay: string, cid: s
)
(call %init_peer_id% ("getDataSrv" "cid") [] cid)
)
(call %init_peer_id% ("getDataSrv" "provider_ipfs") [] provider_ipfs)
(call %init_peer_id% ("getDataSrv" "ipfs") [] ipfs)
)
(call %init_peer_id% ("getDataSrv" "service_id") [] service_id)
)
@ -146,23 +107,104 @@ export async function get_file_size(client: FluenceClient, relay: string, cid: s
)
(xor
(seq
(call relay ("ipfs-adapter" "get_from") [cid provider_ipfs] get_result)
(call relay (service_id "file_size") [get_result.$.path!] size)
(call relay ("ipfs-adapter" "get_from") [cid ipfs] get)
(xor
(match get.$.success! true
(xor
(seq
(call relay (service_id "file_size") [get.$.path!] size)
(xor
(match size.$.success! true
(xor
(seq
(seq
(par
(seq
(call -relay- ("op" "noop") [])
(xor
(call %init_peer_id% ("callbackSrv" "logSize") [size.$.size!])
(seq
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 1])
(call -relay- ("op" "noop") [])
)
)
)
(null)
)
(call relay (service_id "write_file_size") [size.$.size!] write)
)
(xor
(match write.$.success! true
(xor
(call relay ("ipfs-adapter" "put") [write.$.path!] $result)
(seq
(call -relay- ("op" "noop") [])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 2])
)
)
)
(par
(seq
(call -relay- ("op" "noop") [])
(xor
(call %init_peer_id% ("callbackSrv" "error") ["ProcessFiles.write_file_size failed" write.$.error!])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 3])
)
)
(null)
)
)
)
(seq
(call -relay- ("op" "noop") [])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 4])
)
)
)
(par
(seq
(call -relay- ("op" "noop") [])
(xor
(call %init_peer_id% ("callbackSrv" "error") ["ProcessFiles.file_size failed" size.$.error!])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 5])
)
)
(null)
)
)
)
(seq
(call -relay- ("op" "noop") [])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 6])
)
)
)
(par
(seq
(call -relay- ("op" "noop") [])
(xor
(call %init_peer_id% ("callbackSrv" "error") ["Ipfs.get_from failed" get.$.error!])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 7])
)
)
(null)
)
)
)
(seq
(call -relay- ("op" "noop") [])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 1])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 8])
)
)
)
(call -relay- ("op" "noop") [])
)
(xor
(call %init_peer_id% ("callbackSrv" "response") [size])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 2])
(call %init_peer_id% ("callbackSrv" "response") [$result])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 9])
)
)
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 3])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 10])
)
`,
@ -173,8 +215,10 @@ export async function get_file_size(client: FluenceClient, relay: string, cid: s
});
h.on('getDataSrv', 'relay', () => {return relay;});
h.on('getDataSrv', 'cid', () => {return cid;});
h.on('getDataSrv', 'provider_ipfs', () => {return provider_ipfs;});
h.on('getDataSrv', 'ipfs', () => {return ipfs;});
h.on('getDataSrv', 'service_id', () => {return service_id;});
h.on('callbackSrv', 'logSize', (args) => {logSize(args[0]); return {};});
h.on('callbackSrv', 'error', (args) => {error(args[0], args[1]); return {};});
h.onEvent('callbackSrv', 'response', (args) => {
const [res] = args;
resolve(res);
@ -188,7 +232,7 @@ h.on('getDataSrv', 'service_id', () => {return service_id;});
})
.handleScriptError(reject)
.handleTimeout(() => {
reject('Request timed out for get_file_size');
reject('Request timed out for put_file_size');
})
if(config && config.ttl) {
r.withTTL(config.ttl)
@ -272,3 +316,186 @@ h.on('getDataSrv', 'service_id', () => {return service_id;});
return promise;
}
export async function add_module(client: FluenceClient, name: string, path: string, config?: {ttl?: number}): Promise<string> {
let request: RequestFlow;
const promise = new Promise<string>((resolve, reject) => {
const r = new RequestFlowBuilder()
.disableInjections()
.withRawScript(
`
(xor
(seq
(seq
(seq
(seq
(seq
(call %init_peer_id% ("getDataSrv" "-relay-") [] -relay-)
(call %init_peer_id% ("getDataSrv" "name") [] name)
)
(call %init_peer_id% ("getDataSrv" "path") [] path)
)
(call %init_peer_id% ("dist" "default_module_config") [name] config)
)
(call %init_peer_id% ("dist" "add_module_from_vault") [path config] module_hash)
)
(xor
(call %init_peer_id% ("callbackSrv" "response") [module_hash])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 1])
)
)
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 2])
)
`,
)
.configHandler((h) => {
h.on('getDataSrv', '-relay-', () => {
return client.relayPeerId!;
});
h.on('getDataSrv', 'name', () => {return name;});
h.on('getDataSrv', 'path', () => {return path;});
h.onEvent('callbackSrv', 'response', (args) => {
const [res] = args;
resolve(res);
});
h.onEvent('errorHandlingSrv', 'error', (args) => {
// assuming error is the single argument
const [err] = args;
reject(err);
});
})
.handleScriptError(reject)
.handleTimeout(() => {
reject('Request timed out for add_module');
})
if(config && config.ttl) {
r.withTTL(config.ttl)
}
request = r.build();
});
await client.initiateFlow(request!);
return promise;
}
export async function deploy_service(client: FluenceClient, relay: string, cid: string, ipfs: string, error: (arg0: string, arg1: string) => void, config?: {ttl?: number}): Promise<string | null> {
let request: RequestFlow;
const promise = new Promise<string | null>((resolve, reject) => {
const r = new RequestFlowBuilder()
.disableInjections()
.withRawScript(
`
(xor
(seq
(seq
(seq
(seq
(seq
(seq
(seq
(call %init_peer_id% ("getDataSrv" "-relay-") [] -relay-)
(call %init_peer_id% ("getDataSrv" "relay") [] relay)
)
(call %init_peer_id% ("getDataSrv" "cid") [] cid)
)
(call %init_peer_id% ("getDataSrv" "ipfs") [] ipfs)
)
(call -relay- ("op" "noop") [])
)
(xor
(seq
(call relay ("ipfs-adapter" "get_from") [cid ipfs] get_result)
(xor
(match get_result.$.success! true
(xor
(seq
(seq
(seq
(seq
(seq
(seq
(call relay ("dist" "default_module_config") ["process_files"] config)
(call relay ("dist" "add_module_from_vault") [get_result.$.path! config] module_hash)
)
(call relay ("op" "concat_strings") ["hash:" module_hash] prefixed_hash)
)
(call relay ("op" "array") [prefixed_hash] dependencies)
)
(call relay ("dist" "make_blueprint") ["process_files" dependencies] blueprint)
)
(call relay ("dist" "add_blueprint") [blueprint] blueprint_id)
)
(call relay ("srv" "create") [blueprint_id] $service_id)
)
(seq
(call -relay- ("op" "noop") [])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 1])
)
)
)
(par
(seq
(call -relay- ("op" "noop") [])
(xor
(call %init_peer_id% ("callbackSrv" "error") ["Ipfs.get_from failed" get_result.$.error!])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 2])
)
)
(null)
)
)
)
(seq
(call -relay- ("op" "noop") [])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 3])
)
)
)
(call -relay- ("op" "noop") [])
)
(xor
(call %init_peer_id% ("callbackSrv" "response") [$service_id])
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 4])
)
)
(call %init_peer_id% ("errorHandlingSrv" "error") [%last_error% 5])
)
`,
)
.configHandler((h) => {
h.on('getDataSrv', '-relay-', () => {
return client.relayPeerId!;
});
h.on('getDataSrv', 'relay', () => {return relay;});
h.on('getDataSrv', 'cid', () => {return cid;});
h.on('getDataSrv', 'ipfs', () => {return ipfs;});
h.on('callbackSrv', 'error', (args) => {error(args[0], args[1]); return {};});
h.onEvent('callbackSrv', 'response', (args) => {
const [res] = args;
resolve(res);
});
h.onEvent('errorHandlingSrv', 'error', (args) => {
// assuming error is the single argument
const [err] = args;
reject(err);
});
})
.handleScriptError(reject)
.handleTimeout(() => {
reject('Request timed out for deploy_service');
})
if(config && config.ttl) {
r.withTTL(config.ttl)
}
request = r.build();
});
await client.initiateFlow(request!);
return promise;
}

View File

@ -3,5 +3,11 @@ data SizeResult:
success: bool
error: string
data WriteResult:
path: string
success: bool
error: string
service ProcessFiles:
file_size(file_path: string) -> SizeResult
write_file_size(size: u32) -> WriteResult

View File

@ -13,6 +13,7 @@ path = "src/main.rs"
[dependencies]
marine-rs-sdk = { version="0.6.11", features=["logger"] }
log = "0.4.14"
rand = "0.8.4"
[dev-dependencies]
marine-rs-sdk-test = "0.1.11"

View File

@ -5,4 +5,4 @@ mkdir -p artifacts
rm -f artifacts/*.wasm
marine build --release
cp target/wasm32-wasi/release/process_files.wasm artifacts/
marine aqua artifacts/process_files.wasm >../aqua/process_files.aqua
marine aqua artifacts/process_files.wasm >../aqua/src/process_files.aqua

View File

@ -14,7 +14,10 @@
* limitations under the License.
*/
use marine_rs_sdk::{marine, module_manifest};
use marine_rs_sdk::{marine, module_manifest, get_call_parameters};
use std::path::{PathBuf, Path};
use rand::Rng;
use rand::distributions::Alphanumeric;
module_manifest!();
@ -35,3 +38,32 @@ pub fn file_size(file_path: String) -> SizeResult {
}
}
#[marine]
pub struct WriteResult {
pub path: String,
pub success: bool,
pub error: String,
}
#[marine]
pub fn write_file_size(size: u32) -> WriteResult {
let name: String = rand::thread_rng()
.sample_iter(Alphanumeric)
.take(16)
.map(char::from)
.collect();
let file = vault_dir().join(&name);
let file_str = file.to_string_lossy().to_string();
match std::fs::write(&file, size.to_string()) {
Ok(_) => WriteResult { path: file_str, success: true, error: String::new() },
Err(err) => WriteResult { path: String::new(), success: false, error: err.to_string() }
}
}
fn vault_dir() -> PathBuf {
let particle_id = get_call_parameters().particle_id;
let vault = Path::new("/tmp").join("vault").join(particle_id);
vault
}

View File

@ -18,8 +18,6 @@ function App() {
const [client, setClient] = useState<FluenceClient | null>(null);
const [serviceId, setServiceId] = useState<string | null>(null);
const [peerIdInput, setPeerIdInput] = useState<string>("");
const [relayPeerIdInput, setRelayPeerIdInput] = useState<string>("");
const [wasm, setWasm] = useState<string | null>("QmVg9EnanAbwTuEqjjuc1R2uf3AdtEkrNagSifQMkHfyNU");
const [rpcAddr, setRpcAddr] = useState<string | null>("");
const [fileCID, setFileCID] = useState<string>("");
@ -70,7 +68,15 @@ function App() {
return;
}
let size = await get_file_size(client, client.relayPeerId!, fileCID, rpcAddr, serviceId, { ttl: 10000 });
let size = await get_file_size(
client,
client.relayPeerId!, fileCID, rpcAddr, serviceId,
(label, error) => setFileSize("Error: " + label + ": " + error),
{ ttl: 10000 }
);
if (size === null) {
return;
}
if (size.success) {
setFileSize(size.size.toString());
} else {