feat: add distro crate [fixes NET-462] (#233)

.github/workflows/release.yml

@@ -67,6 +67,10 @@ jobs:
         working-directory: service
         run: cargo set-version ${{ steps.version.outputs.version }}
 
+      - name: Set version in distro
+        working-directory: distro
+        run: cargo set-version ${{ steps.version.outputs.version }}
+
       - name: Setup node
         uses: actions/setup-node@v3
         with:
@@ -177,6 +181,41 @@ jobs:
             "sha256": "${{ steps.sha.outputs.sha256 }}"
           }'
 
+      - name: Import secrets
+        uses: hashicorp/vault-action@v2.4.3
+        with:
+          url: https://vault.fluence.dev
+          path: jwt/github
+          role: ci
+          method: jwt
+          jwtGithubAudience: "https://github.com/fluencelabs"
+          jwtTtl: 300
+          exportToken: false
+          secrets: |
+            kv/crates.io/fluencebot token | CARGO_REGISTRY_TOKEN
+
+      - name: Setup Rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Install cargo-workspaces
+        uses: baptiste0928/cargo-install@v2.0.0
+        with:
+          crate: cargo-workspaces
+          version: v0.2.37
+
+      - name: Build distro
+        run: ./build.sh
+
+      - name: Publish to crates.io
+        working-directory: ./distro
+        run: |
+          cargo ws publish \
+            --no-git-commit \
+            --allow-dirty \
+            --from-git \
+            --skip-published \
+            --yes
+
   slack:
     if: always()
     name: "Notify"

.github/workflows/tests.yml

@@ -141,6 +141,9 @@ jobs:
         run: ./build.sh
         working-directory: service
 
+      - name: Build distro
+        run: ./build.sh
+
       - name: Install ipfs
         uses: nahsi/setup-ipfs@v1
 

.gitignore

@@ -3,6 +3,8 @@ builtin-package/*.wasm
 builtin-package/scheduled/*.air
 registry.tar.gz
 registry
+distro/target/
+distro/registry-service/
 
 **/*.rs.bk
 **/.idea

build.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+set -o errexit -o nounset -o pipefail
+
+# set current working directory to script directory to run script from everywhere
+cd "$(dirname "$0")"
+
+# Build the service
+./service/build.sh
+
+DISTRO_TARGET=distro/registry-service
+mkdir -p "$DISTRO_TARGET"
+
+cd aqua
+npx aqua --no-relay --air -i registry-scheduled-scripts.aqua -o "../$DISTRO_TARGET/air"
+cd -
+
+cp service/artifacts/registry.wasm service/artifacts/sqlite3.wasm distro/Config.toml "$DISTRO_TARGET"
+
+cd distro
+cargo build

distro/Cargo.lock

@@ -0,0 +1,177 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "built"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b9c056b9ed43aee5e064b683aa1ec783e19c6acec7559e3ae931b7490472fbe"
+dependencies = [
+ "cargo-lock",
+]
+
+[[package]]
+name = "cargo-lock"
+version = "8.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "031718ddb8f78aa5def78a09e90defe30151d1f6c672f937af4dd916429ed996"
+dependencies = [
+ "semver",
+ "serde",
+ "toml",
+ "url",
+]
+
+[[package]]
+name = "form_urlencoded"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
+dependencies = [
+ "percent-encoding",
+]
+
+[[package]]
+name = "idna"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
+[[package]]
+name = "maplit"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
+
+[[package]]
+name = "percent-encoding"
+version = "2.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.56"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "registry-distro"
+version = "0.8.4"
+dependencies = [
+ "built",
+ "maplit",
+]
+
+[[package]]
+name = "semver"
+version = "1.0.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bebd363326d05ec3e2f532ab7660680f3b02130d780c299bca73469d521bc0ed"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.160"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.160"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "tinyvec"
+version = "1.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
+[[package]]
+name = "toml"
+version = "0.5.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "unicode-bidi"
+version = "0.3.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
+
+[[package]]
+name = "unicode-normalization"
+version = "0.1.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921"
+dependencies = [
+ "tinyvec",
+]
+
+[[package]]
+name = "url"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"
+dependencies = [
+ "form_urlencoded",
+ "idna",
+ "percent-encoding",
+]

distro/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "registry-distro"
+version = "0.8.6"
+edition = "2021"
+build = "built.rs"
+include = [ "/src", "built.rs", "Cargo.toml", "registry-service"]
+description = "Distribution package for the registry service including scheduled scripts"
+license = "Apache-2.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[dependencies]
+maplit = "1.0.2"
+
+[build-dependencies]
+built = "0.5.2"

distro/Config.toml

@@ -0,0 +1,14 @@
+modules_dir = "."
+[[module]]
+    name = "sqlite3"
+    mem_pages_count = 100
+    logger_enabled = false
+
+    [module.wasi]
+    preopened_files = ["./tmp"]
+    mapped_dirs = { "tmp" = "./tmp" }
+
+[[module]]
+    name = "registry"
+    mem_pages_count = 1
+    logger_enabled = false

distro/built.rs

@@ -0,0 +1,3 @@
+fn main() {
+    built::write_built_file().expect("Failed to acquire build-time information")
+}

distro/src/lib.rs

@@ -0,0 +1,53 @@
+pub const TRUST_GRAPH_WASM: &'static [u8] = include_bytes!("../registry-service/registry.wasm");
+pub const SQLITE_WASM: &'static [u8] = include_bytes!("../registry-service/sqlite3.wasm");
+pub const CONFIG: &'static [u8] = include_bytes!("../registry-service/Config.toml");
+pub const CLEAR_EXPIRED_AIR: &'static str =
+    include_str!("../registry-service/air/registry-scheduled-scripts.clearExpired_86400.air");
+pub const CLEAR_EXPIRED_PERIOD_SEC: u32 = 86400;
+
+pub const RENEW_AIR: &'static str =
+    include_str!("../registry-service/air/registry-scheduled-scripts.renew_43200.air");
+pub const RENEW_PERIOD_SEC: u32 = 43200;
+
+pub const REPLICATE_AIR: &'static str =
+    include_str!("../registry-service/air/registry-scheduled-scripts.replicate_3600.air");
+pub const REPLICATE_PERIOD_SEC: u32 = 3600;
+
+pub mod build_info {
+    include!(concat!(env!("OUT_DIR"), "/built.rs"));
+}
+
+pub use build_info::PKG_VERSION as VERSION;
+
+pub fn modules() -> std::collections::HashMap<&'static str, &'static [u8]> {
+    maplit::hashmap! {
+        "sqlite3" => SQLITE_WASM,
+        "registry" => TRUST_GRAPH_WASM,
+    }
+}
+
+pub struct Script {
+    pub name: &'static str,
+    pub air: &'static str,
+    pub period_sec: u32,
+}
+
+pub fn scripts() -> Vec<Script> {
+    vec![
+        Script {
+            name: "registry-clear-expired",
+            air: CLEAR_EXPIRED_AIR,
+            period_sec: CLEAR_EXPIRED_PERIOD_SEC,
+        },
+        Script {
+            name: "registry-renew",
+            air: RENEW_AIR,
+            period_sec: RENEW_PERIOD_SEC,
+        },
+        Script {
+            name: "registry-replicate",
+            air: REPLICATE_AIR,
+            period_sec: REPLICATE_PERIOD_SEC,
+        },
+    ]
+}

service/Config.toml

@@ -1,5 +1,4 @@
-modules_dir = "artifacts/"
+modules_dir = "artifacts"
-
 [[module]]
     name = "sqlite3"
     mem_pages_count = 100
@@ -13,7 +12,3 @@ modules_dir = "artifacts/"
     name = "registry"
     mem_pages_count = 1
     logger_enabled = false
-
-    [module.wasi]
-    preopened_files = ["/tmp"]
-    mapped_dirs = { "tmp" = "/tmp" }

service/build.sh

@@ -17,3 +17,6 @@ curl -L https://github.com/fluencelabs/sqlite/releases/download/sqlite-wasm-v0.1
 
 # generate Aqua bindings
 marine aqua artifacts/registry.wasm -s Registry -i registry >../aqua/registry-service.aqua
+
+mkdir -p ../distro/registry-service
+cp artifacts/registry.wasm artifacts/sqlite3.wasm ../distro/registry-service