chore: Update CI to use registry distro in E2E (#260)

* Get rid of buildin package

* Add snapshot workflow

* Cleanup

* Change name

* Typo

* Fix

* Fix

* Setup node

* Disable rust cache

* Build nox snapshot

* Fix

* Fix

.github/check-registry.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-
-PEERS=(
-  "/ip4/127.0.0.1/tcp/9991/ws/p2p/12D3KooWBM3SdXWqGaawQDGQ6JprtwswEg3FWGvGhmgmMez1vRbR"
-  "/ip4/127.0.0.1/tcp/9992/ws/p2p/12D3KooWQdpukY3p2DhDfUfDgphAqsGu5ZUrmQ4mcHSGrRag6gQK"
-  "/ip4/127.0.0.1/tcp/9993/ws/p2p/12D3KooWRT8V5awYdEZm6aAV9HWweCEbhWd7df4wehqHZXAB7yMZ"
-  "/ip4/127.0.0.1/tcp/9994/ws/p2p/12D3KooWBzLSu9RL7wLP6oUowzCbkCj2AGBSXkHSJKuq4wwTfwof"
-  "/ip4/127.0.0.1/tcp/9995/ws/p2p/12D3KooWBf6hFgrnXwHkBnwPGMysP3b1NJe5HGtAWPYfwmQ2MBiU"
-  "/ip4/127.0.0.1/tcp/9996/ws/p2p/12D3KooWPisGn7JhooWhggndz25WM7vQ2JmA121EV8jUDQ5xMovJ"
-)
-
-cd ${GITHUB_WORKSPACE}/aqua-tests
-
-for PEER_ADDR in ${PEERS[@]}; do
-  echo "Checking ${PEER_ADDR}"
-  if ! npx fluence aqua remote get_interface --addr ${PEER_ADDR} --id registry | jq -ec 'has("function_signatures")'; then
-    exit 1
-  fi
-done

.github/e2e/docker-compose.yml

@@ -24,8 +24,6 @@ services:
       # - --bootstraps=/dns/nox-1/tcp/7771
       # 12D3KooWBM3SdXWqGaawQDGQ6JprtwswEg3FWGvGhmgmMez1vRbR
       - -k=hK62afickoeP2uZbmSkAYXxxqP8ozq16VRN7qfTP719EHC5V5tjrtW57BSjUr8GvsEXmJRbtejUWyPZ2rZMyQdq
-    volumes:
-      - ../../registry:/builtins/registry
     networks:
       nox:
         ipv4_address: 10.50.10.10
@@ -47,8 +45,6 @@ services:
       - --bootstraps=/dns/nox-1/tcp/7771
       # 12D3KooWQdpukY3p2DhDfUfDgphAqsGu5ZUrmQ4mcHSGrRag6gQK
       - -k=2WijTVdhVRzyZamWjqPx4V4iNMrajegNMwNa2PmvPSZV6RRpo5M2fsPWdQr22HVRubuJhhSw8BrWiGt6FPhFAuXy
-    volumes:
-      - ../../registry:/builtins/registry
     networks:
       nox:
         ipv4_address: 10.50.10.20
@@ -70,8 +66,6 @@ services:
       - --bootstraps=/dns/nox-1/tcp/7771
       # 12D3KooWRT8V5awYdEZm6aAV9HWweCEbhWd7df4wehqHZXAB7yMZ
       - -k=2n2wBVanBeu2GWtvKBdrYK9DJAocgG3PrTUXMharq6TTfxqTL4sLdXL9BF23n6rsnkAY5pR9vBtx2uWYDQAiZdrX
-    volumes:
-      - ../../registry:/builtins/registry
     networks:
       nox:
         ipv4_address: 10.50.10.30
@@ -93,8 +87,6 @@ services:
       - --bootstraps=/dns/nox-1/tcp/7771
       # 12D3KooWBzLSu9RL7wLP6oUowzCbkCj2AGBSXkHSJKuq4wwTfwof
       - -k=4zp8ucAikkjB8CmkufYiFBW4QCDUCbQG7yMjviX7W8bMyN5rfChQ2Pi5QCWThrCTbAm9uq5nbFbxtFcNZq3De4dX
-    volumes:
-      - ../../registry:/builtins/registry
     networks:
       nox:
         ipv4_address: 10.50.10.40
@@ -116,8 +108,6 @@ services:
       - --bootstraps=/dns/nox-1/tcp/7771
       # 12D3KooWBf6hFgrnXwHkBnwPGMysP3b1NJe5HGtAWPYfwmQ2MBiU
       - -k=3ry26rm5gkJXvdqRH4FoM3ezWq4xVVsBQF7wtKq4E4pbuaa6p1F84tNqifUS7DdfJL9hs2gcdW64Wc342vHZHMUp
-    volumes:
-      - ../../registry:/builtins/registry
     networks:
       nox:
         ipv4_address: 10.50.10.50
@@ -139,8 +129,6 @@ services:
       - --allow-private-ips
       # 12D3KooWPisGn7JhooWhggndz25WM7vQ2JmA121EV8jUDQ5xMovJ
       - -k=5Qh8bB1sF28uLPwr3HTvEksCeC6mAWQvebCfcgv9y6j4qKwSzNKm2tzLUg4nACUEo2KZpBw11gNCnwaAdM7o1pEn
-    volumes:
-      - ../../registry:/builtins/registry
     networks:
       nox:
         ipv4_address: 10.50.10.60

.github/workflows/e2e.yml

@@ -0,0 +1,71 @@
+name: "e2e"
+
+on:
+  pull_request:
+    paths-ignore:
+      - "**.md"
+      - ".github/**"
+      - "!.github/workflows/e2e.yml"
+      - "!.github/workflows/snapshot.yml"
+    types:
+      - "labeled"
+      - "synchronize"
+      - "opened"
+      - "reopened"
+  push:
+    branches:
+      - "main"
+    paths-ignore:
+      - "**.md"
+      - ".github/**"
+      - "!.github/workflows/e2e.yml"
+      - "!.github/workflows/snapshot.yml"
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
+jobs:
+  snapshot:
+    if: >
+      github.event_name == 'push' ||
+      contains(github.event.pull_request.labels.*.name, 'e2e')
+    name: "registry"
+    uses: ./.github/workflows/snapshot.yml
+    with:
+      ref: ${{ github.ref }}
+
+  nox:
+    needs:
+      - snapshot
+    uses: fluencelabs/nox/.github/workflows/build.yml@master
+    with:
+      cargo-dependencies: |
+        [
+          {
+            "package": "registry-distro",
+            "version": "=${{ needs.snapshot.outputs.cargo-version }}",
+            "registry": "fluence",
+            "manifest": "crates/system-services/Cargo.toml"
+          }
+        ]
+
+  nox-snapshot:
+    name: "nox"
+    needs:
+      - nox
+
+    uses: fluencelabs/nox/.github/workflows/container.yml@master
+    with:
+      image-name: "docker.fluence.dev/registry"
+      flavour: "minimal"
+      nox-sha: "${{ needs.nox.outputs.nox-sha }}"
+
+  aqua-tests:
+    name: "registry"
+    needs:
+      - nox-snapshot
+    uses: ./.github/workflows/tests.yml
+    with:
+      ref: ${{ github.ref }}
+      nox-image: "${{ needs.nox-snapshot.outputs.nox-image }}"

.github/workflows/release.yml

@@ -118,6 +118,8 @@ jobs:
 
       - name: Setup Rust toolchain
         uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache: false
 
       - name: Setup marine
         uses: fluencelabs/setup-marine@v1
@@ -144,43 +146,6 @@ jobs:
         run: npm publish --access public
         working-directory: aqua
 
-      - name: Install ipfs
-        uses: nahsi/setup-ipfs@v1
-
-      - name: Create builtin distribution package
-        run: ./builtin-package/package.sh
-
-      - name: Calculate SHA256
-        id: sha
-        run: |
-          # Calculate sha256
-          du -hs registry.tar.gz
-          sha256sum registry.tar.gz
-          sha=($(sha256sum registry.tar.gz))
-          echo "sha256=${sha}" >> $GITHUB_OUTPUT
-
-      - name: Upload registry package
-        if: needs.release-please.outputs.release-created
-        uses: softprops/action-gh-release@v1
-        with:
-          files: registry.tar.gz
-          tag_name: ${{ needs.release-please.outputs.tag-name }}
-
-      - name: Update version in node-distro repo
-        if: needs.release-please.outputs.release-created
-        uses: benc-uk/workflow-dispatch@v1
-        with:
-          workflow: update_service
-          repo: fluencelabs/node-distro
-          ref: "main"
-          token: ${{ secrets.PERSONAL_TOKEN }}
-          inputs: '{
-            "name": "registry",
-            "version": "${{ needs.release-please.outputs.version }}",
-            "url": "https://github.com/fluencelabs/registry/releases/download/${{ needs.release-please.outputs.tag-name }}/registry.tar.gz",
-            "sha256": "${{ steps.sha.outputs.sha256 }}"
-          }'
-
       - name: Import secrets
         uses: hashicorp/vault-action@v2.4.3
         with:

.github/workflows/run-tests.yml

@@ -26,11 +26,6 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  registry:
-    uses: ./.github/workflows/tests.yml
-    with:
-      ref: ${{ github.ref }}
-
   cargo:
     name: "registry / Run cargo tests"
     runs-on: builder
@@ -70,3 +65,20 @@ jobs:
           NEXTEST_RETRIES: 2
           NEXTEST_TEST_THREADS: 1
         run: cargo nextest run --release --all-features --no-fail-fast
+
+  lints:
+    name: Lints
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+
+      - name: Setup rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+
+      - name: Run cargo fmt
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all --manifest-path service/Cargo.toml -- --check

.github/workflows/snapshot.yml

@@ -0,0 +1,113 @@
+name: Build snapshot
+
+on:
+  workflow_call:
+    inputs:
+      cargo-dependencies:
+        description: "Cargo dependencies map"
+        type: string
+        default: "null"
+      ref:
+        description: "git ref to checkout to"
+        type: string
+        default: "master"
+      snapshot:
+        description: "Whether to publish snapshots"
+        type: boolean
+        default: true
+    outputs:
+      cargo-version:
+        description: "Cargo snapshot version"
+        value: ${{ jobs.snapshot.outputs.version }}
+
+jobs:
+  snapshot:
+    name: "Build crate"
+    runs-on: builder
+    timeout-minutes: 60
+
+    outputs:
+      version: "${{ steps.snapshot.outputs.version }}"
+
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: fluencelabs/registry
+          ref: ${{ inputs.ref }}
+
+      - name: Setup rust toolchain
+        uses: actions-rust-lang/setup-rust-toolchain@v1
+        with:
+          cache: false
+
+      - name: Set dependencies
+        if: inputs.cargo-dependencies != 'null'
+        uses: fluencelabs/github-actions/cargo-set-dependency@main
+        with:
+          dependencies: ${{ inputs.cargo-dependencies }}
+          path: service/
+
+      - name: Download marine artifact
+        id: marine
+        uses: actions/download-artifact@v3
+        continue-on-error: true
+        with:
+          name: marine
+          path: ~/.local/bin
+
+      - name: Make marine executable
+        if: steps.marine.outcome == 'success'
+        run: chmod +x ~/.local/bin/marine
+
+      - name: Setup marine
+        if: steps.marine.outcome == 'failure'
+        uses: fluencelabs/setup-marine@v1
+
+      - name: Import secrets
+        if: inputs.snapshot == true
+        uses: hashicorp/vault-action@v2.5.0
+        with:
+          url: https://vault.fluence.dev
+          path: jwt/github
+          role: ci
+          method: jwt
+          jwtGithubAudience: "https://github.com/fluencelabs"
+          jwtTtl: 300
+          exportToken: false
+          secrets: |
+            kv/cargo-registry/users/ci token | CARGO_REGISTRIES_FLUENCE_TOKEN
+
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          node-version: "18"
+          registry-url: "https://registry.npmjs.org"
+          cache-dependency-path: "aqua/package-lock.json"
+          cache: "npm"
+
+      - run: npm i
+        working-directory: aqua
+
+      - name: Install cargo-workspaces
+        uses: baptiste0928/cargo-install@v2.0.0
+        with:
+          crate: cargo-workspaces
+          version: v0.2.37
+
+      - name: Generate snapshot version
+        id: version
+        uses: fluencelabs/github-actions/generate-snapshot-id@main
+
+      - name: Build distro
+        run: ./build.sh
+
+      - name: Publish crate snapshots
+        id: snapshot
+        uses: fluencelabs/github-actions/cargo-publish-snapshot@main
+        with:
+          id: ${{ steps.version.outputs.id }}
+          path: distro

.github/workflows/tests.yml

@@ -120,15 +120,6 @@ jobs:
       - name: Build distro
         run: ./build.sh
 
-      - name: Install ipfs
-        uses: nahsi/setup-ipfs@v1
-
-      - name: Create builtin distribution package
-        run: ./builtin-package/package.sh
-
-      - name: Unpack registry package
-        run: tar xvf registry.tar.gz
-
       - name: Login to DockerHub
         uses: docker/login-action@v2
         with:
@@ -156,17 +147,6 @@ jobs:
         run: pip install -r requirements.txt
         working-directory: aqua-tests
 
-      # - name: Wait for registry to be deployed
-      #   uses: nick-fields/retry@v2
-      #   with:
-      #     timeout_seconds: 30
-      #     max_attempts: 5
-      #     shell: bash
-      #     command: ${{ github.workspace }}/.github/check-registry.sh
-      #
-      - name: Temporarely wait with sleep
-        run: sleep 20
-
       - name: Install flox dependencies
         env:
           FLUENCE_USER_DIR: "${{ github.workspace }}/tmp/.fluence"

builtin-package/package.sh

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-set -o pipefail -o nounset -o errexit
-
-# set current working directory to script directory to run script from everywhere
-cd "$(dirname "$0")"
-PACKAGE_DIR="$(pwd)/../registry"
-SCHEDULED="$PACKAGE_DIR/scheduled"
-
-(
-    rm -rf $PACKAGE_DIR
-    mkdir -p $PACKAGE_DIR
-)
-(
-    echo "*** compile scheduled scripts ***"
-    cd ../aqua
-    npx fluence --version
-    npx fluence aqua --no-relay --air -i ./registry-scheduled-scripts.aqua -o "$SCHEDULED"
-)
-
-(
-    echo "*** copy wasm files ***"
-    cd ../service
-    cp artifacts/*.wasm "$PACKAGE_DIR"
-)
-
-REGISTRY_CID=$(ipfs add -q --only-hash --cid-version=1 --chunker=size-262144 $PACKAGE_DIR/registry.wasm)
-SQLITE_CID=$(ipfs add -q --only-hash --cid-version=1 --chunker=size-262144 $PACKAGE_DIR/sqlite3.wasm)
-mv $PACKAGE_DIR/registry.wasm "$PACKAGE_DIR"/"$REGISTRY_CID".wasm
-mv $PACKAGE_DIR/sqlite3.wasm "$PACKAGE_DIR"/"$SQLITE_CID".wasm
-cp registry_config.json "$PACKAGE_DIR"/"$REGISTRY_CID"_config.json
-cp sqlite3_config.json "$PACKAGE_DIR"/"$SQLITE_CID"_config.json
-
-# write blueprint.json
-echo "{}" | jq --arg registry_cid "$REGISTRY_CID" --arg sqlite_cid "$SQLITE_CID" '{"name": "registry", "dependencies":[{"/":$sqlite_cid},{"/":$registry_cid}]}' > "$PACKAGE_DIR/blueprint.json"
-
-
-(
-    echo "*** create builtin distribution package ***"
-    cd ..
-    tar -f registry.tar.gz -zcv ./registry
-)
-
-echo "*** done ***"

builtin-package/registry_config.json

@@ -1,8 +0,0 @@
-{
-  "name": "registry",
-  "max_heap_size": "8 MiB",
-  "preopened_files": ["/tmp"],
-  "mapped_dirs": {
-    "tmp": "./tmp"
-  }
-}

builtin-package/sqlite3_config.json

@@ -1,4 +0,0 @@
-{
-  "name": "sqlite3",
-  "max_heap_size": "64 MiB"
-}