diff --git a/.github/check-registry.sh b/.github/check-registry.sh deleted file mode 100755 index 093bf42..0000000 --- a/.github/check-registry.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -PEERS=( - "/ip4/127.0.0.1/tcp/9991/ws/p2p/12D3KooWBM3SdXWqGaawQDGQ6JprtwswEg3FWGvGhmgmMez1vRbR" - "/ip4/127.0.0.1/tcp/9992/ws/p2p/12D3KooWQdpukY3p2DhDfUfDgphAqsGu5ZUrmQ4mcHSGrRag6gQK" - "/ip4/127.0.0.1/tcp/9993/ws/p2p/12D3KooWRT8V5awYdEZm6aAV9HWweCEbhWd7df4wehqHZXAB7yMZ" - "/ip4/127.0.0.1/tcp/9994/ws/p2p/12D3KooWBzLSu9RL7wLP6oUowzCbkCj2AGBSXkHSJKuq4wwTfwof" - "/ip4/127.0.0.1/tcp/9995/ws/p2p/12D3KooWBf6hFgrnXwHkBnwPGMysP3b1NJe5HGtAWPYfwmQ2MBiU" - "/ip4/127.0.0.1/tcp/9996/ws/p2p/12D3KooWPisGn7JhooWhggndz25WM7vQ2JmA121EV8jUDQ5xMovJ" -) - -cd ${GITHUB_WORKSPACE}/aqua-tests - -for PEER_ADDR in ${PEERS[@]}; do - echo "Checking ${PEER_ADDR}" - if ! npx fluence aqua remote get_interface --addr ${PEER_ADDR} --id registry | jq -ec 'has("function_signatures")'; then - exit 1 - fi -done diff --git a/.github/e2e/docker-compose.yml b/.github/e2e/docker-compose.yml index 3cbf2bf..7339b56 100644 --- a/.github/e2e/docker-compose.yml +++ b/.github/e2e/docker-compose.yml @@ -24,8 +24,6 @@ services: # - --bootstraps=/dns/nox-1/tcp/7771 # 12D3KooWBM3SdXWqGaawQDGQ6JprtwswEg3FWGvGhmgmMez1vRbR - -k=hK62afickoeP2uZbmSkAYXxxqP8ozq16VRN7qfTP719EHC5V5tjrtW57BSjUr8GvsEXmJRbtejUWyPZ2rZMyQdq - volumes: - - ../../registry:/builtins/registry networks: nox: ipv4_address: 10.50.10.10 @@ -47,8 +45,6 @@ services: - --bootstraps=/dns/nox-1/tcp/7771 # 12D3KooWQdpukY3p2DhDfUfDgphAqsGu5ZUrmQ4mcHSGrRag6gQK - -k=2WijTVdhVRzyZamWjqPx4V4iNMrajegNMwNa2PmvPSZV6RRpo5M2fsPWdQr22HVRubuJhhSw8BrWiGt6FPhFAuXy - volumes: - - ../../registry:/builtins/registry networks: nox: ipv4_address: 10.50.10.20 @@ -70,8 +66,6 @@ services: - --bootstraps=/dns/nox-1/tcp/7771 # 12D3KooWRT8V5awYdEZm6aAV9HWweCEbhWd7df4wehqHZXAB7yMZ - -k=2n2wBVanBeu2GWtvKBdrYK9DJAocgG3PrTUXMharq6TTfxqTL4sLdXL9BF23n6rsnkAY5pR9vBtx2uWYDQAiZdrX - volumes: - - ../../registry:/builtins/registry networks: nox: ipv4_address: 10.50.10.30 @@ -93,8 +87,6 @@ services: - --bootstraps=/dns/nox-1/tcp/7771 # 12D3KooWBzLSu9RL7wLP6oUowzCbkCj2AGBSXkHSJKuq4wwTfwof - -k=4zp8ucAikkjB8CmkufYiFBW4QCDUCbQG7yMjviX7W8bMyN5rfChQ2Pi5QCWThrCTbAm9uq5nbFbxtFcNZq3De4dX - volumes: - - ../../registry:/builtins/registry networks: nox: ipv4_address: 10.50.10.40 @@ -116,8 +108,6 @@ services: - --bootstraps=/dns/nox-1/tcp/7771 # 12D3KooWBf6hFgrnXwHkBnwPGMysP3b1NJe5HGtAWPYfwmQ2MBiU - -k=3ry26rm5gkJXvdqRH4FoM3ezWq4xVVsBQF7wtKq4E4pbuaa6p1F84tNqifUS7DdfJL9hs2gcdW64Wc342vHZHMUp - volumes: - - ../../registry:/builtins/registry networks: nox: ipv4_address: 10.50.10.50 @@ -139,8 +129,6 @@ services: - --allow-private-ips # 12D3KooWPisGn7JhooWhggndz25WM7vQ2JmA121EV8jUDQ5xMovJ - -k=5Qh8bB1sF28uLPwr3HTvEksCeC6mAWQvebCfcgv9y6j4qKwSzNKm2tzLUg4nACUEo2KZpBw11gNCnwaAdM7o1pEn - volumes: - - ../../registry:/builtins/registry networks: nox: ipv4_address: 10.50.10.60 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 0000000..4daf8d3 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,71 @@ +name: "e2e" + +on: + pull_request: + paths-ignore: + - "**.md" + - ".github/**" + - "!.github/workflows/e2e.yml" + - "!.github/workflows/snapshot.yml" + types: + - "labeled" + - "synchronize" + - "opened" + - "reopened" + push: + branches: + - "main" + paths-ignore: + - "**.md" + - ".github/**" + - "!.github/workflows/e2e.yml" + - "!.github/workflows/snapshot.yml" + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +jobs: + snapshot: + if: > + github.event_name == 'push' || + contains(github.event.pull_request.labels.*.name, 'e2e') + name: "registry" + uses: ./.github/workflows/snapshot.yml + with: + ref: ${{ github.ref }} + + nox: + needs: + - snapshot + uses: fluencelabs/nox/.github/workflows/build.yml@master + with: + cargo-dependencies: | + [ + { + "package": "registry-distro", + "version": "=${{ needs.snapshot.outputs.cargo-version }}", + "registry": "fluence", + "manifest": "crates/system-services/Cargo.toml" + } + ] + + nox-snapshot: + name: "nox" + needs: + - nox + + uses: fluencelabs/nox/.github/workflows/container.yml@master + with: + image-name: "docker.fluence.dev/registry" + flavour: "minimal" + nox-sha: "${{ needs.nox.outputs.nox-sha }}" + + aqua-tests: + name: "registry" + needs: + - nox-snapshot + uses: ./.github/workflows/tests.yml + with: + ref: ${{ github.ref }} + nox-image: "${{ needs.nox-snapshot.outputs.nox-image }}" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2b72f41..5e18a6f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -118,6 +118,8 @@ jobs: - name: Setup Rust toolchain uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false - name: Setup marine uses: fluencelabs/setup-marine@v1 @@ -144,43 +146,6 @@ jobs: run: npm publish --access public working-directory: aqua - - name: Install ipfs - uses: nahsi/setup-ipfs@v1 - - - name: Create builtin distribution package - run: ./builtin-package/package.sh - - - name: Calculate SHA256 - id: sha - run: | - # Calculate sha256 - du -hs registry.tar.gz - sha256sum registry.tar.gz - sha=($(sha256sum registry.tar.gz)) - echo "sha256=${sha}" >> $GITHUB_OUTPUT - - - name: Upload registry package - if: needs.release-please.outputs.release-created - uses: softprops/action-gh-release@v1 - with: - files: registry.tar.gz - tag_name: ${{ needs.release-please.outputs.tag-name }} - - - name: Update version in node-distro repo - if: needs.release-please.outputs.release-created - uses: benc-uk/workflow-dispatch@v1 - with: - workflow: update_service - repo: fluencelabs/node-distro - ref: "main" - token: ${{ secrets.PERSONAL_TOKEN }} - inputs: '{ - "name": "registry", - "version": "${{ needs.release-please.outputs.version }}", - "url": "https://github.com/fluencelabs/registry/releases/download/${{ needs.release-please.outputs.tag-name }}/registry.tar.gz", - "sha256": "${{ steps.sha.outputs.sha256 }}" - }' - - name: Import secrets uses: hashicorp/vault-action@v2.4.3 with: diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 877b848..eddf760 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -26,11 +26,6 @@ concurrency: cancel-in-progress: true jobs: - registry: - uses: ./.github/workflows/tests.yml - with: - ref: ${{ github.ref }} - cargo: name: "registry / Run cargo tests" runs-on: builder @@ -70,3 +65,20 @@ jobs: NEXTEST_RETRIES: 2 NEXTEST_TEST_THREADS: 1 run: cargo nextest run --release --all-features --no-fail-fast + + lints: + name: Lints + runs-on: ubuntu-latest + + steps: + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + + - name: Run cargo fmt + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all --manifest-path service/Cargo.toml -- --check diff --git a/.github/workflows/snapshot.yml b/.github/workflows/snapshot.yml new file mode 100644 index 0000000..ff96cc0 --- /dev/null +++ b/.github/workflows/snapshot.yml @@ -0,0 +1,113 @@ +name: Build snapshot + +on: + workflow_call: + inputs: + cargo-dependencies: + description: "Cargo dependencies map" + type: string + default: "null" + ref: + description: "git ref to checkout to" + type: string + default: "master" + snapshot: + description: "Whether to publish snapshots" + type: boolean + default: true + outputs: + cargo-version: + description: "Cargo snapshot version" + value: ${{ jobs.snapshot.outputs.version }} + +jobs: + snapshot: + name: "Build crate" + runs-on: builder + timeout-minutes: 60 + + outputs: + version: "${{ steps.snapshot.outputs.version }}" + + permissions: + contents: read + id-token: write + + steps: + - uses: actions/checkout@v3 + with: + repository: fluencelabs/registry + ref: ${{ inputs.ref }} + + - name: Setup rust toolchain + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + cache: false + + - name: Set dependencies + if: inputs.cargo-dependencies != 'null' + uses: fluencelabs/github-actions/cargo-set-dependency@main + with: + dependencies: ${{ inputs.cargo-dependencies }} + path: service/ + + - name: Download marine artifact + id: marine + uses: actions/download-artifact@v3 + continue-on-error: true + with: + name: marine + path: ~/.local/bin + + - name: Make marine executable + if: steps.marine.outcome == 'success' + run: chmod +x ~/.local/bin/marine + + - name: Setup marine + if: steps.marine.outcome == 'failure' + uses: fluencelabs/setup-marine@v1 + + - name: Import secrets + if: inputs.snapshot == true + uses: hashicorp/vault-action@v2.5.0 + with: + url: https://vault.fluence.dev + path: jwt/github + role: ci + method: jwt + jwtGithubAudience: "https://github.com/fluencelabs" + jwtTtl: 300 + exportToken: false + secrets: | + kv/cargo-registry/users/ci token | CARGO_REGISTRIES_FLUENCE_TOKEN + + - name: Setup node + uses: actions/setup-node@v3 + with: + node-version: "18" + registry-url: "https://registry.npmjs.org" + cache-dependency-path: "aqua/package-lock.json" + cache: "npm" + + - run: npm i + working-directory: aqua + + - name: Install cargo-workspaces + uses: baptiste0928/cargo-install@v2.0.0 + with: + crate: cargo-workspaces + version: v0.2.37 + + - name: Generate snapshot version + id: version + uses: fluencelabs/github-actions/generate-snapshot-id@main + + - name: Build distro + run: ./build.sh + + - name: Publish crate snapshots + id: snapshot + uses: fluencelabs/github-actions/cargo-publish-snapshot@main + with: + id: ${{ steps.version.outputs.id }} + path: distro diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index f4b4278..1fe492c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -120,15 +120,6 @@ jobs: - name: Build distro run: ./build.sh - - name: Install ipfs - uses: nahsi/setup-ipfs@v1 - - - name: Create builtin distribution package - run: ./builtin-package/package.sh - - - name: Unpack registry package - run: tar xvf registry.tar.gz - - name: Login to DockerHub uses: docker/login-action@v2 with: @@ -156,17 +147,6 @@ jobs: run: pip install -r requirements.txt working-directory: aqua-tests - # - name: Wait for registry to be deployed - # uses: nick-fields/retry@v2 - # with: - # timeout_seconds: 30 - # max_attempts: 5 - # shell: bash - # command: ${{ github.workspace }}/.github/check-registry.sh - # - - name: Temporarely wait with sleep - run: sleep 20 - - name: Install flox dependencies env: FLUENCE_USER_DIR: "${{ github.workspace }}/tmp/.fluence" diff --git a/builtin-package/package.sh b/builtin-package/package.sh deleted file mode 100755 index 6986a4a..0000000 --- a/builtin-package/package.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env bash -set -o pipefail -o nounset -o errexit - -# set current working directory to script directory to run script from everywhere -cd "$(dirname "$0")" -PACKAGE_DIR="$(pwd)/../registry" -SCHEDULED="$PACKAGE_DIR/scheduled" - -( - rm -rf $PACKAGE_DIR - mkdir -p $PACKAGE_DIR -) -( - echo "*** compile scheduled scripts ***" - cd ../aqua - npx fluence --version - npx fluence aqua --no-relay --air -i ./registry-scheduled-scripts.aqua -o "$SCHEDULED" -) - -( - echo "*** copy wasm files ***" - cd ../service - cp artifacts/*.wasm "$PACKAGE_DIR" -) - -REGISTRY_CID=$(ipfs add -q --only-hash --cid-version=1 --chunker=size-262144 $PACKAGE_DIR/registry.wasm) -SQLITE_CID=$(ipfs add -q --only-hash --cid-version=1 --chunker=size-262144 $PACKAGE_DIR/sqlite3.wasm) -mv $PACKAGE_DIR/registry.wasm "$PACKAGE_DIR"/"$REGISTRY_CID".wasm -mv $PACKAGE_DIR/sqlite3.wasm "$PACKAGE_DIR"/"$SQLITE_CID".wasm -cp registry_config.json "$PACKAGE_DIR"/"$REGISTRY_CID"_config.json -cp sqlite3_config.json "$PACKAGE_DIR"/"$SQLITE_CID"_config.json - -# write blueprint.json -echo "{}" | jq --arg registry_cid "$REGISTRY_CID" --arg sqlite_cid "$SQLITE_CID" '{"name": "registry", "dependencies":[{"/":$sqlite_cid},{"/":$registry_cid}]}' > "$PACKAGE_DIR/blueprint.json" - - -( - echo "*** create builtin distribution package ***" - cd .. - tar -f registry.tar.gz -zcv ./registry -) - -echo "*** done ***" diff --git a/builtin-package/registry_config.json b/builtin-package/registry_config.json deleted file mode 100644 index af817ae..0000000 --- a/builtin-package/registry_config.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "name": "registry", - "max_heap_size": "8 MiB", - "preopened_files": ["/tmp"], - "mapped_dirs": { - "tmp": "./tmp" - } -} diff --git a/builtin-package/sqlite3_config.json b/builtin-package/sqlite3_config.json deleted file mode 100644 index c3f8f83..0000000 --- a/builtin-package/sqlite3_config.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "name": "sqlite3", - "max_heap_size": "64 MiB" -}