feat(npm-aqua-compiler): create package (#401)

* Add npm-aqua-compiler package

* Release new package

* Remove noUncheckedIndexedAccess from tsconfig.json

* Fix a test script

* Fix length checks

* Fix

* Update error description

* Try to choose a nicer err message

* New import format and API

* Fix error message

* Improve test

* Don't add empty string key when globalImports prop is empty

* Fix exports

packages/core/npm-aqua-compiler/package.json

@@ -0,0 +1,29 @@
+{
+  "type": "module",
+  "name": "@fluencelabs/npm-aqua-compiler",
+  "version": "0.0.0",
+  "description": "Tool for converting npm imports to aqua compiler input",
+  "types": "./dist/imports.d.ts",
+  "exports": {
+    ".": "./dist/imports.js"
+  },
+  "scripts": {
+    "test": "npm i --prefix ./test/transitive-deps/project && vitest run",
+    "build": "tsc"
+  },
+  "files": [
+    "dist"
+  ],
+  "keywords": [],
+  "author": "Fluence Labs",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "@npmcli/arborist": "^7.2.1",
+    "treeverse": "3.0.0"
+  },
+  "devDependencies": {
+    "@types/npmcli__arborist": "5.6.5",
+    "@types/treeverse": "3.0.4",
+    "vitest": "0.34.6"
+  }
+}

packages/core/npm-aqua-compiler/src/imports.spec.ts

@@ -0,0 +1,115 @@
+/**
+ * Copyright 2023 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { join } from "path";
+import { fileURLToPath } from "url";
+
+import { assert, describe, expect, it } from "vitest";
+
+import { gatherImportsFromNpm } from "./imports.js";
+
+describe("imports", () => {
+  /**
+   * NOTE: This test expects that `npm i` is run
+   * inside `./__test__/data/transitive-deps/project` folder
+   */
+  it("should resolve transitive dependencies", async () => {
+    const npmProjectDirPath = "./test/transitive-deps/project";
+    const aquaToCompileDirPath = "./test/transitive-deps/aqua-project";
+    const globalImports = ["./.fluence/aqua"];
+
+    const expectedResolution: Record<
+      string,
+      Record<string, string[] | string>
+    > = {
+      [aquaToCompileDirPath]: {
+        "": globalImports,
+        A: "./A",
+        B: "./B",
+      },
+      "./A": {
+        C: "./C",
+        D: "./D",
+      },
+      "./B": {
+        C: "./B/C",
+        D: "./B/D",
+      },
+      "./C": {
+        D: "./C/D",
+      },
+      "./B/C": {
+        D: "./B/C/D",
+      },
+    };
+
+    const prefix = join(
+      fileURLToPath(new URL("./", import.meta.url)),
+      "..",
+      "test",
+      "transitive-deps",
+      "project",
+    );
+
+    const buildResolutionKey = (str: string) => {
+      return (
+        "./" +
+        str
+          .slice(prefix.length)
+          .split("/node_modules/")
+          .filter(Boolean)
+          .join("/")
+      );
+    };
+
+    const imports = await gatherImportsFromNpm({
+      npmProjectDirPath,
+      aquaToCompileDirPath,
+      globalImports,
+    });
+
+    expect(Object.keys(imports).length).toBe(
+      Object.keys(expectedResolution).length,
+    );
+
+    Object.entries(imports).forEach(([key, value]) => {
+      const resolutionKey =
+        key === aquaToCompileDirPath ? key : buildResolutionKey(key);
+
+      const resolutionValues = expectedResolution[resolutionKey];
+
+      assert(resolutionValues);
+
+      expect(Object.keys(value).length).toBe(
+        Object.keys(resolutionValues).length,
+      );
+
+      for (const [dep, path] of Object.entries(value)) {
+        if (Array.isArray(path)) {
+          expect(dep).toBe("");
+          expect(expectedResolution[resolutionKey]).toHaveProperty(dep, path);
+
+          continue;
+        }
+
+        expect(expectedResolution[resolutionKey]).toHaveProperty(
+          dep,
+          buildResolutionKey(path),
+        );
+      }
+    });
+  });
+});

packages/core/npm-aqua-compiler/src/imports.ts

@@ -0,0 +1,92 @@
+/**
+ * Copyright 2023 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import Arborist from "@npmcli/arborist";
+import { breadth } from "treeverse";
+
+export interface GatherImportsArg {
+  npmProjectDirPath: string;
+  aquaToCompileDirPath?: string; // Default: npmProjectDirPath
+  globalImports?: string[];
+}
+
+export type GatherImportsResult = Record<
+  string,
+  Record<string, string[] | string>
+>;
+
+export async function gatherImportsFromNpm({
+  npmProjectDirPath,
+  aquaToCompileDirPath,
+  globalImports = [],
+}: GatherImportsArg): Promise<GatherImportsResult> {
+  const arborist = new Arborist({ path: npmProjectDirPath });
+  const tree = await arborist.loadActual();
+
+  /**
+   * Traverse dependency tree to construct map
+   * (real path of a package) -> (real paths of its immediate dependencies)
+   */
+  const result: GatherImportsResult = {};
+
+  breadth({
+    tree,
+    getChildren(node) {
+      const deps: Arborist.Node[] = [];
+
+      for (const edge of node.edgesOut.values()) {
+        // Skip dependencies that are not installed.
+
+        // Looks like Arborist type is incorrect here, so it's possible to have null here
+        // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
+        if (edge.to === null) {
+          continue;
+        }
+
+        // NOTE: Any errors in edge are ignored.
+        const dep = edge.to;
+
+        // Gather dependencies to traverse them.
+        deps.push(dep);
+
+        // Root node should have top-level property pointed to aqua dependency folder
+        if (node.isRoot) {
+          const aquaDepPath = aquaToCompileDirPath ?? npmProjectDirPath;
+
+          result[aquaDepPath] = {
+            ...(result[aquaDepPath] ??
+              (globalImports.length > 0
+                ? {
+                    "": globalImports,
+                  }
+                : {})),
+            [dep.name]: dep.realpath,
+          };
+        } else {
+          // Gather dependencies real paths.
+          result[node.realpath] = {
+            ...(result[node.realpath] ?? {}),
+            [dep.name]: dep.realpath,
+          };
+        }
+      }
+
+      return deps;
+    },
+  });
+
+  return result;
+}

packages/core/npm-aqua-compiler/test/transitive-deps/project/main.aqua

@@ -0,0 +1,8 @@
+use "B.aqua"
+
+func versionAC() -> string:
+    <- A.versionC()
+
+func versionBC() -> string:
+    <- B.versionC()
+

packages/core/npm-aqua-compiler/test/transitive-deps/project/package-lock.json

@@ -0,0 +1,70 @@
+{
+  "name": "project",
+  "version": "0.1.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "project",
+      "version": "0.1.0",
+      "dependencies": {
+        "A": "file:../A-0.1.0.tgz",
+        "B": "file:../B-0.1.0.tgz"
+      }
+    },
+    "node_modules/A": {
+      "version": "0.1.0",
+      "resolved": "file:../A-0.1.0.tgz",
+      "integrity": "sha512-H0nhbQVQxPm3VwXYiePLJ0oyHa2FxPNNPjOcTdz3YWvIoE0/dZFJ1yrqig7fkrETYEYfLuVJaN0yg1BX/HAScg==",
+      "dependencies": {
+        "C": "file:./C-0.2.0.tgz",
+        "D": "file:./D-0.1.0.tgz"
+      }
+    },
+    "node_modules/B": {
+      "version": "0.1.0",
+      "resolved": "file:../B-0.1.0.tgz",
+      "integrity": "sha512-u6n6V5KlxIN/GwRQt82gZQAPwYi0OzqQ2wr8ufmygreLK3fPIfO49f13qagbGXaYiRxN9effXaPqZlMIyTygng==",
+      "dependencies": {
+        "C": "file:./C-0.1.0.tgz",
+        "D": "file:./D-0.2.0.tgz"
+      }
+    },
+    "node_modules/B/node_modules/C": {
+      "version": "0.1.0",
+      "resolved": "file:../C-0.1.0.tgz",
+      "integrity": "sha512-zvzWgHLm+ptWwysP+dJItnogVSca/jvHegWmwi6NmmHFO/wTqlGrMPnC2dEkpXDJBU4X1bUjevFh0q3Xe9e0MA==",
+      "dependencies": {
+        "D": "file:./D-0.1.0.tgz"
+      }
+    },
+    "node_modules/B/node_modules/C/node_modules/D": {
+      "version": "0.1.0",
+      "resolved": "file:../D-0.1.0.tgz",
+      "integrity": "sha512-1rlKmuyzHSGTt9tBhEBY3j7gZvMBg0LnZMogZSucmX4gww4l0+HPQwBIPjJpqOspE2ND8PcLymQoiw06xWXn0g=="
+    },
+    "node_modules/B/node_modules/D": {
+      "version": "0.2.0",
+      "resolved": "file:../D-0.2.0.tgz",
+      "integrity": "sha512-7h1TUU8j60q6BZ0Wq/xDZOUf6iS0S4SgL/lgXOaiyxN76q7ld8Rx/qIxtGKmrWh65v5cjvAG5asbMEkXb6DuYQ=="
+    },
+    "node_modules/C": {
+      "version": "0.2.0",
+      "resolved": "file:../C-0.2.0.tgz",
+      "integrity": "sha512-uNqb8p69kuombZsb3xI/ygeL94WHpwkGR9/GRWgdg+01iKGsRMaZgL5up0UG7D/9DW7NQBozZG8ITPQ8DLgwSQ==",
+      "dependencies": {
+        "D": "file:./D-0.2.0.tgz"
+      }
+    },
+    "node_modules/C/node_modules/D": {
+      "version": "0.2.0",
+      "resolved": "file:../D-0.2.0.tgz",
+      "integrity": "sha512-7h1TUU8j60q6BZ0Wq/xDZOUf6iS0S4SgL/lgXOaiyxN76q7ld8Rx/qIxtGKmrWh65v5cjvAG5asbMEkXb6DuYQ=="
+    },
+    "node_modules/D": {
+      "version": "0.1.0",
+      "resolved": "file:../D-0.1.0.tgz",
+      "integrity": "sha512-1rlKmuyzHSGTt9tBhEBY3j7gZvMBg0LnZMogZSucmX4gww4l0+HPQwBIPjJpqOspE2ND8PcLymQoiw06xWXn0g=="
+    }
+  }
+}

packages/core/npm-aqua-compiler/test/transitive-deps/project/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "project",
+  "version": "0.1.0",
+  "dependencies": {
+    "A": "file:../A-0.1.0.tgz",
+    "B": "file:../B-0.1.0.tgz"
+  }
+}

packages/core/npm-aqua-compiler/tsconfig.json

@@ -0,0 +1,9 @@
+{
+  "extends": "../../../tsconfig.json",
+  "compilerOptions": {
+    "resolveJsonModule": true,
+    "outDir": "./dist"
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist", "src/**/__test__"]
+}