LNG-40 AIR validation on compilation (#546)

backend/air/src/main/scala/aqua/backend/air/AirBackend.scala

@@ -1,6 +1,6 @@
 package aqua.backend.air
 
-import aqua.backend.{Backend, Generated, Version}
+import aqua.backend.{AirString, Backend, Generated, Version}
 import aqua.res.AquaRes
 import cats.syntax.show.*
 
@@ -17,10 +17,9 @@ object AirBackend extends Backend {
                   |
                   |""".stripMargin
 
-    aqua.funcs.toList
-      .map{ fr =>
-        val airStr = docs + FuncAirGen(fr).generate.show
-        Generated("." + fr.funcName + ext, airStr)
+    aqua.funcs.toList.map { fr =>
+      val airStr = FuncAirGen(fr).generate.show
+      Generated("." + fr.funcName + ext, docs + airStr, AirString(fr.funcName, airStr) :: Nil)
     }
   }
 }

backend/src/main/scala/aqua/backend/AirString.scala

@@ -0,0 +1,3 @@
+package aqua.backend
+
+case class AirString(name: String, air: String)

backend/src/main/scala/aqua/backend/Generated.scala

@@ -6,4 +6,4 @@ package aqua.backend
  * @param suffix extension or another info that will be added to a resulted file
  * @param content compiled code
  */
-case class Generated(suffix: String, content: String)
+case class Generated(suffix: String, content: String, air: List[AirString])

backend/ts/src/main/scala/aqua/backend/OutputFile.scala

@@ -6,22 +6,28 @@ import aqua.res.AquaRes
 
 case class OutputFile(res: AquaRes) {
 
-  def generate(types: Types, isJs: Boolean, isCommonJS: Boolean): String = {
+  def generate(types: Types, isJs: Boolean, isCommonJS: Boolean): (List[AirString], String) = {
     import types.*
     val services = res.services
       .map(s => OutputService(s, types))
       .map(_.generate)
       .toList
       .mkString("\n\n")
-    val functions =
-      res.funcs.map(f => OutputFunc(f, types)).map(_.generate).toList.mkString("\n\n")
+    val scripts =
+      res.funcs.map(f => OutputFunc(f, types)).map(_.generate)
+
+    val (airs, functions) = scripts.toList.unzip
+
+    (
+      airs,
       s"""${Header.header(isJs, isCommonJS)}
          |
          |// Services
          |$services
          |// Functions
-       |$functions
+         |${functions.mkString("\n\n")}
          |""".stripMargin
+    )
   }
 
 }

backend/ts/src/main/scala/aqua/backend/OutputFunc.scala

@@ -22,13 +22,15 @@ case class OutputFunc(func: FuncRes, types: Types) {
   import funcTypes.*
   import TypeDefinition.*
 
-  def generate: String = {
+  def generate: (AirString, String) = {
     val tsAir = FuncAirGen(func).generate
     val codeLeftSpace = " " * 20
 
     val script = tsAir.show.linesIterator.map(codeLeftSpace + _).mkString("\n")
     val funcDef = FunctionDef(func)
 
+    (
+      AirString(func.funcName, script),
       s"""${funcTypes.generate}
          |export function ${func.funcName}(${typed("...args", "any")}) {
          |
@@ -41,6 +43,7 @@ case class OutputFunc(func: FuncRes, types: Types) {
          |        script
          |    )
          |}""".stripMargin
+    )
   }
 
 }

backend/ts/src/main/scala/aqua/backend/js/JavaScriptBackend.scala

@@ -27,12 +27,13 @@ case class JavaScriptBackend(isCommonJS: Boolean) extends Backend {
                   |$functions
                   |""".stripMargin
 
-    Generated(tsExt, body)
+    Generated(tsExt, body, Nil)
   }
 
   override def generate(res: AquaRes): Seq[Generated] =
     if (res.isEmpty) Nil
     else {
-      Generated(ext, OutputFile(res).generate(EmptyTypes, true, isCommonJS)):: typesFile(res) :: Nil
+      val (airs, script) = OutputFile(res).generate(EmptyTypes, true, isCommonJS)
+      Generated(ext, script, airs) :: typesFile(res) :: Nil
     }
 }

backend/ts/src/main/scala/aqua/backend/ts/TypeScriptBackend.scala

@@ -9,5 +9,9 @@ object TypeScriptBackend extends Backend {
   val ext = ".ts"
 
   override def generate(res: AquaRes): Seq[Generated] =
-    if (res.isEmpty) Nil else Generated(ext, OutputFile(res).generate(TypeScriptTypes, false, false)) :: Nil
+    if (res.isEmpty) Nil
+    else {
+      val (airs, script) = OutputFile(res).generate(TypeScriptTypes, false, false)
+      Generated(ext, script, airs) :: Nil
+    }
 }

cli/.js/src/main/scala/aqua/air/AirValidation.scala

@@ -0,0 +1,47 @@
+package aqua.air
+
+import aqua.backend.AirString
+import aqua.js.Fluence
+import cats.data.Validated.{invalid, validNec}
+import cats.data.{Chain, NonEmptyChain, ValidatedNec}
+import cats.effect.Async
+import cats.syntax.traverse.*
+import cats.syntax.functor.*
+import cats.syntax.applicative.*
+import cats.syntax.flatMap.*
+import cats.syntax.functor.*
+
+import scala.concurrent.ExecutionContext
+import scala.scalajs.js
+
+object AirValidation {
+
+  // HACK: memoize doesn't work in scalajs, so, call this function once before `validate`
+  def init[F[_]: Async](): F[Unit] = {
+    Async[F].fromFuture(Fluence.start(js.undefined).toFuture.pure[F]).as(())
+  }
+
+  def validate[F[_]: Async](
+    airs: List[AirString]
+  ): F[ValidatedNec[String, Unit]] =
+    Async[F].fromFuture {
+
+      Async[F].executionContext.map { implicit ec =>
+        for {
+          statuses <- airs
+            .map(a => Fluence.getPeer().internals.parseAst(a.air).toFuture.map(s => (a.name, s)))
+            .sequence
+        } yield {
+          val errors = NonEmptyChain.fromSeq(statuses.filterNot(_._2.success))
+          errors.map { errs =>
+            val errorsStrs = errs.map { case (fName, status) =>
+              s"Cannot compile AIR for '$fName' function: ${js.JSON.stringify(status.data)}\n\n" +
+              "This is unexpected error. Please, dump your Aqua code and make an issue here https://github.com/fluencelabs/aqua/issues."
+            }
+            invalid(errorsStrs)
+          }.getOrElse(validNec(()))
+        }
+      }
+    }
+
+}

cli/.js/src/main/scala/aqua/js/FluenceJsTypes.scala

@@ -217,6 +217,15 @@ case class PeerConfig(
   debug: js.UndefOr[Debug]
 )
 
+trait AstStatus extends js.Object {
+  def success: Boolean
+  def data: js.Any
+}
+
+trait Internals extends js.Object {
+  def parseAst(air: String): js.Promise[AstStatus]
+}
+
 /**
  * This class implements the Fluence protocol for javascript-based environments.
  * It provides all the necessary features to communicate with Fluence network
@@ -226,6 +235,7 @@ case class PeerConfig(
 class FluencePeer extends js.Object {
   def getStatus(): PeerStatus = js.native
   def stop(): js.Promise[Unit] = js.native
+  def internals: Internals = js.native
 }
 
 object V3 {
@@ -258,7 +268,7 @@ object FluenceUtils {
 @js.native
 @JSImport("@fluencelabs/fluence", "Fluence")
 object Fluence extends js.Object {
-  def start(config: PeerConfig): js.Promise[js.Any] = js.native
+  def start(config: js.UndefOrOps[PeerConfig]): js.Promise[js.Any] = js.native
   def stop(): js.Promise[js.Any] = js.native
   def getPeer(): FluencePeer = js.native
   def getStatus(): PeerStatus = js.native

cli/.js/src/main/scala/aqua/run/FuncCaller.scala

@@ -19,7 +19,8 @@ import cats.syntax.show.*
 import scala.concurrent.duration.Duration
 import scala.concurrent.{ExecutionContext, Future, Promise, TimeoutException}
 import scala.scalajs.js
-import scala.scalajs.js.{JSON, JavaScriptException, timers}
+import scala.scalajs.js.JSConverters.*
+import scala.scalajs.js.{timers, JSON, JavaScriptException}
 
 object FuncCaller {
 
@@ -57,12 +58,14 @@ object FuncCaller {
             )
             _ <- Fluence
               .start(
+                Some(
                   PeerConfig(
                     config.common.multiaddr,
                     config.common.timeout.toMillis.toInt : js.UndefOr[Int],
                     keyPair,
                     Debug(printParticleId = config.common.flags.verbose, marineLogLevel = logLevel)
                   )
+                ).orUndefined
               )
               .toFuture
             _ =
@@ -90,6 +93,7 @@ object FuncCaller {
             // use a timeout in finisher if we have an async function and it hangs on node's side
             finisher = setTimeout(name, finisherFuture, config.common.timeout)
             _ <- finisher
+            _ <- Fluence.stop().toFuture
           } yield validNec(()))
             .recover(handleFuncCallErrors(name, config.common.timeout))
             .pure[F]

cli/.js/src/main/scala/aqua/run/RunCommand.scala

@@ -53,10 +53,6 @@ object RunCommand extends Logging {
     }.getOrElse(KeyPair.randomEd25519().toFuture)
   }
 
-  private def findFunction(contexts: Chain[AquaContext], funcName: String): Option[FuncArrow] =
-    contexts
-      .collectFirstSome(_.allFuncs.get(funcName))
-
   /**
    * Runs a function that is located in `input` file with FluenceJS SDK. Returns no output
    * @param func

cli/.js/src/main/scala/aqua/script/ScriptOpts.scala

@@ -105,10 +105,6 @@ object ScriptOpts extends Logging {
     Opts
       .option[String]("script-id", "Script id to remove", "c")
 
-  private def findFunction(contexts: Chain[AquaContext], funcName: String): Option[FuncArrow] =
-    contexts
-      .collectFirstSome(_.allFuncs.get(funcName))
-
   def generateAir(callable: FuncArrow, transformConfig: TransformConfig): String = {
     val funcRes = Transform.funcRes(callable, transformConfig).value
     AirGen(funcRes.body).generate.show

cli/.jvm/src/main/scala/aqua/Test.scala

@@ -26,7 +26,8 @@ object Test extends IOApp.Simple {
           List(Path("./aqua")),
           Option(Path("./target")),
           TypeScriptBackend,
-          TransformConfig(wrapWithXor = false)
+          TransformConfig(wrapWithXor = false),
+          false
         )
         .map {
           case Validated.Invalid(errs) =>

cli/.jvm/src/main/scala/aqua/air/AirValidation.scala

@@ -0,0 +1,15 @@
+package aqua.air
+import aqua.backend.AirString
+import cats.data.ValidatedNec
+import cats.effect.Async
+import cats.data.Validated.validNec
+
+import scala.concurrent.ExecutionContext
+
+object AirValidation {
+
+  def init[F[_]: Async](): F[Unit] = Async[F].pure(())
+
+  def validate[F[_]: Async](airs: List[AirString]): F[ValidatedNec[String, Unit]] = Async[F].pure(validNec(()))
+
+}

cli/.jvm/src/test/scala/SourcesSpec.scala

@@ -149,7 +149,7 @@ class SourcesSpec extends AsyncFlatSpec with Matchers {
       content = "some random content"
       compiled = AquaCompiled[FileModuleId](
           FileModuleId(filePath),
-        Seq(Generated("_hey.custom", content)),
+          Seq(Generated("_hey.custom", content, Nil)),
           1,
           1
         )

cli/.jvm/src/test/scala/WriteFileSpec.scala

@@ -21,7 +21,7 @@ class WriteFileSpec extends AnyFlatSpec with Matchers {
 
     val bc = TransformConfig()
     AquaPathCompiler
-      .compileFilesTo[IO](src, List.empty, Option(targetTs), TypeScriptBackend, bc)
+      .compileFilesTo[IO](src, List.empty, Option(targetTs), TypeScriptBackend, bc, false)
       .unsafeRunSync()
       .leftMap { err =>
         println(err)
@@ -33,7 +33,7 @@ class WriteFileSpec extends AnyFlatSpec with Matchers {
     Files[IO].deleteIfExists(targetTsFile).unsafeRunSync()
 
     AquaPathCompiler
-      .compileFilesTo[IO](src, List.empty, Option(targetJs), JavaScriptBackend(false), bc)
+      .compileFilesTo[IO](src, List.empty, Option(targetJs), JavaScriptBackend(false), bc, false)
       .unsafeRunSync()
       .leftMap { err =>
         println(err)
@@ -45,7 +45,7 @@ class WriteFileSpec extends AnyFlatSpec with Matchers {
     Files[IO].deleteIfExists(targetJsFile).unsafeRunSync()
 
     AquaPathCompiler
-      .compileFilesTo[IO](src, List.empty, Option(targetAir), AirBackend, bc)
+      .compileFilesTo[IO](src, List.empty, Option(targetAir), AirBackend, bc, false)
       .unsafeRunSync()
       .leftMap { err =>
         println(err)

cli/src/main/scala/aqua/AppOpts.scala

@@ -122,6 +122,12 @@ object AppOpts {
       }
       .withDefault(List.empty)
 
+  val noAirValidation: Opts[Boolean] =
+    Opts
+      .flag("no-air-validation", "Don't parse and validate AIR after compilation")
+      .map(_ => false)
+      .withDefault(true)
+
   val compileToAir: Opts[Boolean] =
     Opts
       .flag("air", "Generate .air file instead of .ts", "a")

cli/src/main/scala/aqua/AquaCli.scala

@@ -98,7 +98,8 @@ object AquaCli extends IOApp with Logging {
       FluenceOpts.logLevelOpt,
       constantOpts,
       dryOpt,
-      scriptOpt
+      scriptOpt,
+      noAirValidation
     ).mapN {
       case (
             inputF,
@@ -113,7 +114,8 @@ object AquaCli extends IOApp with Logging {
             logLevel,
             constants,
             isDryRun,
-            isScheduled
+            isScheduled,
+            disableAirValidation
           ) =>
         val toAir = toAirOp || isScheduled
         val noXor = noXorOp || isScheduled
@@ -156,7 +158,8 @@ object AquaCli extends IOApp with Logging {
                         imports,
                         resultOutput,
                         targetToBackend(target),
-                        bc
+                        bc,
+                        disableAirValidation
                       )
                   }
                 }

cli/src/main/scala/aqua/AquaPathCompiler.scala

@@ -1,9 +1,18 @@
 package aqua
 
 import aqua.backend.{Backend, Generated}
-import aqua.compiler.{AquaCompiled, AquaCompiler, AquaCompilerConf, AquaError, CompilerAPI}
+import aqua.compiler.{
+  AirValidator,
+  AquaCompiled,
+  AquaCompiler,
+  AquaCompilerConf,
+  AquaError,
+  CompilerAPI
+}
 import aqua.files.{AquaFileSources, FileModuleId}
 import aqua.io.*
+import aqua.air.AirValidation
+import aqua.backend.AirString
 import aqua.model.AquaContext
 import aqua.model.transform.TransformConfig
 import aqua.model.transform.Transform
@@ -14,6 +23,7 @@ import aqua.parser.{Ast, LexerError, Parser}
 import aqua.raw.ConstantRaw
 import aqua.res.AquaRes
 import cats.data.*
+import cats.effect.Async
 import cats.parse.LocationMap
 import cats.syntax.applicative.*
 import cats.syntax.functor.*
@@ -22,6 +32,7 @@ import cats.syntax.show.*
 import cats.{~>, Applicative, Eval, Monad, Show}
 import fs2.io.file.{Files, Path}
 import scribe.Logging
+import cats.data.Validated.validNec
 
 object AquaPathCompiler extends Logging {
 
@@ -33,21 +44,38 @@ object AquaPathCompiler extends Logging {
    * @param transformConfig transformation configuration for a model
    * @return errors or result messages
    */
-  def compileFilesTo[F[_]: AquaIO: Monad: Files](
+  def compileFilesTo[F[_]: AquaIO: Monad: Files: Async](
     srcPath: Path,
     imports: List[Path],
     targetPath: Option[Path],
     backend: Backend,
-    transformConfig: TransformConfig
+    transformConfig: TransformConfig,
+    disableAirValidation: Boolean
   ): F[ValidatedNec[String, Chain[String]]] = {
     import ErrorRendering.showError
     (for {
       prelude <- Prelude.init()
       sources = new AquaFileSources[F](srcPath, imports ++ prelude.importPaths)
+      validator =
+        if (disableAirValidation) {
+          new AirValidator[F] {
+            override def init(): F[Unit] = Applicative[F].pure(())
+            override def validate(airs: List[AirString]): F[ValidatedNec[String, Unit]] =
+              Applicative[F].pure(validNec(()))
+          }
+        } else {
+          new AirValidator[F] {
+            override def init(): F[Unit] = AirValidation.init[F]()
+            override def validate(
+              airs: List[AirString]
+            ): F[ValidatedNec[String, Unit]] = AirValidation.validate[F](airs)
+          }
+        }
       compiler <- CompilerAPI
         .compileTo[F, AquaFileError, FileModuleId, FileSpan.F, String](
           sources,
           SpanParser.parser,
+          validator,
           new Backend.Transform:
             override def transform(ex: AquaContext): AquaRes =
               Transform.contextRes(ex, transformConfig)

cli/src/main/scala/aqua/ErrorRendering.scala

@@ -65,6 +65,8 @@ object ErrorRendering {
       }
     case SourcesErr(err) =>
       Console.RED + err.showForConsole + Console.RESET
+    case AirValidationError(errors) =>
+      Console.RED + errors.toChain.toList.mkString("\n") + Console.RESET
     case ResolveImportsErr(_, token, err) =>
       val span = token.unit._1
       showForConsole("Cannot resolve imports", span, err.showForConsole :: Nil)

compiler/src/main/scala/aqua/compiler/AirValidator.scala

@@ -0,0 +1,13 @@
+package aqua.compiler
+
+import aqua.backend.AirString
+import cats.data.ValidatedNec
+
+
+trait AirValidator[F[_]] {
+  def init(): F[Unit]
+
+  def validate(
+    airs: List[AirString]
+  ): F[ValidatedNec[String, Unit]]
+}

compiler/src/main/scala/aqua/compiler/AquaError.scala

@@ -3,6 +3,7 @@ package aqua.compiler
 import aqua.parser.ParserError
 import aqua.parser.lexer.Token
 import aqua.semantics.SemanticError
+import cats.data.NonEmptyChain
 
 trait AquaError[I, E, S[_]]
 case class SourcesErr[I, E, S[_]](err: E) extends AquaError[I, E, S]
@@ -16,3 +17,4 @@ case class CycleError[I, E, S[_]](modules: List[I]) extends AquaError[I, E, S]
 
 case class CompileError[I, E, S[_]](err: SemanticError[S]) extends AquaError[I, E, S]
 case class OutputError[I, E, S[_]](compiled: AquaCompiled[I], err: E) extends AquaError[I, E, S]
+case class AirValidationError[I, E, S[_]](errors: NonEmptyChain[String]) extends AquaError[I, E, S]

compiler/src/main/scala/aqua/compiler/CompilerAPI.scala

@@ -1,6 +1,6 @@
 package aqua.compiler
 
-import aqua.backend.Backend
+import aqua.backend.{AirString, Backend}
 import aqua.linker.{AquaModule, Linker, Modules}
 import aqua.model.AquaContext
 import aqua.parser.lift.{LiftParser, Span}
@@ -12,7 +12,7 @@ import aqua.semantics.{CompilerState, LspSemantics, RawSemantics, Semantics}
 import aqua.semantics.header.{HeaderHandler, HeaderSem}
 import aqua.semantics.lsp.LspContext
 import cats.data.*
-import cats.data.Validated.{Invalid, Valid, validNec}
+import cats.data.Validated.{validNec, Invalid, Valid, invalid}
 import cats.parse.Parser0
 import cats.syntax.applicative.*
 import cats.syntax.flatMap.*
@@ -134,34 +134,53 @@ object CompilerAPI extends Logging {
   def compile[F[_]: Monad, E, I: Order, S[_]: Comonad](
     sources: AquaSources[F, E, I],
     parser: I => String => ValidatedNec[ParserError[S], Ast[S]],
+    airValidator: AirValidator[F],
     backend: Backend.Transform,
     config: AquaCompilerConf
   ): F[ValidatedNec[AquaError[I, E, S], Chain[AquaCompiled[I]]]] = {
     val compiler = getAquaCompiler[F, E, I, S](config)
 
-    compiler
+    for {
+      compiledV <- compiler
           .compileRaw(sources, parser)
           .map(_.andThen { filesWithContext =>
             toAquaProcessed(filesWithContext)
           })
-      .map(_.map { compiled =>
+      _ <- airValidator.init()
+      result <- compiledV.map { compiled =>
           compiled.map { ap =>
             logger.trace("generating output...")
             val res = backend.transform(ap.context)
             val compiled = backend.generate(res)
+            airValidator
+              .validate(
+                compiled.toList.flatMap(_.air)
+              )
+              .map(
+                _.bimap(
+                  errs => NonEmptyChain.one(AirValidationError(errs): AquaError[I, E, S]),
+                  _ =>
                     AquaCompiled(ap.id, compiled, res.funcs.length.toInt, res.services.length.toInt)
+                )
+              )
+          }.sequence.map(_.sequence)
+        } match {
+          case Valid(f) => f
+          case Invalid(e) =>
+            invalid[NonEmptyChain[AquaError[I, E, S]], Chain[AquaCompiled[I]]](e).pure[F]
         }
-      })
+    } yield result
   }
 
   def compileTo[F[_]: Monad, E, I: Order, S[_]: Comonad, T](
     sources: AquaSources[F, E, I],
     parser: I => String => ValidatedNec[ParserError[S], Ast[S]],
+    airValidator: AirValidator[F],
     backend: Backend.Transform,
     config: AquaCompilerConf,
     write: AquaCompiled[I] => F[Seq[Validated[E, T]]]
   ): F[ValidatedNec[AquaError[I, E, S], Chain[T]]] =
-    compile[F, E, I, S](sources, parser, backend, config).flatMap {
+    compile[F, E, I, S](sources, parser, airValidator, backend, config).flatMap {
       case Valid(compiled) =>
         compiled.map { ac =>
           write(ac).map(

npm/package.json

@@ -23,7 +23,7 @@
   "dependencies": {
     "@fluencelabs/aqua-ipfs": "0.5.2",
     "@fluencelabs/aqua-lib": "0.5.1",
-    "@fluencelabs/fluence": "0.23.1",
+    "@fluencelabs/fluence": "0.23.3",
     "@fluencelabs/fluence-network-environment": "1.0.13",
     "ipfs-http-client": "50.1.2"
   },