Module and Use expressions (#245)

* Module and Use expressions

* UseFromExpr

* ImportFromExpr

* PubExpr

* Export, declares

* Collecting all the needed info WIP

* Got all the needed data

* Tests fixed

* HeaderSem

* HeaderSem wip

* Everything except `export`/`declares` should be working

* Compile bug fixed

* Fix readme: cli/assembly

* Handle declares, exports

* Compile only exports in AquaRes

* Call services imported from modules

* Import consts, types, services from modules

* Resolve arrows from modules

* Bugfix

compiler/src/main/scala/aqua/compiler/AquaCompiler.scala

@@ -6,19 +6,22 @@ import aqua.model.AquaContext
 import aqua.model.transform.TransformConfig
 import aqua.model.transform.res.AquaRes
 import aqua.parser.lift.LiftParser
+import aqua.parser.Ast
 import aqua.semantics.Semantics
+import aqua.semantics.header.HeaderSem
 import cats.data.Validated.{validNec, Invalid, Valid}
-import cats.data.{Chain, NonEmptyChain, Validated, ValidatedNec}
+import cats.data.{Chain, NonEmptyChain, NonEmptyMap, Validated, ValidatedNec}
 import cats.syntax.applicative.*
 import cats.syntax.flatMap.*
 import cats.syntax.functor.*
 import cats.syntax.traverse.*
-import cats.{Comonad, Monad}
+import cats.syntax.monoid.*
+import cats.{Comonad, Monad, Monoid, Order}
 import scribe.Logging
 
 object AquaCompiler extends Logging {
 
-  def compile[F[_]: Monad, E, I, S[_]: Comonad](
+  def compile[F[_]: Monad, E, I: Order, S[_]: Comonad](
     sources: AquaSources[F, E, I],
     liftI: (I, String) => LiftParser[S],
     backend: Backend,
@@ -26,27 +29,56 @@ object AquaCompiler extends Logging {
   ): F[ValidatedNec[AquaError[I, E, S], Chain[AquaCompiled[I]]]] = {
     import config.aquaContextMonoid
     type Err = AquaError[I, E, S]
+    type Ctx = NonEmptyMap[I, AquaContext]
+    type ValidatedCtx = ValidatedNec[Err, Ctx]
+
     new AquaParser[F, E, I, S](sources, liftI)
-      .resolve[ValidatedNec[Err, AquaContext]] { ast => context =>
-        context.andThen { ctx =>
-          Semantics
-            .process(ast, ctx)
-            .leftMap(_.map[Err](CompileError(_)))
-        }
-      }
-      .map {
-        case Valid(modules) =>
-          Linker.link[I, AquaError[I, E, S], ValidatedNec[Err, AquaContext]](
-            modules,
-            cycle => CycleError[I, E, S](cycle.map(_.id))
-          ) match {
-            case Valid(filesWithContext) =>
+      .resolve[ValidatedCtx](mod =>
+        context =>
+          // Context with prepared imports
+          context.andThen(ctx =>
+            // To manage imports, exports run HeaderSem
+            HeaderSem
+              .sem(
+                mod.imports.view
+                  .mapValues(ctx(_))
+                  .collect { case (fn, Some(fc)) => fn -> fc }
+                  .toMap,
+                mod.body.head
+              )
+              .andThen { headerSem =>
+                // Analyze the body, with prepared initial context
+                Semantics
+                  .process(
+                    mod.body,
+                    headerSem.initCtx
+                  )
+                  // Handle exports, declares – finalize the resulting context
+                  .andThen(headerSem.finCtx)
+                  .map(rc => NonEmptyMap.one(mod.id, rc))
+              }
+              // The whole chain returns a semantics error finally
+              .leftMap(_.map[Err](CompileError(_)))
+          )
+      )
+      .map(
+        _.andThen(modules =>
+          Linker
+            .link[I, AquaError[I, E, S], ValidatedCtx](
+              modules,
+              cycle => CycleError[I, E, S](cycle.map(_.id)),
+              // By default, provide an empty context for this module's id
+              i => validNec(NonEmptyMap.one(i, Monoid.empty[AquaContext]))
+            )
+            .andThen { filesWithContext =>
               filesWithContext
                 .foldLeft[ValidatedNec[Err, Chain[AquaProcessed[I]]]](
                   validNec(Chain.nil)
                 ) {
                   case (acc, (i, Valid(context))) =>
-                    acc combine validNec(Chain.one(AquaProcessed(i, context)))
+                    acc combine validNec(
+                      Chain.fromSeq(context.toNel.toList.map { case (i, c) => AquaProcessed(i, c) })
+                    )
                   case (acc, (_, Invalid(errs))) =>
                     acc combine Invalid(errs)
                 }
@@ -56,13 +88,12 @@ object AquaCompiler extends Logging {
                     AquaCompiled(ap.id, compiled)
                   }
                 )
-            case i @ Invalid(_) => i
-          }
-        case i @ Invalid(_) => i
-      }
+            }
+        )
+      )
   }
 
-  def compileTo[F[_]: Monad, E, I, S[_]: Comonad, T](
+  def compileTo[F[_]: Monad, E, I: Order, S[_]: Comonad, T](
     sources: AquaSources[F, E, I],
     liftI: (I, String) => LiftParser[S],
     backend: Backend,

compiler/src/main/scala/aqua/compiler/AquaParser.scala

@@ -2,7 +2,7 @@ package aqua.compiler
 
 import aqua.linker.{AquaModule, Modules}
 import aqua.parser.Ast
-import aqua.parser.head.ImportExpr
+import aqua.parser.head.{FilenameExpr, ImportExpr}
 import aqua.parser.lift.LiftParser
 import cats.data.{Chain, NonEmptyChain, Validated, ValidatedNec}
 import cats.syntax.applicative.*
@@ -32,23 +32,37 @@ class AquaParser[F[_]: Monad, E, I, S[_]: Comonad](
       )
 
   // Resolve imports (not parse, just resolve) of the given file
-  def resolveImports(id: I, ast: Ast[S]): F[ValidatedNec[Err, Map[I, Err]]] =
+  def resolveImports(id: I, ast: Body): F[ValidatedNec[Err, AquaModule[I, Err, Body]]] =
     ast.head.tailForced
       .map(_.head)
-      .collect { case ImportExpr(filename) =>
+      .collect { case fe: FilenameExpr[F] =>
         sources
-          .resolveImport(id, filename.value.drop(1).dropRight(1))
+          .resolveImport(id, fe.fileValue)
           .map(
             _.bimap(
-              _.map(ResolveImportsErr(id, filename, _)),
-              importId => Chain.one[(I, Err)](importId -> ImportErr(filename))
+              _.map[Err](ResolveImportsErr(id, fe.filename, _)),
+              importId =>
+                Chain.one[(I, (String, Err))](importId -> (fe.fileValue, ImportErr(fe.filename)))
             )
           )
       }
       .traverse(identity)
       .map(
-        _.foldLeft(Validated.validNec[Err, Chain[(I, Err)]](Chain.nil))(_ combine _)
-          .map(_.toList.toMap)
+        _.foldLeft(Validated.validNec[Err, Chain[(I, (String, Err))]](Chain.nil))(_ combine _).map {
+          collected =>
+            AquaModule[I, Err, Body](
+              id,
+              // How filenames correspond to the resolved IDs
+              collected.map { case (i, (fn, _)) =>
+                fn -> i
+              }.toList.toMap[String, I],
+              // Resolved IDs to errors that point to the import in source code
+              collected.map { case (i, (_, err)) =>
+                i -> err
+              }.toList.toMap[I, Err],
+              ast
+            )
+        }
       )
 
   // Parse sources, convert to modules
@@ -56,7 +70,7 @@ class AquaParser[F[_]: Monad, E, I, S[_]: Comonad](
     parseSources.flatMap {
       case Validated.Valid(srcs) =>
         srcs.traverse { case (id, ast) =>
-          resolveImports(id, ast).map(_.map(AquaModule(id, _, ast)).map(Chain.one))
+          resolveImports(id, ast).map(_.map(Chain.one))
         }.map(
           _.foldLeft(Validated.validNec[Err, Chain[AquaModule[I, Err, Body]]](Chain.empty))(
             _ combine _
@@ -66,7 +80,7 @@ class AquaParser[F[_]: Monad, E, I, S[_]: Comonad](
         Validated.invalid[NonEmptyChain[Err], Chain[AquaModule[I, Err, Body]]](errs).pure[F]
     }.map(_.map(_.foldLeft(Modules[I, Err, Body]())(_.add(_, toExport = true))))
 
-  def loadModule(imp: I): F[ValidatedNec[Err, AquaModule[I, Err, Ast[S]]]] =
+  def loadModule(imp: I): F[ValidatedNec[Err, AquaModule[I, Err, Body]]] =
     sources
       .load(imp)
       .map(_.leftMap(_.map[Err](SourcesErr(_))).andThen { src =>
@@ -75,7 +89,7 @@ class AquaParser[F[_]: Monad, E, I, S[_]: Comonad](
       })
       .flatMap {
         case Validated.Valid(ast) =>
-          resolveImports(imp, ast).map(_.map(AquaModule(imp, _, ast)))
+          resolveImports(imp, ast)
         case Validated.Invalid(errs) =>
           Validated.invalid[NonEmptyChain[Err], AquaModule[I, Err, Ast[S]]](errs).pure[F]
       }
@@ -106,7 +120,9 @@ class AquaParser[F[_]: Monad, E, I, S[_]: Comonad](
       case err => err.pure[F]
     }
 
-  def resolve[T](transpile: Ast[S] => T => T): F[ValidatedNec[Err, Modules[I, Err, T => T]]] =
-    resolveSources.map(_.map(_.map(transpile)))
+  def resolve[T](
+    transpile: AquaModule[I, Err, Body] => T => T
+  ): F[ValidatedNec[Err, Modules[I, Err, T => T]]] =
+    resolveSources.map(_.map(_.mapModuleToBody(transpile)))
 
 }

compiler/src/main/scala/aqua/compiler/AquaProcessed.scala

@@ -3,5 +3,5 @@ package aqua.compiler
 import aqua.model.AquaContext
 
 case class AquaProcessed[I](id: I, context: AquaContext) {
-  def hasOutput: Boolean = context.funcs.nonEmpty
+  def hasOutput: Boolean = context.funcs.nonEmpty || context.services.nonEmpty
 }