fluencelabs/aqua
1
0
Fork 0
mirror of https://github.com/fluencelabs/aqua.git synced 2025-04-24 22:42:13 +00:00
Code Issues Projects Releases Wiki Activity

Generate types from JSON (#492)

Browse Source
This commit is contained in:
Dima 2022-04-20 14:45:42 +03:00 committed by GitHub
parent e3957c1296
commit 10061ade3b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 400 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
build.sbt
View File

@ -12,12 +12,12 @@ val fs2V = "3.2.5"
val catsEffectV = "3.3.7"
val declineV = "2.2.0"
val circeVersion = "0.14.1"
val scribeV = "3.6.6"
val scribeV = "3.7.1"
name := "aqua-hll"
val commons = Seq(
baseAquaVersion := "0.7.1",
baseAquaVersion := "0.7.2",
version := baseAquaVersion.value + "-" + sys.env.getOrElse("BUILD_NUMBER", "SNAPSHOT"),
scalaVersion := dottyVersion,
libraryDependencies ++= Seq(

50
cli/.js/src/main/scala/aqua/ArgOpts.scala
View File

@ -1,23 +1,26 @@
package aqua
import aqua.builder.ArgumentGetter
import aqua.json.JsonEncoder
import aqua.parser.expr.func.CallArrowExpr
import aqua.parser.lexer.{CallArrowToken, LiteralToken, VarToken}
import aqua.parser.lexer.{CallArrowToken, CollectionToken, LiteralToken, VarToken}
import aqua.parser.lift.Span
import aqua.raw.value.{LiteralRaw, ValueRaw, VarRaw}
import aqua.types.*
import cats.data.Validated.{invalid, invalidNec, valid, validNec, validNel}
import cats.data.Validated.{invalid, invalidNec, invalidNel, valid, validNec, validNel}
import cats.data.*
import cats.effect.Concurrent
import cats.syntax.applicative.*
import cats.syntax.apply.*
import cats.syntax.flatMap.*
import cats.syntax.semigroup.*
import cats.syntax.functor.*
import cats.syntax.traverse.*
import cats.{~>, Id}
import cats.{~>, Id, Semigroup}
import com.monovore.decline.Opts
import fs2.io.file.Files
import scala.collection.immutable.SortedMap
import scala.scalajs.js
import scala.scalajs.js.JSON
@ -42,15 +45,19 @@ object ArgOpts {
case Right(exprSpan) =>
val expr = exprSpan.mapK(spanToId)
val args = expr.args.collect {
val argsV = expr.args.collect {
case LiteralToken(value, ts) =>
LiteralRaw(value, ts)
validNel(LiteralRaw(value, ts))
case VarToken(name, _) =>
// TODO why BottomType?
VarRaw(name.value, BottomType)
}
validNel(VarRaw(name.value, BottomType))
case CollectionToken(_, _) =>
invalidNel("Array argument in function call not supported. Pass it through JSON.")
case CallArrowToken(_, _, _) =>
invalidNel("Function call as argument not supported.")
}.sequence
argsV.andThen(args =>
validNel(CliFunc(expr.funcName.value, args, expr.ability.map(_.name)))
)
case Left(err) => invalid(err.expected.map(_.context.mkString("\n")))
}
@ -81,29 +88,6 @@ object ArgOpts {
}
}
// TODO: it is hack, will be deleted after we will have context with types on this stage
def jsTypeToAqua(name: String, arg: js.Dynamic): ValidatedNec[String, Type] = {
arg match {
case a if js.typeOf(a) == "string" => validNec(ScalarType.string)
case a if js.typeOf(a) == "number" => validNec(ScalarType.u64)
case a if js.typeOf(a) == "boolean" => validNec(ScalarType.bool)
case a if js.Array.isArray(a) =>
// if all types are similar it will be array array with this type
// otherwise array with bottom type
val elementsTypesV: ValidatedNec[String, List[Type]] =
a.asInstanceOf[js.Array[js.Dynamic]].map(ar => jsTypeToAqua(name, ar)).toList.sequence
elementsTypesV.andThen { elementsTypes =>
if (elementsTypes.isEmpty) validNec(ArrayType(BottomType))
else if (elementsTypes.forall(_ == elementsTypes.head))
validNec(ArrayType(elementsTypes.head))
else invalidNec(s"All array elements in '$name' argument must be of the same type.")
}
case _ => validNec(BottomType)
}
}
// checks if data is presented if there is non-literals in function arguments
// creates services to add this data into a call
def checkDataGetServices(
@ -128,7 +112,7 @@ object ArgOpts {
else a
}
val typeV = jsTypeToAqua(vm.name, arg)
val typeV = JsonEncoder.aquaTypeFromJson(vm.name, arg)
typeV.map(t => (vm.copy(baseType = t), arg))
}.sequence

2
cli/.js/src/main/scala/aqua/CommandBuilder.scala
View File

@ -60,7 +60,7 @@ class SubCommandBuilder[F[_]: Async](
)
}
case Validated.Invalid(errs) =>
errs.map(logger.error)
errs.map(e => logger.error(e))
ExitCode.Error.pure[F]
}
}

5
cli/.js/src/main/scala/aqua/builder/IPFSUploader.scala
View File

@ -14,9 +14,12 @@ object IPFSUploader extends Logging {
private def uploadFunc(funcName: String): AquaFunction = new AquaFunction {
override def fnName: String = funcName
private def logError(s: String) = logger.error(s)
private def logInfo(s: String) = logger.info(s)
override def handler: ServiceHandler = args => {
IpfsApi
.uploadFile(args(0), args(1), logger.info: String => Unit, logger.error: String => Unit)
.uploadFile(args(0), args(1), logError, logInfo)
.`catch` { err =>
js.Dynamic.literal(error = "Error on uploading file: " + err)
}

120
cli/.js/src/main/scala/aqua/json/JsonEncoder.scala Normal file
View File

@ -0,0 +1,120 @@
package aqua.json
import aqua.types.{ArrayType, BottomType, LiteralType, OptionType, StructType, Type}
import cats.data.{NonEmptyMap, Validated, ValidatedNec}
import cats.data.Validated.{invalid, invalidNec, invalidNel, valid, validNec, validNel}
import cats.syntax.applicative.*
import cats.syntax.apply.*
import cats.syntax.flatMap.*
import cats.syntax.semigroup.*
import cats.syntax.functor.*
import cats.syntax.traverse.*
import scala.collection.immutable.SortedMap
import scala.scalajs.js
object JsonEncoder {
/* Get widest possible type from JSON arrays. For example:
JSON: {
field1: [
{
a: "a",
b: [1,2,3],
c: 4
},
{
c: 3
}
]
}
There type in array must be { a: ?string, b: []number, c: number
*/
def compareAndGetWidestType(
name: String,
ltV: ValidatedNec[String, Type],
rtV: ValidatedNec[String, Type]
): ValidatedNec[String, Type] = {
(ltV, rtV) match {
case (Validated.Valid(lt), Validated.Valid(rt)) =>
(lt, rt) match {
case (lt, rt) if lt == rt => validNec(lt)
case (BottomType, ra @ ArrayType(_)) => validNec(ra)
case (la @ ArrayType(_), BottomType) => validNec(la)
case (lo @ OptionType(lel), rtt) if lel == rtt => validNec(lo)
case (ltt, ro @ OptionType(rel)) if ltt == rel => validNec(ro)
case (BottomType, rb) => validNec(OptionType(rb))
case (lb, BottomType) => validNec(OptionType(lb))
case (lst: StructType, rst: StructType) =>
val lFieldsSM: SortedMap[String, Type] = lst.fields.toSortedMap
val rFieldsSM: SortedMap[String, Type] = rst.fields.toSortedMap
(lFieldsSM.toList ++ rFieldsSM.toList)
.groupBy(_._1)
.view
.mapValues(_.map(_._2))
.map {
case (name, t :: Nil) =>
compareAndGetWidestType(name, validNec(t), validNec(BottomType)).map(t =>
(name, t)
)
case (name, lt :: rt :: Nil) =>
compareAndGetWidestType(name, validNec(lt), validNec(rt)).map(t => (name, t))
case _ =>
// this is internal error.This Can't happen
invalidNec("Unexpected. The list can only have 1 or 2 arguments.")
}
.toList
.sequence
.map(processedFields => NonEmptyMap.fromMap(SortedMap(processedFields: _*)).get)
.map(mt => StructType("", mt))
case (a, b) =>
invalidNec(s"Types in '$name' array should be the same")
}
case (Validated.Invalid(lerr), Validated.Invalid(rerr)) =>
Validated.Invalid(lerr ++ rerr)
case (l @ Validated.Invalid(_), _) =>
l
case (_, r @ Validated.Invalid(_)) =>
r
}
}
// Gather all information about all fields in JSON and create Aqua type.
def aquaTypeFromJson(name: String, arg: js.Dynamic): ValidatedNec[String, Type] = {
val t = js.typeOf(arg)
arg match {
case a if t == "string" => validNec(LiteralType.string)
case a if t == "number" => validNec(LiteralType.number)
case a if t == "boolean" => validNec(LiteralType.bool)
case a if js.Array.isArray(a) =>
// if all types are similar it will be array array with this type
// otherwise array with bottom type
val elementsTypesV: ValidatedNec[String, List[Type]] =
a.asInstanceOf[js.Array[js.Dynamic]].map(ar => aquaTypeFromJson(name, ar)).toList.sequence
elementsTypesV.andThen { elementsTypes =>
if (elementsTypes.isEmpty) validNec(ArrayType(BottomType))
else {
elementsTypes
.map(el => validNec(el))
.reduce[ValidatedNec[String, Type]] { case (l, t) =>
compareAndGetWidestType(name, l, t)
}
.map(t => ArrayType(t))
}
}
case a if t == "object" && !js.isUndefined(arg) && arg != null =>
val dict = arg.asInstanceOf[js.Dictionary[js.Dynamic]]
val keys = dict.keys
keys
.map(k => aquaTypeFromJson(k, arg.selectDynamic(k)).map(t => k -> t))
.toList
.sequence
.map { fields =>
StructType("", NonEmptyMap.fromMap(SortedMap(fields: _*)).get)
}
case _ => validNec(BottomType)
}
}
}

74
cli/.js/src/main/scala/aqua/run/Runner.scala
View File

@ -5,6 +5,7 @@ import aqua.backend.FunctionDef
import aqua.backend.air.FuncAirGen
import aqua.builder.{ArgumentGetter, Finisher, ResultPrinter}
import aqua.io.OutputPrinter
import cats.data.Validated.{invalidNec, validNec}
import aqua.model.transform.{Transform, TransformConfig}
import aqua.model.{FuncArrow, ValueModel, VarModel}
import aqua.raw.ops.{Call, CallArrowRawTag, FuncOp, SeqTag}
@ -14,8 +15,11 @@ import cats.data.{Validated, ValidatedNec}
import cats.effect.kernel.Async
import cats.syntax.applicative.*
import cats.syntax.show.*
import cats.syntax.flatMap.*
import cats.syntax.traverse.*
import cats.syntax.partialOrder._
import scala.collection.immutable.SortedMap
import scala.concurrent.ExecutionContext
import scala.scalajs.js
@ -30,9 +34,75 @@ class Runner(
funcCallable.arrowType.codomain.toList.zipWithIndex.map { case (t, idx) =>
name + idx
}
import aqua.types.Type.typesPartialOrder
// Compare and validate type generated from JSON and type from Aqua file.
// Also, validation will be success if array or optional field will be missed in JSON type
def validateTypes(name: String, lt: Type, rtOp: Option[Type]): ValidatedNec[String, Unit] = {
rtOp match {
case None =>
lt match {
case tb: BoxType =>
validNec(())
case _ =>
invalidNec(s"Missed field $name in arguments")
}
case Some(rt) =>
(lt, rt) match {
case (l: StructType, r: StructType) =>
val lsm: SortedMap[String, Type] = l.fields.toSortedMap
val rsm: SortedMap[String, Type] = r.fields.toSortedMap
lsm.map { case (n, ltt) =>
validateTypes(s"$name.$n", ltt, rsm.get(n))
}.toList.sequence.map(_ => ())
case (l: BoxType, r: BoxType) =>
validateTypes(name, l.element, Some(r.element))
case (l: BoxType, r) =>
validateTypes(name, l.element, Some(r))
case (l, r) =>
if (l >= r) validNec(())
else
invalidNec(
s"Type for field '$name' mismatch with argument. Expected: '$l' Given: '$r'"
)
}
}
}
def validateArguments(
funcDomain: List[(String, Type)],
args: List[ValueRaw]
): ValidatedNec[String, Unit] = {
if (funcDomain.size != args.length) {
invalidNec(
s"Number of arguments (${args.length}) does not match what the function requires (${funcDomain.size})."
)
} else {
funcDomain
.zip(args)
.map { case ((name, lt), rt) =>
rt match {
case VarRaw(n, t) =>
validateTypes(n, lt, Some(rt.`type`))
case _ =>
validateTypes(name, lt, Some(rt.`type`))
}
}
.sequence
.map(_ => ())
}
}
// Wraps function with necessary services, registers services and calls wrapped function with FluenceJS
def run[F[_]: Async](): F[ValidatedNec[String, Unit]] = {
validateArguments(
funcCallable.arrowType.domain.labelledData,
func.args
) match {
case Validated.Valid(_) =>
val resultNames = resultVariableNames(funcCallable, config.resultName)
val resultPrinterService =
ResultPrinter(config.resultPrinterServiceId, config.resultPrinterName, resultNames)
@ -54,6 +124,10 @@ class Runner(
)
case i @ Validated.Invalid(_) => i.pure[F]
}
case v @ Validated.Invalid(_) =>
v.pure[F]
}
}
// Generates air from function, register all services and make a call through FluenceJS

150
cli/.js/src/test/scala/aqua/JsonEncoderSpec.scala Normal file
View File

@ -0,0 +1,150 @@
package aqua
import aqua.json.JsonEncoder
import aqua.types.{ArrayType, LiteralType, OptionType, StructType}
import cats.Id
import org.scalatest.flatspec.AnyFlatSpec
import org.scalatest.matchers.should.Matchers
import cats.data.{NonEmptyList, NonEmptyMap}
class JsonEncoderSpec extends AnyFlatSpec with Matchers {
"json encoder" should "get type from json" in {
val json = scalajs.js.JSON.parse("""{
|"arr2": [{
| "a": "fef",
| "b": [1,2,3,4],
| "c": "erfer"
| },{
| "a": "ferfer",
| "b": [1,2,3,4],
| "c": "erfer"
| }, {
| "a": "as",
| "d": "gerrt"
| }]
|} """.stripMargin)
val res = JsonEncoder.aquaTypeFromJson("n", json)
res.isValid shouldBe true
val elType = StructType(
"",
NonEmptyMap.of(
("a", LiteralType.string),
("b", ArrayType(LiteralType.number)),
("c", OptionType(LiteralType.string)),
("d", OptionType(LiteralType.string))
)
)
res.toOption.get shouldBe StructType("", NonEmptyMap.of(("arr2", ArrayType(elType))))
}
"json encoder" should "get type from json 1" in {
val json = scalajs.js.JSON.parse("""{
|"arr2": [{
| "b": [1,2,3,4]
| },{
| "b": [1,2,3,4]
| }, {
| "b": "gerrt"
| }]
|} """.stripMargin)
val res = JsonEncoder.aquaTypeFromJson("n", json)
res.isValid shouldBe false
}
"json encoder" should "get type from json 2" in {
val json =
scalajs.js.JSON.parse(
"""{
|"arr1": [{"a": [{"c": "", "d": 123}, {"c": ""}], "b": ""}, {"b": ""}],
|"arr2": [1,2,3,4],
|"arr3": ["fre", "grt", "rtgrt"],
|"str": "egrerg",
|"num": 123
|} """.stripMargin
)
val res = JsonEncoder.aquaTypeFromJson("n", json)
res.isValid shouldBe true
val innerElType = StructType(
"",
NonEmptyMap.of(
("c", LiteralType.string),
("d", OptionType(LiteralType.number))
)
)
val elType = StructType(
"",
NonEmptyMap.of(
("a", ArrayType(innerElType)),
("b", LiteralType.string)
)
)
val t = StructType(
"",
NonEmptyMap.of(
("arr1", ArrayType(elType)),
("arr2", ArrayType(LiteralType.number)),
("arr3", ArrayType(LiteralType.string)),
("str", LiteralType.string),
("num", LiteralType.number)
)
)
res.toOption.get shouldBe t
}
"json encoder" should "get type from json 3" in {
val json = scalajs.js.JSON.parse("""{
|"arr2": [{
| "b": [1,2,3,4]
| },{
| "b": [1,2,3,4]
| }, {
| "b": "gerrt"
| }]
|} """.stripMargin)
val res = JsonEncoder.aquaTypeFromJson("n", json)
res.isValid shouldBe false
}
"json encoder" should "get type from json 4" in {
val json =
scalajs.js.JSON.parse(
"""{
|"arr4": [{"a": "", "b": {"c": "", "d": [1,2,3,4]}}, {"a": ""}]
|} """.stripMargin
)
val res = JsonEncoder.aquaTypeFromJson("n", json)
res.isValid shouldBe true
val arr4InnerType = OptionType(
StructType(
"",
NonEmptyMap.of(
("c", LiteralType.string),
("d", ArrayType(LiteralType.number))
)
)
)
val arr4ElType = StructType(
"",
NonEmptyMap.of(
("a", LiteralType.string),
("b", arr4InnerType)
)
)
val t = StructType(
"",
NonEmptyMap.of(
("arr4", ArrayType(arr4ElType))
)
)
res.toOption.get shouldBe t
}
}

2
cli/src/main/scala/aqua/AquaCli.scala
View File

@ -82,8 +82,6 @@ object AquaCli extends IOApp with Logging {
isDryRun,
isScheduled
) =>
LogFormatter.initLogger(Some(logLevel))
val toAir = toAirOp || isScheduled
val noXor = noXorOp || isScheduled
val noRelay = noRelayOp || isScheduled

2
cli/src/main/scala/aqua/io/OutputPrinter.scala
View File

@ -9,6 +9,6 @@ object OutputPrinter {
}
def error(str: String): Unit = {
println(str)
println(Console.RED + str + Console.RESET)
}
}

20
types/src/main/scala/aqua/types/CompareTypes.scala
View File

@ -54,21 +54,29 @@ object CompareTypes {
case _ => Double.NaN
}
private def compareStructs(lf: NonEmptyMap[String, Type], rf: NonEmptyMap[String, Type]): Double =
if (lf.toSortedMap == rf.toSortedMap) 0.0
private def compareStructs(
lfNEM: NonEmptyMap[String, Type],
rfNEM: NonEmptyMap[String, Type]
): Double = {
val lf = lfNEM.toSortedMap
val rf = rfNEM.toSortedMap
val lfView = lf.view
val rfView = rf.view
if (lf == rf) 0.0
else if (
lf.keys.forall(rf.contains) && compareTypesList(
lf.toSortedMap.toList.map(_._2),
rf.toSortedMap.view.filterKeys(lf.keys.contains).toList.map(_._2)
lfView.values.toList,
rfView.filterKeys(lfNEM.keys.contains).values.toList
) == -1.0
) 1.0
else if (
rf.keys.forall(lf.contains) && compareTypesList(
lf.toSortedMap.view.filterKeys(rf.keys.contains).toList.map(_._2),
rf.toSortedMap.toList.map(_._2)
lfView.filterKeys(rfNEM.keys.contains).values.toList,
rfView.values.toList
) == 1.0
) -1.0
else NaN
}
private def compareProducts(l: ProductType, r: ProductType): Double = ((l, r): @unchecked) match {
case (NilType, NilType) => 0.0