Make iterator in fold resolvable (#23)

crates/air-parser/src/ast.rs

@@ -74,7 +74,7 @@ pub struct Xor<'i>(pub Box<Instruction<'i>>, pub Box<Instruction<'i>>);
 
 #[derive(Debug, PartialEq, Eq)]
 pub struct Fold<'i> {
-    pub iterable: &'i str,
+    pub iterable: Value<'i>,
     pub iterator: &'i str,
     pub instruction: Rc<Instruction<'i>>,
 }

crates/air-parser/src/lalrpop/aqua.lalrpop

@@ -20,7 +20,7 @@ pub Instr: Box<Instruction<'input>> = {
         Box::new(Instruction::Call(Call{peer_part: p, function_part: f, args, output}))
     },
 
-    "(" "fold" <iterable:Alphanumeric> <iterator:Alphanumeric> <i:Instr> ")" => {
+    "(" "fold" <iterable:Value> <iterator:Alphanumeric> <i:Instr> ")" => {
         let instruction = Rc::new(*i);
         Box::new(Instruction::Fold(Fold{ iterable, iterator, instruction }))
     },

crates/air-parser/src/lalrpop/aqua.rs

@@ -1,5 +1,5 @@
 // auto-generated: "lalrpop 0.19.1"
-// sha256: 9467c547a220171aaf78fe3f8795514c281b4cff41de6a38a5021d6edcea737
+// sha256: 16d0e636128bfae4daa48833950cdbf2289de5d19282908961d73fa82a56
 use crate::ast::*;
 use crate::lalrpop::parser::InstructionError;
 use lalrpop_util::ErrorRecovery;
@@ -42,7 +42,7 @@ mod __parse__Instr {
         // State 1
         9, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 31, 32, 0,
         // State 2
-        0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0,
+        9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 31, 32, 0,
         // State 3
         0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0,
         // State 4
@@ -299,7 +299,6 @@ mod __parse__Instr {
         match nt {
             2 => 21,
             3 => match state {
-                2 => 10,
                 10 => 17,
                 3 => 32,
                 8 => 36,
@@ -337,6 +336,7 @@ mod __parse__Instr {
                 _ => 20,
             },
             14 => match state {
+                2 => 10,
                 7 | 20 => 35,
                 15..=16 => 42,
                 19 | 21 => 54,
@@ -1427,12 +1427,12 @@ mod __parse__Instr {
         _: ::std::marker::PhantomData<(&'input (), &'err ())>,
     ) -> (usize, usize)
     {
-        // Instr = "(", "fold", Alphanumeric, Alphanumeric, Instr, ")" => ActionFn(5);
+        // Instr = "(", "fold", Value, Alphanumeric, Instr, ")" => ActionFn(5);
         assert!(__symbols.len() >= 6);
         let __sym5 = __pop_Variant0(__symbols);
         let __sym4 = __pop_Variant6(__symbols);
         let __sym3 = __pop_Variant0(__symbols);
-        let __sym2 = __pop_Variant0(__symbols);
+        let __sym2 = __pop_Variant2(__symbols);
         let __sym1 = __pop_Variant0(__symbols);
         let __sym0 = __pop_Variant0(__symbols);
         let __start = __sym0.0.clone();
@@ -1893,7 +1893,7 @@ fn __action5<
     input: &'input str,
     (_, _, _): (usize, &'input str, usize),
     (_, _, _): (usize, &'input str, usize),
-    (_, iterable, _): (usize, &'input str, usize),
+    (_, iterable, _): (usize, Value<'input>, usize),
     (_, iterator, _): (usize, &'input str, usize),
     (_, i, _): (usize, Box<Instruction<'input>>, usize),
     (_, _, _): (usize, &'input str, usize),

crates/air-parser/src/lalrpop/tests.rs

@@ -21,6 +21,7 @@ use PeerPart::*;
 use Value::*;
 
 use fstrings::f;
+use std::rc::Rc;
 
 fn parse(source_code: &str) -> Instruction {
     *crate::parse(source_code).expect("parsing failed")
@@ -212,7 +213,7 @@ fn parse_fold() {
         )
         "#;
     let instruction = parse(&source_code.as_ref());
-    let expected = fold("iterable", "i", null());
+    let expected = fold(Value::Variable("iterable"), "i", null());
     assert_eq!(instruction, expected);
 }
 
@@ -227,7 +228,7 @@ fn parse_fold_with_xor_par_seq() {
         let source_code = source_fold_with(name);
         let instruction = parse(&source_code.as_ref());
         let instr = binary_instruction(*name);
-        let expected = fold("iterable", "i", instr(null(), null()));
+        let expected = fold(Value::Variable("iterable"), "i", instr(null(), null()));
         assert_eq!(instruction, expected);
     }
 }
@@ -371,6 +372,23 @@ fn no_output() {
     assert_eq!(instruction, expected);
 }
 
+#[test]
+fn fold_json_path() {
+    let source_code = r#"
+    (fold members.$.["users"] m (null))
+    "#;
+    let instruction = parse(&source_code.as_ref());
+    let expected = Instruction::Fold(Fold {
+        iterable: JsonPath {
+            variable: "members",
+            path: "$.[\"users\"]",
+        },
+        iterator: "m",
+        instruction: Rc::new(null()),
+    });
+    assert_eq!(instruction, expected);
+}
+
 // Test DSL
 
 fn seq<'a>(l: Instruction<'a>, r: Instruction<'a>) -> Instruction<'a> {
@@ -388,7 +406,11 @@ fn seqnn() -> Instruction<'static> {
 fn null() -> Instruction<'static> {
     Instruction::Null(Null)
 }
-fn fold<'a>(iterable: &'a str, iterator: &'a str, instruction: Instruction<'a>) -> Instruction<'a> {
+fn fold<'a>(
+    iterable: Value<'a>,
+    iterator: &'a str,
+    instruction: Instruction<'a>,
+) -> Instruction<'a> {
     Instruction::Fold(Fold {
         iterable,
         iterator,

stepper-lib/src/air/call/parsed_call.rs

@@ -17,9 +17,10 @@
 #![allow(unused_unsafe)] // for wasm_bindgen target where calling FFI is safe
 
 use super::triplet::{ResolvedTriplet, Triplet};
-use super::utils::{resolve_jvalue, set_local_call_result, set_remote_call_result};
+use super::utils::{set_local_call_result, set_remote_call_result};
 use super::Call;
 
+use crate::air::resolve::resolve_jvalue;
 use crate::air::ExecutionCtx;
 use crate::build_targets::CALL_SERVICE_SUCCESS;
 use crate::call_evidence::{CallEvidenceCtx, CallResult, EvidenceState};

stepper-lib/src/air/call/triplet.rs

@@ -14,11 +14,10 @@
  * limitations under the License.
  */
 
-use super::utils::resolve_value;
-
 use crate::air::ExecutionCtx;
 use crate::{AquamarineError, Result};
 
+use crate::air::resolve::resolve_value;
 use air_parser::ast::{FunctionPart, PeerPart, Value};
 
 /// Triplet represents a location of the executable code in the network

stepper-lib/src/air/call/utils.rs

@@ -24,9 +24,9 @@ use crate::AquamarineError;
 use crate::JValue;
 use crate::Result;
 
-use air_parser::ast::{CallOutput, Value};
+use air_parser::ast::CallOutput;
 
-use std::{borrow::Cow, cell::RefCell, rc::Rc};
+use std::{cell::RefCell, rc::Rc};
 
 /// Writes result of a local `Call` instruction to `ExecutionCtx` at `output`
 pub(super) fn set_local_call_result<'i>(
@@ -79,97 +79,3 @@ pub(super) fn set_remote_call_result<'i>(
     );
     call_ctx.new_path.push_back(new_evidence_state);
 }
-
-/// Applies `json_path` to `jvalue`
-pub(super) fn find_by_json_path<'jvalue, 'json_path>(
-    jvalue: &'jvalue JValue,
-    json_path: &'json_path str,
-) -> Result<Vec<&'jvalue JValue>> {
-    use AquamarineError::VariableNotInJsonPath as JsonPathError;
-
-    jsonpath_lib::select(jvalue, json_path).map_err(|e| JsonPathError(jvalue.clone(), String::from(json_path), e))
-}
-
-/// Takes variable's value from `ExecutionCtx::data_cache`
-/// TODO: maybe return &'i JValue?
-pub(super) fn resolve_variable<'exec_ctx, 'i>(variable: &'i str, ctx: &'exec_ctx ExecutionCtx<'i>) -> Result<JValue> {
-    use AquamarineError::VariableNotFound;
-
-    let value = ctx
-        .data_cache
-        .get(variable)
-        .ok_or_else(|| VariableNotFound(variable.to_string()))?;
-
-    match value {
-        AValue::JValueFoldCursor(fold_state) => {
-            if let JValue::Array(array) = fold_state.iterable.as_ref() {
-                Ok(array[fold_state.cursor].clone())
-            } else {
-                unreachable!("fold state must be well-formed because it is changed only by stepper")
-            }
-        }
-        AValue::JValueRef(value) => Ok(value.as_ref().clone()),
-        AValue::JValueAccumulatorRef(acc) => {
-            let owned_acc = acc.borrow().iter().map(|v| v.as_ref()).cloned().collect::<Vec<_>>();
-            Ok(JValue::Array(owned_acc))
-        }
-    }
-}
-
-pub(super) fn apply_json_path<'i>(jvalue: JValue, json_path: &'i str) -> Result<JValue> {
-    let values = find_by_json_path(&jvalue, json_path)?;
-    if values.is_empty() {
-        return Err(AquamarineError::VariableNotFound(json_path.to_string()));
-    }
-
-    if values.len() != 1 {
-        return Err(AquamarineError::MultipleValuesInJsonPath(json_path.to_string()));
-    }
-
-    // TODO: sure need this clone?
-    Ok(values[0].clone())
-}
-
-pub(super) fn require_string(value: JValue) -> Result<String> {
-    if let JValue::String(s) = value {
-        Ok(s)
-    } else {
-        Err(AquamarineError::IncompatibleJValueType(value, "string".to_string()))
-    }
-}
-
-/// Resolve value to string by either resolving variable from `ExecutionCtx`, taking literal value, or etc
-pub(super) fn resolve_value<'i, 'a: 'i>(value: &'a Value<'i>, ctx: &'a ExecutionCtx<'i>) -> Result<Cow<'i, str>> {
-    let resolved = match value {
-        Value::CurrentPeerId => Cow::Borrowed(ctx.current_peer_id.as_str()),
-        Value::Literal(value) => Cow::Borrowed(*value),
-        Value::Variable(name) => {
-            let resolved = resolve_variable(name, ctx)?;
-            let resolved = require_string(resolved)?;
-            Cow::Owned(resolved)
-        }
-        Value::JsonPath { variable, path } => {
-            let resolved = resolve_variable(variable, ctx)?;
-            let resolved = apply_json_path(resolved, path)?;
-            let resolved = require_string(resolved)?;
-            Cow::Owned(resolved)
-        }
-    };
-
-    Ok(resolved)
-}
-
-/// Resolve value to JValue, similar to `resolve_value`
-pub(super) fn resolve_jvalue<'i>(value: &Value<'i>, ctx: &ExecutionCtx<'i>) -> Result<JValue> {
-    let value = match value {
-        Value::CurrentPeerId => JValue::String(ctx.current_peer_id.clone()),
-        Value::Literal(value) => JValue::String(value.to_string()),
-        Value::Variable(name) => resolve_variable(name, ctx)?,
-        Value::JsonPath { variable, path } => {
-            let value = resolve_variable(variable, ctx)?;
-            apply_json_path(value, path)?
-        }
-    };
-
-    Ok(value)
-}

stepper-lib/src/air/fold.rs

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+use super::resolve::resolve_jvalue;
 use super::CallEvidenceCtx;
 use super::ExecutionCtx;
 use super::Instruction;
@@ -50,28 +51,25 @@ impl<'i> super::ExecutableInstruction<'i> for Fold<'i> {
 
         log_instruction!(fold, exec_ctx, call_ctx);
 
+        // TODO: implement and call resolve_avalue to reuse existing Rc's
+        let iterable = resolve_jvalue(&self.iterable, exec_ctx)?;
         // check that value exists and has array type
-        let iterable = match exec_ctx.data_cache.get(self.iterable) {
+        let iterable = match &iterable {
-            Some(AValue::JValueRef(jvalue_rc)) => {
+            JValue::Array(ref array) => {
-                match jvalue_rc.as_ref() {
+                if array.is_empty() {
-                    JValue::Array(array) => {
+                    // skip fold if array is empty
-                        if array.is_empty() {
+                    return Ok(());
-                            // skip fold if array is empty
-                            return Ok(());
-                        }
-
-                        jvalue_rc
-                    }
-                    v => return Err(IncompatibleJValueType(v.clone(), String::from("Array"))),
                 }
+
+                iterable
             }
-            Some(v) => return Err(IncompatibleAValueType(format!("{:?}", v), String::from("JValueRef"))),
+            v => return Err(IncompatibleJValueType(v.clone(), String::from("Array"))),
-            None => return Err(VariableNotFound(self.iterable.to_string())),
         };
 
         let fold_state = FoldState {
             cursor: 0,
-            iterable: iterable.clone(),
+            // TODO: reuse existing Rc from JValueRef, if there was some
+            iterable: Rc::new(iterable),
             instr_head: self.instruction.clone(),
         };
 
@@ -80,7 +78,7 @@ impl<'i> super::ExecutableInstruction<'i> for Fold<'i> {
             .insert(self.iterator.to_string(), AValue::JValueFoldCursor(fold_state));
 
         if previous_value.is_some() {
-            return Err(MultipleFoldStates(self.iterable.to_string()));
+            return Err(MultipleFoldStates(self.iterator.to_string()));
         }
 
         self.instruction.execute(exec_ctx, call_ctx)?;
@@ -152,7 +150,7 @@ mod tests {
 
     #[test]
     fn lfold() {
-        env_logger::init();
+        env_logger::try_init().ok();
 
         use crate::call_evidence::CallResult::*;
         use crate::call_evidence::EvidenceState::*;
@@ -301,7 +299,7 @@ mod tests {
 
         assert_eq!(
             error,
-            StepperError::MultipleFoldStates(String::from("multiple fold states found for iterable Iterable2"))
+            StepperError::MultipleFoldStates(String::from("multiple fold states found for iterable i"))
         );
     }
 
@@ -333,4 +331,43 @@ mod tests {
         assert_eq!(res.len(), 1);
         assert_eq!(res[0], Call(Executed(Rc::new(json!([])))));
     }
+
+    #[test]
+    fn json_path() {
+        use crate::call_evidence::CallResult::*;
+        use crate::call_evidence::EvidenceState::*;
+
+        let mut vm = create_aqua_vm(echo_number_call_service(), "A");
+        let mut set_variable_vm = create_aqua_vm(
+            set_variable_call_service(r#"{ "array": ["1","2","3","4","5"] }"#),
+            "set_variable",
+        );
+
+        let lfold = String::from(
+            r#"
+            (seq
+                (call "set_variable" ("" "") [] Iterable)
+                (fold Iterable.$["array"] i
+                    (seq
+                        (call "A" ("" "") [i] acc[])
+                        (next i)
+                    )
+                )
+            )"#,
+        );
+
+        let res = call_vm!(set_variable_vm, "", lfold.clone(), "[]", "[]");
+        let res = call_vm!(vm, "", lfold, "[]", res.data);
+        let res: CallEvidencePath = serde_json::from_str(&res.data).expect("should be valid call evidence path");
+
+        assert_eq!(res.len(), 6);
+        assert_eq!(
+            res[0],
+            Call(Executed(Rc::new(json!({ "array": ["1", "2", "3", "4", "5"] }))))
+        );
+
+        for i in 1..=5 {
+            assert_eq!(res[i], Call(Executed(Rc::new(JValue::Number(i.into())))));
+        }
+    }
 }

stepper-lib/src/air/mod.rs

@@ -19,6 +19,7 @@ mod execution_context;
 mod fold;
 mod null;
 mod par;
+pub(crate) mod resolve;
 mod seq;
 mod xor;
 

stepper-lib/src/air/resolve.rs

@@ -0,0 +1,119 @@
+/*
+ * Copyright 2020 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use super::ExecutionCtx;
+use crate::AValue;
+use crate::AquamarineError;
+use crate::JValue;
+use crate::Result;
+
+use air_parser::ast::Value;
+
+use std::borrow::Cow;
+
+/// Resolve value to JValue, similar to `resolve_value`
+pub(crate) fn resolve_jvalue<'i>(value: &Value<'i>, ctx: &ExecutionCtx<'i>) -> Result<JValue> {
+    let value = match value {
+        Value::CurrentPeerId => JValue::String(ctx.current_peer_id.clone()),
+        Value::Literal(value) => JValue::String(value.to_string()),
+        Value::Variable(name) => resolve_variable(name, ctx)?,
+        Value::JsonPath { variable, path } => {
+            let value = resolve_variable(variable, ctx)?;
+            apply_json_path(value, path)?
+        }
+    };
+
+    Ok(value)
+}
+
+/// Takes variable's value from `ExecutionCtx::data_cache`
+/// TODO: maybe return &'i JValue?
+pub(crate) fn resolve_variable<'exec_ctx, 'i>(variable: &'i str, ctx: &'exec_ctx ExecutionCtx<'i>) -> Result<JValue> {
+    use AquamarineError::VariableNotFound;
+
+    let value = ctx
+        .data_cache
+        .get(variable)
+        .ok_or_else(|| VariableNotFound(variable.to_string()))?;
+
+    match value {
+        AValue::JValueFoldCursor(fold_state) => {
+            if let JValue::Array(array) = fold_state.iterable.as_ref() {
+                Ok(array[fold_state.cursor].clone())
+            } else {
+                unreachable!("fold state must be well-formed because it is changed only by stepper")
+            }
+        }
+        AValue::JValueRef(value) => Ok(value.as_ref().clone()),
+        AValue::JValueAccumulatorRef(acc) => {
+            let owned_acc = acc.borrow().iter().map(|v| v.as_ref()).cloned().collect::<Vec<_>>();
+            Ok(JValue::Array(owned_acc))
+        }
+    }
+}
+
+/// Resolve value to string by either resolving variable from `ExecutionCtx`, taking literal value, or etc
+pub(crate) fn resolve_value<'i, 'a: 'i>(value: &'a Value<'i>, ctx: &'a ExecutionCtx<'i>) -> Result<Cow<'i, str>> {
+    let resolved = match value {
+        Value::CurrentPeerId => Cow::Borrowed(ctx.current_peer_id.as_str()),
+        Value::Literal(value) => Cow::Borrowed(*value),
+        Value::Variable(name) => {
+            let resolved = resolve_variable(name, ctx)?;
+            let resolved = require_string(resolved)?;
+            Cow::Owned(resolved)
+        }
+        Value::JsonPath { variable, path } => {
+            let resolved = resolve_variable(variable, ctx)?;
+            let resolved = apply_json_path(resolved, path)?;
+            let resolved = require_string(resolved)?;
+            Cow::Owned(resolved)
+        }
+    };
+
+    Ok(resolved)
+}
+
+pub(crate) fn require_string(value: JValue) -> Result<String> {
+    if let JValue::String(s) = value {
+        Ok(s)
+    } else {
+        Err(AquamarineError::IncompatibleJValueType(value, "string".to_string()))
+    }
+}
+
+pub(crate) fn apply_json_path<'i>(jvalue: JValue, json_path: &'i str) -> Result<JValue> {
+    let values = find_by_json_path(&jvalue, json_path)?;
+    if values.is_empty() {
+        return Err(AquamarineError::VariableNotFound(json_path.to_string()));
+    }
+
+    if values.len() != 1 {
+        return Err(AquamarineError::MultipleValuesInJsonPath(json_path.to_string()));
+    }
+
+    // TODO: sure need this clone?
+    Ok(values[0].clone())
+}
+
+/// Applies `json_path` to `jvalue`
+fn find_by_json_path<'jvalue, 'json_path>(
+    jvalue: &'jvalue JValue,
+    json_path: &'json_path str,
+) -> Result<Vec<&'jvalue JValue>> {
+    use AquamarineError::VariableNotInJsonPath as JsonPathError;
+
+    jsonpath_lib::select(jvalue, json_path).map_err(|e| JsonPathError(jvalue.clone(), String::from(json_path), e))
+}