Change subtrace_len computation algo in FoldFSM (#138)

air/src/execution_step/air/call/resolved_call.rs

@@ -206,7 +206,7 @@ fn try_to_service_result(
             let error = CallServiceFailed(i32::MAX, error_msg.clone());
             trace_ctx.meet_call_end(error);
 
-            Err(Rc::new(ExecutionError::LocalServiceError(i32::MAX, error_msg.clone())))
+            Err(Rc::new(ExecutionError::LocalServiceError(i32::MAX, error_msg)))
         }
     }
 }

air/src/execution_step/errors.rs

@@ -114,7 +114,7 @@ pub(crate) enum ExecutionError {
     JsonPathVariableTypeError(JValue),
 
     /// Errors bubbled from a trace handler.
-    #[error("{0}")]
+    #[error(transparent)]
     TraceError(#[from] TraceHandlerError),
 
     /// Errors occurred while insertion of a value inside stream that doesn't have corresponding generation.

air/src/execution_step/trace_handler/data_keeper/errors.rs

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+use super::ExecutedState;
 use thiserror::Error as ThisError;
 
 /// Errors arose out while accessing various interpreter data.
@@ -21,7 +22,7 @@ use thiserror::Error as ThisError;
 pub(crate) enum KeeperError {
     /// Errors occurred when trace_len - trace_position < requested_subtrace_len.
     #[error(
-        "executed trace has {trace_len} elements and current position is {trace_position},\
+        "executed trace has {trace_len} elements and current position is {trace_position}, \
         but tried to set {requested_subtrace_len} subtrace_len"
     )]
     SetSubtraceLenFailed {
@@ -33,7 +34,7 @@ pub(crate) enum KeeperError {
     /// Errors occurred when
     ///     requested_subtrace_len != 0 && requested_pos + requested_subtrace_len > trace_len.
     #[error(
-        "executed trace has {trace_len} elements,\
+        "executed trace has {trace_len} elements, \
         but tried to set {requested_subtrace_len} subtrace_len and {requested_pos} position"
     )]
     SetSubtraceLenAndPosFailed {
@@ -41,4 +42,14 @@ pub(crate) enum KeeperError {
         requested_subtrace_len: usize,
         trace_len: usize,
     },
+
+    /// Errors occurred when Fold FSM tries to obtain stream generation by value_pos from a trace,
+    /// but this value_pos is bigger than the trace length.
+    #[error("requested an element at position '{position}', but executed trace contains only '{trace_len}' elements")]
+    NoElementAtPosition { position: usize, trace_len: usize },
+
+    /// Errors occurred when Fold FSM tries to obtain stream generation by value_pos from a trace,
+    /// but such state doesn't belong to values in streams (it doesn't contain a generation).
+    #[error("expected a state of CallResult(Value::Stream) or Ap types but '{state}' obtained")]
+    NoStreamState { state: ExecutedState },
 }

air/src/execution_step/trace_handler/data_keeper/merge_ctx.rs

@@ -15,6 +15,8 @@
  */
 
 use super::ExecutionTrace;
+use super::KeeperError;
+use super::KeeperResult;
 use super::TraceSlider;
 
 use air_interpreter_data::InterpreterData;
@@ -27,57 +29,51 @@ use std::collections::HashMap;
 pub(crate) struct MergeCtx {
     pub(crate) slider: TraceSlider,
     pub(crate) streams: StreamGenerations,
-    /// This value is used to track the whole trace that each fold is described.
-    /// total_subtrace_len and subtrace_len from a slider are changed in the following way:
-    ///     fold:
-    ///         start: total = fold_states_count, subtrace_len = len of the first iteration
-    ///         i iteration_end: total -= iteration_i len, subtrace_len = len of the i+1 iteration
-    ///         end: total = 0
-    ///     par => total -= [left, right], new_subtrace_len = total - [left, right], pos += [left, right]
-    total_subtrace_len: usize,
 }
 
 impl MergeCtx {
     #[allow(dead_code)]
     pub(crate) fn from_trace(trace: ExecutionTrace) -> Self {
-        let total_subtrace_len = trace.len();
         let slider = TraceSlider::new(trace);
 
         Self {
             slider,
             streams: HashMap::new(),
-            total_subtrace_len,
         }
     }
 
     pub(crate) fn from_data(data: InterpreterData) -> Self {
-        let total_subtrace_len = data.trace.len();
         let slider = TraceSlider::new(data.trace);
 
         Self {
             slider,
             streams: data.streams,
-            total_subtrace_len,
+        }
+    }
+
+    pub(crate) fn try_get_generation(&self, position: u32) -> KeeperResult<u32> {
+        use air_interpreter_data::*;
+
+        let position = position as usize;
+        let state = self
+            .slider
+            .state_at_position(position)
+            .ok_or_else(|| KeeperError::NoElementAtPosition {
+                position,
+                trace_len: self.slider.trace_len(),
+            })?;
+
+        match state {
+            ExecutedState::Call(CallResult::Executed(Value::Stream { generation, .. })) => Ok(*generation),
+            // such Aps are always preceded by Fold where corresponding stream could be used,
+            // so it's been already checked that res_generation is well-formed
+            // and accessing 0th element is safe here
+            ExecutedState::Ap(ap_result) => Ok(ap_result.res_generations[0]),
+            state => Err(KeeperError::NoStreamState { state: state.clone() }),
         }
     }
 
     pub(crate) fn stream_generation(&self, stream_name: &str) -> Option<u32> {
         self.streams.get(stream_name).copied()
     }
-
-    pub(crate) fn set_total_subtrace_len(&mut self, total_subtrace_len: usize) {
-        if total_subtrace_len == 0 {
-            // setting empty subtrace_len is always possible
-            let _ = self.slider.set_subtrace_len(0);
-        }
-
-        self.total_subtrace_len = total_subtrace_len;
-    }
-
-    pub(crate) fn total_subtrace_len(&self) -> usize {
-        if self.total_subtrace_len < self.slider.seen_elements() {
-            return self.slider.subtrace_len();
-        }
-        self.total_subtrace_len - self.slider.seen_elements()
-    }
 }

air/src/execution_step/trace_handler/data_keeper/trace_slider.rs

@@ -103,7 +103,11 @@ impl TraceSlider {
         self.subtrace_len - self.seen_elements
     }
 
-    pub(crate) fn seen_elements(&self) -> usize {
+    pub(super) fn state_at_position(&self, position: usize) -> Option<&ExecutedState> {
-        self.seen_elements
+        self.trace.get(position)
+    }
+
+    pub(super) fn trace_len(&self) -> usize {
+        self.trace.len()
     }
 }

air/src/execution_step/trace_handler/errors.rs

@@ -24,12 +24,12 @@ use thiserror::Error as ThisError;
 #[derive(ThisError, Debug)]
 #[allow(clippy::enum_variant_names)]
 pub(crate) enum TraceHandlerError {
-    #[error("{0}")]
+    #[error(transparent)]
     KeeperError(#[from] KeeperError),
 
-    #[error("{0}")]
+    #[error(transparent)]
     MergeError(#[from] MergeError),
 
-    #[error("{0}")]
+    #[error(transparent)]
     StateFSMError(#[from] StateFSMError),
 }

air/src/execution_step/trace_handler/merger/errors.rs

@@ -36,22 +36,17 @@ pub(crate) enum MergeError {
     #[error("state from {1} `{0:?}` is incompatible with expected {2}")]
     DifferentExecutedStateExpected(ExecutedState, DataType, &'static str),
 
-    #[error("{0:?} contains several subtraces with the same value_pos {1}")]
+    #[error(transparent)]
-    ManyRecordsWithSamePos(FoldResult, usize),
-
-    /// Errors occurred when one of the fold subtrace lore doesn't contain 2 descriptors.
-    #[error("fold contains {0} sublore descriptors, but 2 is expected")]
-    FoldIncorrectSubtracesCount(usize),
-
-    /// Errors bubbled from DataKeeper.
-    #[error("{0}")]
     KeeperError(#[from] KeeperError),
 
-    #[error("{0}")]
+    #[error(transparent)]
     IncorrectApResult(#[from] ApResultError),
 
-    #[error("{0}")]
+    #[error(transparent)]
     IncorrectCallResult(#[from] CallResultError),
+
+    #[error(transparent)]
+    IncorrectFoldResult(#[from] FoldResultError),
 }
 
 #[derive(ThisError, Debug)]
@@ -77,6 +72,20 @@ pub(crate) enum CallResultError {
     DataNotMatchAIR { air_type: String, data_value: Value },
 }
 
+#[derive(ThisError, Debug)]
+pub(crate) enum FoldResultError {
+    #[error("the first {count} subtrace descriptors lens of fold {fold_result:?} overflows")]
+    SubtraceLenOverflow { fold_result: FoldResult, count: usize },
+
+    /// There are several lores with the same value_pos.
+    #[error("{0:?} contains several subtraces with the same value_pos {1}")]
+    SeveralRecordsWithSamePos(FoldResult, usize),
+
+    /// Errors occurred when one of the fold subtrace lore doesn't contain 2 descriptors.
+    #[error("fold contains {0} sublore descriptors, but 2 is expected")]
+    FoldIncorrectSubtracesCount(usize),
+}
+
 impl MergeError {
     // shouldn't be called with both Nones
     pub(crate) fn incompatible_states(

air/src/execution_step/trace_handler/merger/fold_merger.rs

@@ -14,10 +14,10 @@
  * limitations under the License.
  */
 
-mod utils;
+mod fold_lore_resolver;
 
 use super::*;
-pub(crate) use utils::*;
+pub(crate) use fold_lore_resolver::*;
 
 #[derive(Debug, Default, Clone)]
 pub(crate) struct MergerFoldResult {
@@ -33,10 +33,14 @@ pub(crate) fn try_merge_next_state_as_fold(data_keeper: &mut DataKeeper) -> Merg
 
     let fold_result = match (prev_state, current_state) {
         (Some(Fold(prev_fold)), Some(Fold(current_fold))) => {
-            MergerFoldResult::from_fold_results(&prev_fold, &current_fold)
+            MergerFoldResult::from_fold_results(&prev_fold, &current_fold, data_keeper)
+        }
+        (None, Some(Fold(current_fold))) => {
+            MergerFoldResult::from_fold_result(&current_fold, MergeCtxType::Current, data_keeper)
+        }
+        (Some(Fold(prev_fold)), None) => {
+            MergerFoldResult::from_fold_result(&prev_fold, MergeCtxType::Previous, data_keeper)
         }
-        (None, Some(Fold(current_fold))) => MergerFoldResult::from_fold_result(&current_fold, MergeCtxType::Current),
-        (Some(Fold(prev_fold)), None) => MergerFoldResult::from_fold_result(&prev_fold, MergeCtxType::Previous),
         (None, None) => return Ok(MergerFoldResult::default()),
         (prev_state, current_state) => return Err(MergeError::incompatible_states(prev_state, current_state, "fold")),
     }?;
@@ -45,14 +49,18 @@ pub(crate) fn try_merge_next_state_as_fold(data_keeper: &mut DataKeeper) -> Merg
 }
 
 impl MergerFoldResult {
-    pub(self) fn from_fold_result(fold: &FoldResult, ctx_type: MergeCtxType) -> MergeResult<Self> {
+    pub(self) fn from_fold_result(
+        fold: &FoldResult,
+        ctx_type: MergeCtxType,
+        data_keeper: &DataKeeper,
+    ) -> MergeResult<Self> {
         let (prev_fold_lore, current_fold_lore) = match ctx_type {
             MergeCtxType::Previous => {
-                let fold_lore = resolve_fold_lore(fold)?;
+                let fold_lore = resolve_fold_lore(fold, &data_keeper.prev_ctx)?;
                 (fold_lore, <_>::default())
             }
             MergeCtxType::Current => {
-                let fold_lore = resolve_fold_lore(fold)?;
+                let fold_lore = resolve_fold_lore(fold, &data_keeper.current_ctx)?;
                 (<_>::default(), fold_lore)
             }
         };
@@ -65,9 +73,13 @@ impl MergerFoldResult {
         Ok(merge_result)
     }
 
-    pub(self) fn from_fold_results(prev_fold: &FoldResult, current_fold: &FoldResult) -> MergeResult<Self> {
+    pub(self) fn from_fold_results(
-        let prev_fold_lore = resolve_fold_lore(prev_fold)?;
+        prev_fold: &FoldResult,
-        let current_fold_lore = resolve_fold_lore(current_fold)?;
+        current_fold: &FoldResult,
+        data_keeper: &DataKeeper,
+    ) -> MergeResult<Self> {
+        let prev_fold_lore = resolve_fold_lore(prev_fold, &data_keeper.prev_ctx)?;
+        let current_fold_lore = resolve_fold_lore(current_fold, &data_keeper.current_ctx)?;
 
         let merge_result = Self {
             prev_fold_lore,

air/src/execution_step/trace_handler/merger/fold_merger/fold_lore_resolver.rs

@@ -0,0 +1,186 @@
+/*
+ * Copyright 2021 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use super::*;
+use crate::execution_step::trace_handler::data_keeper::MergeCtx;
+
+use air_interpreter_data::FoldSubTraceLore;
+use air_interpreter_data::SubTraceDesc;
+
+use std::collections::HashMap;
+
+#[derive(Debug, Default, Clone, PartialEq, Eq)]
+pub(crate) struct ResolvedFold {
+    pub(crate) lore: HashMap<usize, ResolvedSubTraceDescs>,
+    pub(crate) fold_states_count: usize,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(crate) struct ResolvedSubTraceDescs {
+    pub(crate) before_subtrace: SubTraceDesc,
+    pub(crate) after_subtrace: SubTraceDesc,
+}
+
+pub(super) fn resolve_fold_lore(fold: &FoldResult, merge_ctx: &MergeCtx) -> MergeResult<ResolvedFold> {
+    let (fold_states_count, lens) = compute_lens_convolution(fold, merge_ctx)?;
+
+    let lore = fold.lore.iter().zip(lens).try_fold::<_, _, MergeResult<_>>(
+        HashMap::with_capacity(fold.lore.len()),
+        |mut resolved_lore, (lore, lens)| {
+            let before_subtrace = SubTraceDesc::new(lore.subtraces_desc[0].begin_pos as _, lens.before_len as _);
+            let after_subtrace = SubTraceDesc::new(lore.subtraces_desc[1].begin_pos as _, lens.after_len as _);
+            let resolved_descs = ResolvedSubTraceDescs::new(before_subtrace, after_subtrace);
+
+            match resolved_lore.insert(lore.value_pos as usize, resolved_descs) {
+                Some(_) => Err(FoldResultError::SeveralRecordsWithSamePos(
+                    fold.clone(),
+                    lore.value_pos as usize,
+                ))
+                .map_err(Into::into),
+                None => Ok(resolved_lore),
+            }
+        },
+    )?;
+
+    let resolved_fold_lore = ResolvedFold::new(lore, fold_states_count);
+    Ok(resolved_fold_lore)
+}
+
+/// This function does conversion subtrace_lens of a fold result, it's better to explain it on
+/// examples.
+///
+/// Imagine a fold on stream with 3 elements that have the same generation, in this case the
+/// conversion will look like this:
+/// [1, 1] [2, 2] [3, 3] => [6, 1] [5, 3] [3, 6]
+///   g0     g0     g0
+/// here a number before comma represents count of elements before next, and after the comma - after
+///
+/// For fold with 5 elements of two generations:
+/// [1, 1] [2, 2] [3, 3] [4, 4] [5, 5] [1, 1] => [6, 1] [5, 3] [3, 6] [9, 4] [5, 9] [1, 1]
+///   g0     g0     g0     g1     g1     g2
+///
+/// It could be seen that this function does a convolution of lens with respect to generations.
+/// This is needed to handle (fold (par (next ... cases, because of subtrace_len of a Fold state
+/// describes only states inside this iteration without states that next brings, however a Par
+/// lens describe the whole subtree, where "next" states are included.
+
+// TODO: in future it's possible to change a format of a Fold state to one behaves like Par,
+// because this function adds some overhead
+fn compute_lens_convolution(fold: &FoldResult, merge_ctx: &MergeCtx) -> MergeResult<(usize, Vec<LoresLen>)> {
+    let subtraces_count = fold.lore.len();
+    let mut lens = Vec::with_capacity(subtraces_count);
+    let mut fold_states_count: usize = 0;
+    let mut last_seen_generation = 0;
+    let mut last_seen_generation_pos = 0;
+    let mut cum_after_len = 0;
+
+    for subtrace_id in 0..subtraces_count {
+        let subtrace_lore = &fold.lore[subtrace_id];
+        check_subtrace_lore(subtrace_lore)?;
+
+        let current_generation = merge_ctx.try_get_generation(subtrace_lore.value_pos)?;
+        // TODO: check sequence for monotone
+        if last_seen_generation != current_generation {
+            if subtrace_id > 0 {
+                // do a back traversal for
+                compute_before_lens(&mut lens, last_seen_generation_pos, subtrace_id - 1);
+            }
+            last_seen_generation = current_generation;
+            last_seen_generation_pos = subtrace_id;
+            cum_after_len = 0;
+        }
+
+        let before_len = subtrace_lore.subtraces_desc[0].subtrace_len;
+        let after_len = subtrace_lore.subtraces_desc[1].subtrace_len;
+        // this checks for overflow both cum_before_len and cum_after_len
+        fold_states_count = fold_states_count
+            .checked_add(before_len as usize)
+            .and_then(|v| v.checked_add(after_len as usize))
+            .ok_or_else(|| FoldResultError::SubtraceLenOverflow {
+                fold_result: fold.clone(),
+                count: subtrace_id,
+            })?;
+
+        cum_after_len += after_len;
+
+        // temporary set not cumulative before len
+        let new_lens = LoresLen::new(before_len, cum_after_len);
+        lens.push(new_lens);
+    }
+
+    if subtraces_count > 0 {
+        compute_before_lens(&mut lens, last_seen_generation_pos, subtraces_count - 1);
+    }
+
+    Ok((fold_states_count, lens))
+}
+
+fn compute_before_lens(lore_lens: &mut [LoresLen], begin_pos: usize, end_pos: usize) {
+    let mut cum_before_len = 0;
+
+    for subtrace_id in (begin_pos..=end_pos).rev() {
+        let lens = &mut lore_lens[subtrace_id];
+
+        let current_before_len = lens.before_len;
+        cum_before_len += current_before_len;
+        lens.before_len = cum_before_len;
+    }
+}
+
+fn check_subtrace_lore(subtrace_lore: &FoldSubTraceLore) -> MergeResult<()> {
+    // this limitation is due to current constraint on count of next inside one fold,
+    // for more info please see comments in the interpreter-data crate
+    const SUBTRACE_DESC_COUNT: usize = 2;
+
+    if subtrace_lore.subtraces_desc.len() != SUBTRACE_DESC_COUNT {
+        return Err(FoldResultError::FoldIncorrectSubtracesCount(
+            subtrace_lore.subtraces_desc.len(),
+        ))
+        .map_err(Into::into);
+    }
+
+    Ok(())
+}
+
+impl ResolvedFold {
+    pub(crate) fn new(lore: HashMap<usize, ResolvedSubTraceDescs>, fold_states_count: usize) -> Self {
+        Self {
+            lore,
+            fold_states_count,
+        }
+    }
+}
+
+impl ResolvedSubTraceDescs {
+    pub(self) fn new(before_subtrace: SubTraceDesc, after_subtrace: SubTraceDesc) -> Self {
+        Self {
+            before_subtrace,
+            after_subtrace,
+        }
+    }
+}
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+struct LoresLen {
+    pub(self) before_len: u32,
+    pub(self) after_len: u32,
+}
+
+impl LoresLen {
+    pub(self) fn new(before_len: u32, after_len: u32) -> Self {
+        Self { before_len, after_len }
+    }
+}

air/src/execution_step/trace_handler/merger/fold_merger/utils.rs

@@ -1,88 +0,0 @@
-/*
- * Copyright 2021 Fluence Labs Limited
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-use super::*;
-use air_interpreter_data::FoldSubTraceLore;
-use air_interpreter_data::SubTraceDesc;
-
-use std::collections::HashMap;
-
-#[derive(Debug, Default, Clone, PartialEq, Eq)]
-pub(crate) struct ResolvedFold {
-    pub(crate) lore: HashMap<usize, ResolvedSubTraceDescs>,
-    pub(crate) fold_states_count: usize,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub(crate) struct ResolvedSubTraceDescs {
-    pub(crate) before_subtrace: SubTraceDesc,
-    pub(crate) after_subtrace: SubTraceDesc,
-}
-
-pub(super) fn resolve_fold_lore(fold: &FoldResult) -> MergeResult<ResolvedFold> {
-    let mut lore = HashMap::with_capacity(fold.lore.len());
-    let mut fold_states_count = 0usize;
-
-    for subtrace_lore in fold.lore.iter() {
-        check_subtrace_lore(subtrace_lore)?;
-
-        let resolved_descs = ResolvedSubTraceDescs {
-            before_subtrace: subtrace_lore.subtraces_desc[0],
-            after_subtrace: subtrace_lore.subtraces_desc[1],
-        };
-
-        fold_states_count += resolved_descs.len();
-
-        if lore.insert(subtrace_lore.value_pos as usize, resolved_descs).is_some() {
-            return Err(MergeError::ManyRecordsWithSamePos(
-                fold.clone(),
-                subtrace_lore.value_pos as usize,
-            ));
-        }
-    }
-
-    let resolved_fold_lore = ResolvedFold::new(lore, fold_states_count);
-    Ok(resolved_fold_lore)
-}
-
-fn check_subtrace_lore(subtrace_lore: &FoldSubTraceLore) -> MergeResult<()> {
-    // this limitation is due to current constraint on count of next inside one fold,
-    // for more info please see comments in the interpreter-data crate
-    const SUBTRACE_DESC_COUNT: usize = 2;
-
-    if subtrace_lore.subtraces_desc.len() != SUBTRACE_DESC_COUNT {
-        return Err(MergeError::FoldIncorrectSubtracesCount(
-            subtrace_lore.subtraces_desc.len(),
-        ));
-    }
-
-    Ok(())
-}
-
-impl ResolvedFold {
-    pub(crate) fn new(lore: HashMap<usize, ResolvedSubTraceDescs>, fold_states_count: usize) -> Self {
-        Self {
-            lore,
-            fold_states_count,
-        }
-    }
-}
-
-impl ResolvedSubTraceDescs {
-    pub(crate) fn len(&self) -> usize {
-        self.before_subtrace.subtrace_len as usize + self.after_subtrace.subtrace_len as usize
-    }
-}

air/src/execution_step/trace_handler/merger/mod.rs

@@ -26,6 +26,7 @@ pub(super) use call_merger::try_merge_next_state_as_call;
 pub(crate) use call_merger::MergerCallResult;
 pub(crate) use errors::ApResultError;
 pub(crate) use errors::CallResultError;
+pub(crate) use errors::FoldResultError;
 pub(crate) use errors::MergeError;
 pub(crate) use fold_merger::try_merge_next_state_as_fold;
 pub(crate) use fold_merger::MergerFoldResult;

air/src/execution_step/trace_handler/mod.rs

@@ -30,6 +30,7 @@ pub(crate) type TraceHandlerResult<T> = std::result::Result<T, TraceHandlerError
 
 use air_interpreter_data::*;
 use data_keeper::DataKeeper;
+use data_keeper::MergeCtx;
 use merger::MergeCtxType;
 use merger::MergerFoldResult;
 use merger::ResolvedFold;

air/src/execution_step/trace_handler/state_automata/errors.rs

@@ -53,11 +53,7 @@ pub(crate) enum StateFSMError {
     #[error("underflow is occurred while calculating the new position of a {2} slider for resolved fold {0:?} and current subtrace len {1}'")]
     FoldLenUnderflow(ResolvedFold, usize, MergeCtxType),
 
-    /// Errors occurred while trying to set a total_subtrace_len that is less than
-    #[error("trying to set total_subtrace_len {0} that is less then len set before {1} for {2} ctx")]
-    TotalSubtraceLenIsLess(usize, usize, MergeCtxType),
-
     /// Errors bubbled from DataKeeper.
-    #[error("{0}")]
+    #[error(transparent)]
     KeeperError(#[from] KeeperError),
 }

air/src/execution_step/trace_handler/state_automata/fold_fsm.rs

@@ -49,21 +49,14 @@ pub(crate) struct FoldFSM {
 impl FoldFSM {
     pub(crate) fn from_fold_start(fold_result: MergerFoldResult, data_keeper: &mut DataKeeper) -> FSMResult<Self> {
         let state_inserter = StateInserter::from_keeper(data_keeper);
-        let state_updater =
+        let state_handler =
             CtxStateHandler::prepare(&fold_result.prev_fold_lore, &fold_result.current_fold_lore, data_keeper)?;
 
-        data_keeper
-            .prev_ctx
-            .set_total_subtrace_len(fold_result.prev_fold_lore.fold_states_count);
-        data_keeper
-            .current_ctx
-            .set_total_subtrace_len(fold_result.current_fold_lore.fold_states_count);
-
         let fold_fsm = Self {
             prev_fold: fold_result.prev_fold_lore,
             current_fold: fold_result.current_fold_lore,
             state_inserter,
-            state_handler: state_updater,
+            state_handler,
             ..<_>::default()
         };
 

air/src/execution_step/trace_handler/state_automata/fold_fsm/state_handler.rs

@@ -51,18 +51,15 @@ fn compute_new_state(fold: &ResolvedFold, data_keeper: &DataKeeper, ctx_type: Me
     };
 
     let current_position = ctx.slider.position();
-    let current_len = ctx.slider.subtrace_len();
-
     let pos = current_position
         .checked_add(fold.fold_states_count)
         .ok_or_else(|| StateFSMError::FoldPosOverflow(fold.clone(), current_position, ctx_type))?;
 
+    let current_len = ctx.slider.subtrace_len();
     let subtrace_len = current_len
         .checked_sub(fold.fold_states_count)
         .ok_or_else(|| StateFSMError::FoldLenUnderflow(fold.clone(), current_position, ctx_type))?;
 
-    let total_subtrace_len = ctx.total_subtrace_len() - fold.fold_states_count;
+    let state = CtxState::new(pos, subtrace_len);
-
-    let state = CtxState::new(pos, subtrace_len, total_subtrace_len);
     Ok(state)
 }

air/src/execution_step/trace_handler/state_automata/mod.rs

@@ -36,6 +36,7 @@ use super::DataKeeper;
 use super::ExecutedState;
 use super::FoldResult;
 use super::FoldSubTraceLore;
+use super::MergeCtx;
 use super::MergeCtxType;
 use super::MergerFoldResult;
 use super::ParResult;

air/src/execution_step/trace_handler/state_automata/par_fsm.rs

@@ -48,14 +48,14 @@ impl ParFSM {
         let current_par = ingredients.current_par.unwrap_or_default();
 
         let state_inserter = StateInserter::from_keeper(data_keeper);
-        let state_updater = CtxStateHandler::prepare(prev_par, current_par, data_keeper)?;
+        let state_handler = CtxStateHandler::prepare(prev_par, current_par, data_keeper)?;
         let par_builder = ParBuilder::from_keeper(data_keeper, &state_inserter);
 
         let par_fsm = Self {
             prev_par,
             current_par,
             state_inserter,
-            state_handler: state_updater,
+            state_handler,
             par_builder,
         };
 

air/src/execution_step/trace_handler/state_automata/par_fsm/state_handler.rs

@@ -15,12 +15,9 @@
  */
 
 mod new_states_calculation;
-mod utils;
 
 use super::*;
-use crate::execution_step::trace_handler::state_automata::par_fsm::state_handler::utils::compute_par_total_lens;
 use new_states_calculation::compute_new_states;
-use utils::prepare_total_lens;
 
 /// At the end of a Par execution it's needed to update subtrace_len and positions of both sliders.
 ///
@@ -68,13 +65,14 @@ pub(super) struct CtxStateHandler {
 }
 
 impl CtxStateHandler {
+    /// Prepare new states that sliders will have after finishing executing of each subtree.
     pub(super) fn prepare(
         prev_par: ParResult,
         current_par: ParResult,
         data_keeper: &mut DataKeeper,
     ) -> FSMResult<Self> {
-        let left_pair = prepare_left_pair(prev_par, current_par, data_keeper)?;
+        let left_pair = compute_new_states(data_keeper, prev_par, current_par, SubtreeType::Left)?;
-        let right_pair = prepare_right_pair(prev_par, current_par, data_keeper)?;
+        let right_pair = compute_new_states(data_keeper, prev_par, current_par, SubtreeType::Right)?;
 
         let handler = Self { left_pair, right_pair };
 
@@ -88,34 +86,3 @@ impl CtxStateHandler {
         }
     }
 }
-
-fn prepare_left_pair(
-    prev_par: ParResult,
-    current_par: ParResult,
-    data_keeper: &mut DataKeeper,
-) -> FSMResult<CtxStatesPair> {
-    let (prev_nibble, current_nibble) = compute_new_states(data_keeper, prev_par, current_par, SubtreeType::Left)?;
-    let prev_state = CtxState::from_nibble(prev_nibble, prev_nibble.subtrace_len);
-    let current_state = CtxState::from_nibble(current_nibble, current_nibble.subtrace_len);
-    let pair = CtxStatesPair::new(prev_state, current_state);
-
-    Ok(pair)
-}
-
-fn prepare_right_pair(
-    prev_par: ParResult,
-    current_par: ParResult,
-    data_keeper: &mut DataKeeper,
-) -> FSMResult<CtxStatesPair> {
-    let (prev_par_len, current_par_len) = compute_par_total_lens(prev_par, current_par)?;
-    let (prev_total_len, current_total_len) = prepare_total_lens(prev_par_len, current_par_len, data_keeper)?;
-
-    let prev_pos = data_keeper.prev_slider().position() + prev_par_len;
-    let current_pos = data_keeper.current_slider().position() + current_par_len;
-
-    let prev_state = CtxState::new(prev_pos, prev_total_len, prev_total_len);
-    let current_state = CtxState::new(current_pos, current_total_len, current_total_len);
-    let pair = CtxStatesPair::new(prev_state, current_state);
-
-    Ok(pair)
-}

air/src/execution_step/trace_handler/state_automata/par_fsm/state_handler/new_states_calculation.rs

@@ -15,35 +15,32 @@
  */
 
 use super::*;
+use crate::execution_step::trace_handler::data_keeper::TraceSlider;
 
 pub(super) fn compute_new_states(
     data_keeper: &DataKeeper,
     prev_par: ParResult,
     current_par: ParResult,
     subtree_type: SubtreeType,
-) -> FSMResult<(CtxStateNibble, CtxStateNibble)> {
+) -> FSMResult<CtxStatesPair> {
     let (prev_len, current_len) = match subtree_type {
         SubtreeType::Left => (prev_par.left_size, current_par.left_size),
-        SubtreeType::Right => (prev_par.right_size, current_par.right_size),
+        SubtreeType::Right => {
+            let prev_par_size = prev_par.size().ok_or(StateFSMError::ParLenOverflow(prev_par))?;
+            let current_par_size = current_par.size().ok_or(StateFSMError::ParLenOverflow(current_par))?;
+
+            (prev_par_size as u32, current_par_size as u32)
+        }
     };
 
-    let prev_nibble = compute_new_state(data_keeper, prev_len as usize, MergeCtxType::Previous, prev_par)?;
+    let prev_state = compute_new_state(prev_len as usize, data_keeper.prev_slider(), prev_par)?;
-    let current_nibble = compute_new_state(data_keeper, current_len as usize, MergeCtxType::Current, current_par)?;
+    let current_state = compute_new_state(current_len as usize, data_keeper.current_slider(), current_par)?;
 
-    Ok((prev_nibble, current_nibble))
+    let pair = CtxStatesPair::new(prev_state, current_state);
+    Ok(pair)
 }
 
-fn compute_new_state(
+fn compute_new_state(par_subtree_len: usize, slider: &TraceSlider, par: ParResult) -> FSMResult<CtxState> {
-    data_keeper: &DataKeeper,
-    par_subtree_len: usize,
-    ctx_type: MergeCtxType,
-    par: ParResult,
-) -> FSMResult<CtxStateNibble> {
-    let slider = match ctx_type {
-        MergeCtxType::Previous => data_keeper.prev_slider(),
-        MergeCtxType::Current => data_keeper.current_slider(),
-    };
-
     let pos = slider
         .position()
         .checked_add(par_subtree_len)
@@ -54,6 +51,6 @@ fn compute_new_state(
         .checked_sub(par_subtree_len)
         .ok_or_else(|| StateFSMError::ParLenUnderflow(par, slider.subtrace_len(), MergeCtxType::Current))?;
 
-    let nibble = CtxStateNibble::new(pos, subtrace_len);
+    let new_state = CtxState::new(pos, subtrace_len);
-    Ok(nibble)
+    Ok(new_state)
 }

air/src/execution_step/trace_handler/state_automata/par_fsm/state_handler/utils.rs

@@ -1,63 +0,0 @@
-/*
- * Copyright 2021 Fluence Labs Limited
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-use super::*;
-
-pub(super) fn prepare_total_lens(
-    prev_size: usize,
-    current_size: usize,
-    data_keeper: &mut DataKeeper,
-) -> FSMResult<(usize, usize)> {
-    sizes_suits(prev_size, current_size, data_keeper)?;
-
-    // these lens should be set after end of a par
-    let prev_total_len = data_keeper.prev_ctx.total_subtrace_len() - prev_size;
-    let current_total_len = data_keeper.current_ctx.total_subtrace_len() - current_size;
-
-    data_keeper.prev_ctx.set_total_subtrace_len(prev_size);
-    data_keeper.current_ctx.set_total_subtrace_len(current_size);
-
-    Ok((prev_total_len, current_total_len))
-}
-
-pub(super) fn compute_par_total_lens(prev_par: ParResult, current_par: ParResult) -> FSMResult<(usize, usize)> {
-    let prev_par_len = prev_par.size().ok_or(StateFSMError::ParLenOverflow(prev_par))?;
-    let current_par_len = current_par.size().ok_or(StateFSMError::ParLenOverflow(prev_par))?;
-
-    Ok((prev_par_len, current_par_len))
-}
-
-fn sizes_suits(prev_par_len: usize, current_par_len: usize, data_keeper: &DataKeeper) -> FSMResult<()> {
-    let prev_total_len = data_keeper.prev_ctx.total_subtrace_len();
-    if prev_par_len > prev_total_len {
-        return Err(StateFSMError::TotalSubtraceLenIsLess(
-            prev_par_len,
-            prev_total_len,
-            MergeCtxType::Previous,
-        ));
-    }
-
-    let current_total_len = data_keeper.current_ctx.total_subtrace_len();
-    if current_par_len > current_total_len {
-        return Err(StateFSMError::TotalSubtraceLenIsLess(
-            current_par_len,
-            current_total_len,
-            MergeCtxType::Current,
-        ));
-    }
-
-    Ok(())
-}

air/src/execution_step/trace_handler/state_automata/utils.rs

@@ -15,12 +15,13 @@
  */
 
 use super::DataKeeper;
+use super::FSMResult;
+use super::MergeCtx;
 
 #[derive(Debug, Default, Clone, Copy)]
 pub(super) struct CtxState {
     pub(super) pos: usize,
     pub(super) subtrace_len: usize,
-    pub(super) total_subtrace_len: usize,
 }
 
 #[derive(Debug, Default, Clone, Copy)]
@@ -29,27 +30,15 @@ pub(super) struct CtxStatesPair {
     pub(super) current_state: CtxState,
 }
 
-#[derive(Clone, Copy, Debug, Default)]
-pub(crate) struct CtxStateNibble {
-    pub(crate) pos: usize,
-    pub(crate) subtrace_len: usize,
-}
-
 impl CtxState {
-    pub(super) fn new(pos: usize, subtrace_len: usize, total_subtrace_len: usize) -> Self {
+    pub(super) fn new(pos: usize, subtrace_len: usize) -> Self {
-        Self {
+        Self { pos, subtrace_len }
-            pos,
-            subtrace_len,
-            total_subtrace_len,
-        }
     }
 
-    pub(super) fn from_nibble(nibble: CtxStateNibble, total_subtrace_len: usize) -> Self {
+    pub(super) fn update_ctx_state(self, ctx: &mut MergeCtx) -> FSMResult<()> {
-        Self {
+        ctx.slider
-            pos: nibble.pos,
+            .set_position_and_len(self.pos, self.subtrace_len)
-            subtrace_len: nibble.subtrace_len,
+            .map_err(Into::into)
-            total_subtrace_len,
-        }
     }
 }
 
@@ -62,31 +51,11 @@ impl CtxStatesPair {
     }
 }
 
-impl CtxStateNibble {
-    pub(super) fn new(pos: usize, subtrace_len: usize) -> Self {
-        Self { pos, subtrace_len }
-    }
-}
-
 pub(super) fn update_ctx_states(state_pair: CtxStatesPair, data_keeper: &mut DataKeeper) {
     // these calls shouldn't produce a error, because sizes become less and
     // they have been already checked in a state updater ctor. It's important
     // to make it in a such way, because this function could be called from
     // error_exit that shouldn't fail.
-    let prev_state = state_pair.prev_state;
+    let _ = state_pair.prev_state.update_ctx_state(&mut data_keeper.prev_ctx);
-    let current_state = state_pair.current_state;
+    let _ = state_pair.current_state.update_ctx_state(&mut data_keeper.current_ctx);
-
-    let _ = data_keeper
-        .prev_slider_mut()
-        .set_position_and_len(prev_state.pos, prev_state.subtrace_len);
-    data_keeper
-        .prev_ctx
-        .set_total_subtrace_len(prev_state.total_subtrace_len);
-
-    let _ = data_keeper
-        .current_slider_mut()
-        .set_position_and_len(current_state.pos, current_state.subtrace_len);
-    data_keeper
-        .current_ctx
-        .set_total_subtrace_len(current_state.subtrace_len);
 }

air/tests/test_module/integration/streams.rs

@@ -14,16 +14,7 @@
  * limitations under the License.
  */
 
-use air_test_utils::checked_call_vm;
+use air_test_utils::*;
-use air_test_utils::create_avm;
-use air_test_utils::executed_state;
-use air_test_utils::set_variable_call_service;
-use air_test_utils::trace_from_result;
-use air_test_utils::unit_call_service;
-use air_test_utils::CallServiceClosure;
-use air_test_utils::IValue;
-use air_test_utils::NEVec;
-use air_test_utils::SubTraceDesc;
 
 use serde_json::json;
 use serde_json::Value as JValue;

air/tests/test_module/integration/streams_early_exit.rs

@@ -39,8 +39,8 @@ fn par_early_exit() {
     let init_result_1 = checked_call_vm!(init, "", &script, "", "");
     let setter_1_res = checked_call_vm!(setter_1, "", &script, "", init_result_1.data.clone());
     let setter_2_res = checked_call_vm!(setter_2, "", &script, "", init_result_1.data.clone());
-    let setter_3_res = checked_call_vm!(setter_3, "", &script, "", init_result_1.data.clone());
+    let setter_3_res_1 = checked_call_vm!(setter_3, "", &script, "", init_result_1.data.clone());
-    let actual_trace_3 = trace_from_result(&setter_3_res);
+    let actual_trace_1 = trace_from_result(&setter_3_res_1);
 
     let expected_trace = vec![
         executed_state::scalar_string("test"),
@@ -60,12 +60,51 @@ fn par_early_exit() {
         executed_state::service_failed(1, "error"),
         executed_state::request_sent_by(setter_3_id),
     ];
-    assert_eq!(actual_trace_3, expected_trace);
+    assert_eq!(actual_trace_1, expected_trace);
 
-    let init_result_2 = checked_call_vm!(init, "", &script, init_result_1.data.clone(), setter_1_res.data.clone());
+    let setter_3_res_2 = checked_call_vm!(
-    let init_result_3 = checked_call_vm!(init, "", &script, init_result_2.data.clone(), setter_2_res.data.clone());
+        setter_3,
-    let init_result_4 = checked_call_vm!(init, "", &script, init_result_3.data.clone(), setter_3_res.data.clone());
+        "",
-    let actual_trace_4 = trace_from_result(&init_result_4);
+        &script,
+        setter_3_res_1.data.clone(),
+        setter_1_res.data.clone()
+    );
+    let setter_3_res_3 = checked_call_vm!(
+        setter_3,
+        "",
+        &script,
+        setter_3_res_2.data.clone(),
+        setter_2_res.data.clone()
+    );
+    let init_result_2 = checked_call_vm!(
+        init,
+        "",
+        &script,
+        init_result_1.data.clone(),
+        setter_3_res_3.data.clone()
+    );
+    let actual_trace_2 = trace_from_result(&setter_3_res_3);
+    let actual_trace_3 = trace_from_result(&init_result_2);
+
+    let expected_trace = vec![
+        executed_state::scalar_string("test"),
+        executed_state::par(12, 1),
+        executed_state::par(9, 1),
+        executed_state::par(7, 1),
+        executed_state::par(5, 1),
+        executed_state::par(3, 1),
+        executed_state::par(1, 1),
+        executed_state::stream_string("1", 1),
+        executed_state::stream_string("2", 2),
+        executed_state::stream_string("1", 1),
+        executed_state::stream_string("res", 0),
+        executed_state::service_failed(1, "error"),
+        executed_state::stream_string("res", 0),
+        executed_state::service_failed(1, "error"),
+        executed_state::service_failed(1, "error"),
+        executed_state::request_sent_by("setter_3"),
+    ];
+    assert_eq!(actual_trace_2, expected_trace);
 
     let expected_trace = vec![
         executed_state::scalar_string("test"),
@@ -76,16 +115,16 @@ fn par_early_exit() {
         executed_state::par(3, 1),
         executed_state::par(1, 1),
         executed_state::stream_string("1", 0),
-        executed_state::stream_string("2", 1),
+        executed_state::stream_string("2", 0),
         executed_state::stream_string("1", 0),
-        executed_state::stream_string("res", 2),
+        executed_state::stream_string("res", 0),
         executed_state::service_failed(1, "error"),
-        executed_state::stream_string("res", 2),
+        executed_state::stream_string("res", 0),
         executed_state::service_failed(1, "error"),
         executed_state::service_failed(1, "error"),
         executed_state::scalar_string("test"),
     ];
-    assert_eq!(actual_trace_4, expected_trace);
+    assert_eq!(actual_trace_3, expected_trace);
 
     let setter_3_malicious_trace = vec![
         executed_state::scalar_string("test"),
@@ -103,16 +142,16 @@ fn par_early_exit() {
         executed_state::request_sent_by(setter_3_id),
     ];
     let setter_3_malicious_data = raw_data_from_trace(setter_3_malicious_trace);
-    let init_result_5 = call_vm!(init, "", &script, init_result_3.data.clone(), setter_3_malicious_data);
+    let init_result_3 = call_vm!(init, "", &script, init_result_2.data.clone(), setter_3_malicious_data);
-    assert_eq!(init_result_5.ret_code, 1018);
+    assert_eq!(init_result_3.ret_code, 1018);
 
-    let actual_trace = trace_from_result(&init_result_5);
+    let actual_trace = trace_from_result(&init_result_3);
-    let expected_trace = trace_from_result(&init_result_3);
+    let expected_trace = trace_from_result(&init_result_2);
     assert_eq!(actual_trace, expected_trace);
 }
 
 #[test]
-fn fold_early_exit() {
+fn fold_early_exit__() {
     let variables_setter_id = "set_variable_id";
     let stream_setter_id = "stream_setter_id";
     let fold_executor_id = "fold_executor_id";

avm/server/src/errors.rs

@@ -24,7 +24,7 @@ use std::path::PathBuf;
 #[derive(Debug, ThisError)]
 pub enum AVMError {
     /// FaaS errors.
-    #[error("{0}")]
+    #[error(transparent)]
     FaaSError(#[from] FaaSError),
 
     /// AIR interpreter result deserialization errors.

crates/air-parser/src/parser/errors.rs

@@ -19,7 +19,7 @@ use thiserror::Error as ThisError;
 
 #[derive(ThisError, Debug, Clone, PartialEq, Eq)]
 pub enum ParserError {
-    #[error("{0}")]
+    #[error(transparent)]
     LexerError(#[from] LexerError),
 
     #[error(