feat!(execution-engine): Store call executed values as CIDs in the data (#401)

The trace stores CID strings for call result values.  These strings are to be resolved to real values with `InterpreterData::cid_store` map.

crates/air-lib/interpreter-data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## Version 0.5.0
+
+- Call result values are stored as CIDs in the data trace.  These CIDs refer
+  to a new `cid_store` data's field that maps a CID string to a value.
+
 ## Version 0.4.1
 
 [PR 367](https://github.com/fluencelabs/aquavm/pull/367):  

crates/air-lib/interpreter-data/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "air-interpreter-data"
 description = "Data format of the AIR interpreter"
-version = "0.4.1"
+version = "0.5.0"
 authors = ["Fluence Labs"]
 edition = "2018"
 license = "Apache-2.0"
@@ -16,6 +16,9 @@ path = "src/lib.rs"
 [dependencies]
 air-utils = { path = "../utils" }
 air-parser = { path = "../air-parser" }
+# TODO version?
+air-interpreter-interface = { path = "../interpreter-interface" }
+air-interpreter-cid = { version = "0.1.0", path = "../interpreter-cid" }
 
 serde = {version = "1.0.147", features = ["derive", "rc"]}
 serde_json = "1.0.89"

crates/air-lib/interpreter-data/src/cid_store.rs

@@ -0,0 +1,205 @@
+/*
+ * Copyright 2022 Fluence Labs Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+use crate::JValue;
+
+use air_interpreter_cid::value_to_json_cid;
+use air_interpreter_cid::CidCalculationError;
+use air_interpreter_cid::CID;
+use serde::Deserialize;
+use serde::Serialize;
+
+use std::{collections::HashMap, rc::Rc};
+
+/// Stores CID to Value corresponance.
+#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
+#[serde(transparent)]
+pub struct CidStore<Val>(HashMap<Rc<CID>, Rc<Val>>);
+
+impl<Val> CidStore<Val> {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn get(&self, cid: &CID) -> Option<Rc<Val>> {
+        self.0.get(cid).cloned()
+    }
+}
+
+impl<Val> Default for CidStore<Val> {
+    fn default() -> Self {
+        Self(Default::default())
+    }
+}
+
+#[derive(Clone, Debug)]
+pub struct CidTracker<Val = JValue> {
+    cids: HashMap<Rc<CID>, Rc<Val>>,
+}
+
+impl<Val> CidTracker<Val> {
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    pub fn from_cid_stores(prev_cid_map: CidStore<Val>, current_cid_map: CidStore<Val>) -> Self {
+        let mut cids = prev_cid_map.0;
+        for (cid, val) in current_cid_map.0 {
+            // TODO check that values matches?
+            cids.insert(cid, val);
+        }
+        Self { cids }
+    }
+
+    pub fn get(&self, cid: &CID) -> Option<Rc<Val>> {
+        self.cids.get(cid).cloned()
+    }
+}
+
+impl<Val: Serialize> CidTracker<Val> {
+    pub fn record_value(
+        &mut self,
+        value: impl Into<Rc<Val>>,
+    ) -> Result<Rc<CID>, CidCalculationError> {
+        let value = value.into();
+        let cid = Rc::new(value_to_json_cid(&value)?);
+        self.cids.insert(cid.clone(), value);
+        Ok(cid)
+    }
+}
+
+impl<Val> Default for CidTracker<Val> {
+    fn default() -> Self {
+        Self {
+            cids: Default::default(),
+        }
+    }
+}
+
+impl<Val> From<CidTracker<Val>> for CidStore<Val> {
+    fn from(value: CidTracker<Val>) -> Self {
+        Self(value.cids)
+    }
+}
+
+impl<Val> IntoIterator for CidStore<Val> {
+    type Item = (Rc<CID>, Rc<Val>);
+
+    type IntoIter = std::collections::hash_map::IntoIter<Rc<CID>, Rc<Val>>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        self.0.into_iter()
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::iter::FromIterator;
+
+    use super::*;
+    use serde_json::json;
+
+    #[test]
+    fn test_iter() {
+        let mut tracker = CidTracker::new();
+        tracker.record_value(json!("test")).unwrap();
+        tracker.record_value(json!(1)).unwrap();
+        tracker.record_value(json!([1, 2, 3])).unwrap();
+        tracker
+            .record_value(json!({
+                "key": 42,
+            }))
+            .unwrap();
+        let store = CidStore::from(tracker);
+        assert_eq!(
+            store.into_iter().collect::<HashMap<_, _>>(),
+            HashMap::from_iter(vec![
+                (
+                    CID::new("bagaaierajwlhumardpzj6dv2ahcerm3vyfrjwl7nahg7zq5o3eprwv6v3vpa")
+                        .into(),
+                    json!("test").into()
+                ),
+                (
+                    CID::new("bagaaierauyk65lxcdxsrphpaqdpiymcszdnjaejyibv2ohbyyaziix35kt2a")
+                        .into(),
+                    json!([1, 2, 3]).into(),
+                ),
+                (
+                    CID::new("bagaaieranodle477gt6odhllqbhp6wr7k5d23jhkuixr2soadzjn3n4hlnfq")
+                        .into(),
+                    json!(1).into(),
+                ),
+                (
+                    CID::new("bagaaierad7lci6475zdrps4h6fmcpmqyknz5z6bw6p6tmpjkfyueavqw4kaq")
+                        .into(),
+                    json!({
+                        "key": 42,
+                    })
+                    .into(),
+                )
+            ])
+        );
+    }
+
+    #[test]
+    fn test_store() {
+        let mut tracker = CidTracker::new();
+        tracker.record_value(json!("test")).unwrap();
+        tracker.record_value(json!(1)).unwrap();
+        tracker.record_value(json!([1, 2, 3])).unwrap();
+        tracker
+            .record_value(json!({
+                "key": 42,
+            }))
+            .unwrap();
+        let store = CidStore::from(tracker);
+
+        assert_eq!(
+            &*store
+                .get(&CID::new(
+                    "bagaaierajwlhumardpzj6dv2ahcerm3vyfrjwl7nahg7zq5o3eprwv6v3vpa"
+                ))
+                .unwrap(),
+            &json!("test"),
+        );
+        assert_eq!(
+            &*store
+                .get(&CID::new(
+                    "bagaaierauyk65lxcdxsrphpaqdpiymcszdnjaejyibv2ohbyyaziix35kt2a"
+                ))
+                .unwrap(),
+            &json!([1, 2, 3]),
+        );
+        assert_eq!(
+            &*store
+                .get(&CID::new(
+                    "bagaaieranodle477gt6odhllqbhp6wr7k5d23jhkuixr2soadzjn3n4hlnfq"
+                ))
+                .unwrap(),
+            &json!(1),
+        );
+        assert_eq!(
+            &*store
+                .get(&CID::new(
+                    "bagaaierad7lci6475zdrps4h6fmcpmqyknz5z6bw6p6tmpjkfyueavqw4kaq"
+                ))
+                .unwrap(),
+            &json!({"key": 42}),
+        );
+
+        assert_eq!(store.get(&CID::new("loremimpsumdolorsitament")), None,);
+    }
+}

crates/air-lib/interpreter-data/src/executed_state.rs

@@ -17,13 +17,15 @@
 mod impls;
 mod se_de;
 
+use crate::JValue;
 use crate::TracePos;
 
+use air_interpreter_cid::CID;
 use se_de::par_serializer;
 use se_de::sender_serializer;
 use serde::Deserialize;
 use serde::Serialize;
-use serde_json::Value as JValue;
+
 use std::fmt::Formatter;
 use std::rc::Rc;
 
@@ -48,7 +50,7 @@ pub enum CallResult {
     RequestSentBy(Sender),
 
     /// A corresponding call's been already executed with such value as a result.
-    Executed(Value),
+    Executed(ValueRef),
 
     /// call_service ended with a service error.
     #[serde(rename = "failed")]
@@ -57,9 +59,9 @@ pub enum CallResult {
 
 #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "snake_case")]
-pub enum Value {
-    Scalar(Rc<JValue>),
-    Stream { value: Rc<JValue>, generation: u32 },
+pub enum ValueRef {
+    Scalar(Rc<CID>),
+    Stream { cid: Rc<CID>, generation: u32 },
 }
 
 /// Let's consider an example of trace that could be produces by the following fold:

crates/air-lib/interpreter-data/src/executed_state/impls.rs

@@ -41,14 +41,14 @@ impl CallResult {
         CallResult::RequestSentBy(Sender::PeerIdWithCallId { peer_id, call_id })
     }
 
-    pub fn executed_scalar(value: Rc<JValue>) -> CallResult {
-        let value = Value::Scalar(value);
+    pub fn executed_scalar(cid: Rc<CID>) -> CallResult {
+        let value = ValueRef::Scalar(cid);
 
         CallResult::Executed(value)
     }
 
-    pub fn executed_stream(value: Rc<JValue>, generation: u32) -> CallResult {
-        let value = Value::Stream { value, generation };
+    pub fn executed_stream(cid: Rc<CID>, generation: u32) -> CallResult {
+        let value = ValueRef::Stream { cid, generation };
 
         CallResult::Executed(value)
     }
@@ -136,12 +136,12 @@ impl std::fmt::Display for ExecutedState {
     }
 }
 
-impl std::fmt::Display for Value {
+impl std::fmt::Display for ValueRef {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match self {
-            Value::Scalar(value) => write!(f, "scalar: {value}"),
-            Value::Stream { value, generation } => {
-                write!(f, "stream: {value} generation: {generation}")
+            ValueRef::Scalar(cid) => write!(f, "scalar: {cid:?}"),
+            ValueRef::Stream { cid, generation } => {
+                write!(f, "stream: {cid:?} generation: {generation}")
             }
         }
     }

crates/air-lib/interpreter-data/src/interpreter_data.rs

@@ -16,7 +16,10 @@
 
 use super::GlobalStreamGens;
 use super::RestrictedStreamGens;
+use crate::cid_store::CidStore;
 use crate::ExecutionTrace;
+use crate::JValue;
+
 use air_utils::measure;
 
 use serde::Deserialize;
@@ -53,6 +56,9 @@ pub struct InterpreterData {
 
     /// Version of interpreter produced this data.
     pub interpreter_version: semver::Version,
+
+    /// Map CID to values
+    pub cid_store: CidStore<JValue>,
 }
 
 impl InterpreterData {
@@ -64,6 +70,7 @@ impl InterpreterData {
             last_call_request_id: 0,
             restricted_streams: RestrictedStreamGens::new(),
             interpreter_version,
+            cid_store: <_>::default(),
         }
     }
 
@@ -71,9 +78,12 @@ impl InterpreterData {
         trace: ExecutionTrace,
         streams: GlobalStreamGens,
         restricted_streams: RestrictedStreamGens,
+        cid_store: impl Into<CidStore<JValue>>,
         last_call_request_id: u32,
         interpreter_version: semver::Version,
     ) -> Self {
+        let cid_store = cid_store.into();
+
         Self {
             trace,
             global_streams: streams,
@@ -81,6 +91,7 @@ impl InterpreterData {
             last_call_request_id,
             restricted_streams,
             interpreter_version,
+            cid_store,
         }
     }
 

crates/air-lib/interpreter-data/src/lib.rs

@@ -26,12 +26,14 @@
     unreachable_patterns
 )]
 
+mod cid_store;
 mod executed_state;
 mod interpreter_data;
 mod stream_generations;
 mod trace;
 mod trace_pos;
 
+pub use cid_store::*;
 pub use executed_state::*;
 pub use interpreter_data::*;
 pub use stream_generations::*;
@@ -39,6 +41,8 @@ pub use trace::*;
 pub use trace_pos::*;
 
 use once_cell::sync::Lazy;
+use serde_json::Value as JValue;
+
 use std::str::FromStr;
 
 static DATA_FORMAT_VERSION: Lazy<semver::Version> = Lazy::new(|| {