Merge pull request #36 from tomaka/datastore-arbitrary

Allow datastore to store values of arbitrary type

datastore/Cargo.toml

@@ -7,5 +7,6 @@ authors = ["Parity Technologies <admin@parity.io>"]
 base64 = "0.7"
 futures = "0.1"
 parking_lot = "0.4"
+serde = "1.0"
 serde_json = "1.0"
 tempfile = "2.2"

datastore/src/json_file.rs

@@ -19,43 +19,47 @@
 // DEALINGS IN THE SOFTWARE.
 
 use Datastore;
-use base64;
 use futures::Future;
 use futures::stream::{Stream, iter_ok};
+use parking_lot::Mutex;
 use query::{Query, naive_apply_query};
-use serde_json::{from_reader, to_writer};
+use serde::Serialize;
-use serde_json::map::Map;
+use serde::de::DeserializeOwned;
+use serde_json::{Map, from_value, to_value, from_reader, to_writer};
 use serde_json::value::Value;
 use std::borrow::Cow;
+use std::collections::HashMap;
 use std::fs;
 use std::io::Cursor;
 use std::io::Error as IoError;
 use std::io::ErrorKind as IoErrorKind;
 use std::io::Read;
 use std::path::PathBuf;
-use parking_lot::Mutex;
 use tempfile::NamedTempFile;
 
 /// Implementation of `Datastore` that uses a single plain JSON file.
-pub struct JsonFileDatastore {
+pub struct JsonFileDatastore<T>
+	where T: Serialize + DeserializeOwned
+{
 	path: PathBuf,
-    content: Mutex<Map<String, Value>>,
+	content: Mutex<HashMap<String, T>>,
 }
 
-impl JsonFileDatastore {
+impl<T> JsonFileDatastore<T>
+	where T: Serialize + DeserializeOwned
+{
 	/// Opens or creates the datastore. If the path refers to an existing path, then this function
 	/// will attempt to load an existing set of values from it (which can result in an error).
 	/// Otherwise if the path doesn't exist, a new empty datastore will be created.
-    pub fn new<P>(path: P) -> Result<JsonFileDatastore, IoError>
+	pub fn new<P>(path: P) -> Result<JsonFileDatastore<T>, IoError>
-    where
+		where P: Into<PathBuf>
-        P: Into<PathBuf>,
 	{
 		let path = path.into();
 
 		if !path.exists() {
 			return Ok(JsonFileDatastore {
 				path: path,
-                content: Mutex::new(Map::new()),
+				content: Mutex::new(HashMap::new()),
 			});
 		}
 
@@ -70,18 +74,27 @@ impl JsonFileDatastore {
 			let mut first_byte = [0];
 			if file.read(&mut first_byte)? == 0 {
 				// File is empty.
-                Map::new()
+				HashMap::new()
 			} else {
 				match from_reader::<_, Value>(Cursor::new(first_byte).chain(file)) {
-                    Ok(Value::Null) => Map::new(),
+					Ok(Value::Null) => HashMap::new(),
-                    Ok(Value::Object(map)) => map,
+					Ok(Value::Object(map)) => {
-                    Ok(_) => {
+						let mut out = HashMap::with_capacity(map.len());
-                        return Err(IoError::new(
+						for (key, value) in map.into_iter() {
-                            IoErrorKind::InvalidData,
+							let value = match from_value(value) {
-                            "expected JSON object",
+								Ok(v) => v,
-                        ));
-                    }
 								Err(err) => return Err(IoError::new(IoErrorKind::InvalidData, err)),
+							};
+							out.insert(key, value);
+						}
+						out
+					}
+					Ok(_) => {
+						return Err(IoError::new(IoErrorKind::InvalidData, "expected JSON object"));
+					}
+					Err(err) => {
+						return Err(IoError::new(IoErrorKind::InvalidData, err));
+					}
 				}
 			}
 		};
@@ -100,14 +113,19 @@ impl JsonFileDatastore {
 	pub fn flush(&self) -> Result<(), IoError> {
 		// Create a temporary file in the same directory as the destination, which avoids the
 		// problem of having a file cleaner delete our file while we use it.
-        let self_path_parent = self.path.parent().ok_or(IoError::new(
+		let self_path_parent = self.path
+								   .parent()
+								   .ok_or(IoError::new(
 			IoErrorKind::Other,
 			"couldn't get parent directory of destination",
 		))?;
 		let mut temporary_file = NamedTempFile::new_in(self_path_parent)?;
 
 		let content = self.content.lock();
-        to_writer(&mut temporary_file, &*content)?;
+		to_writer(
+			&mut temporary_file,
+			&content.iter().map(|(k, v)| (k.clone(), to_value(v).unwrap())).collect::<Map<_, _>>(),
+		)?; // TODO: panic!
 		temporary_file.sync_data()?;
 
 		// Note that `persist` will fail if we try to persist across filesystems. However that
@@ -118,19 +136,19 @@ impl JsonFileDatastore {
 	}
 }
 
-impl Datastore for JsonFileDatastore {
+impl<T> Datastore<T> for JsonFileDatastore<T>
-    fn put(&self, key: Cow<str>, value: Vec<u8>) {
+	where T: Clone + Serialize + DeserializeOwned + Default + Ord + 'static
+{
+	#[inline]
+	fn put(&self, key: Cow<str>, value: T) {
 		let mut content = self.content.lock();
-        content.insert(key.into_owned(), Value::String(base64::encode(&value)));
+		content.insert(key.into_owned(), value);
 	}
 
-    fn get(&self, key: &str) -> Option<Vec<u8>> {
+	fn get(&self, key: &str) -> Option<T> {
 		let content = self.content.lock();
 		// If the JSON is malformed, we just ignore the value.
-        content.get(key).and_then(|val| match val {
+		content.get(key).cloned()
-            &Value::String(ref s) => base64::decode(s).ok(),
-            _ => None,
-        })
 	}
 
 	fn has(&self, key: &str) -> bool {
@@ -145,27 +163,15 @@ impl Datastore for JsonFileDatastore {
 
 	fn query<'a>(
 		&'a self,
-        query: Query,
+		query: Query<T>,
-    ) -> Box<Stream<Item = (String, Vec<u8>), Error = IoError> + 'a> {
+	) -> Box<Stream<Item = (String, T), Error = IoError> + 'a> {
 		let content = self.content.lock();
 
 		let keys_only = query.keys_only;
 
 		let content_stream = iter_ok(content.iter().filter_map(|(key, value)| {
 			// Skip values that are malformed.
-            let value = if keys_only {
+			let value = if keys_only { Default::default() } else { value.clone() };
-                Vec::with_capacity(0)
-            } else {
-                match value {
-                    &Value::String(ref s) => {
-                        match base64::decode(s) {
-                            Ok(s) => s,
-                            Err(_) => return None,
-                        }
-                    }
-                    _ => return None,
-                }
-            };
 
 			Some((key.clone(), value))
 		}));
@@ -182,7 +188,9 @@ impl Datastore for JsonFileDatastore {
 	}
 }
 
-impl Drop for JsonFileDatastore {
+impl<T> Drop for JsonFileDatastore<T>
+	where T: Serialize + DeserializeOwned
+{
 	#[inline]
 	fn drop(&mut self) {
 		// Unfortunately there's not much we can do here in case of an error, as panicking would be
@@ -206,7 +214,7 @@ mod tests {
 	#[test]
 	fn open_and_flush() {
 		let temp_file = NamedTempFile::new().unwrap();
-        let datastore = JsonFileDatastore::new(temp_file.path()).unwrap();
+		let datastore = JsonFileDatastore::<Vec<u8>>::new(temp_file.path()).unwrap();
 		datastore.flush().unwrap();
 	}
 
@@ -214,13 +222,13 @@ mod tests {
 	fn values_store_and_reload() {
 		let temp_file = NamedTempFile::new().unwrap();
 
-        let datastore = JsonFileDatastore::new(temp_file.path()).unwrap();
+		let datastore = JsonFileDatastore::<Vec<u8>>::new(temp_file.path()).unwrap();
 		datastore.put("foo".into(), vec![1, 2, 3]);
 		datastore.put("bar".into(), vec![0, 255, 127]);
 		datastore.flush().unwrap();
 		drop(datastore);
 
-        let reload = JsonFileDatastore::new(temp_file.path()).unwrap();
+		let reload = JsonFileDatastore::<Vec<u8>>::new(temp_file.path()).unwrap();
 		assert_eq!(reload.get("bar").unwrap(), &[0, 255, 127]);
 		assert_eq!(reload.get("foo").unwrap(), &[1, 2, 3]);
 	}
@@ -229,22 +237,21 @@ mod tests {
 	fn query_basic() {
 		let temp_file = NamedTempFile::new().unwrap();
 
-        let datastore = JsonFileDatastore::new(temp_file.path()).unwrap();
+		let datastore = JsonFileDatastore::<Vec<u8>>::new(temp_file.path()).unwrap();
-        datastore.put("foo1".into(), vec![1, 2, 3]);
+		datastore.put("foo1".into(), vec![6, 7, 8]);
-        datastore.put("foo2".into(), vec![4, 5, 6]);
+		datastore.put("foo2".into(), vec![6, 7, 8]);
 		datastore.put("foo3".into(), vec![7, 8, 9]);
 		datastore.put("foo4".into(), vec![10, 11, 12]);
 		datastore.put("foo5".into(), vec![13, 14, 15]);
 		datastore.put("bar1".into(), vec![0, 255, 127]);
 		datastore.flush().unwrap();
 
-        let query = datastore
+		let query = datastore.query(Query {
-            .query(Query {
 			prefix: "fo".into(),
 			filters: vec![
 				Filter {
-                        ty: FilterTy::ValueCompare(vec![6, 7, 8].into()),
+					ty: FilterTy::ValueCompare(&vec![6, 7, 8].into()),
-                        operation: FilterOp::Greater,
+					operation: FilterOp::NotEqual,
 				},
 			],
 			orders: vec![Order::ByKeyDesc],

datastore/src/lib.rs

@@ -22,6 +22,7 @@ extern crate base64;
 #[macro_use]
 extern crate futures;
 extern crate parking_lot;
+extern crate serde;
 extern crate serde_json;
 extern crate tempfile;
 
@@ -36,13 +37,13 @@ pub use self::json_file::JsonFileDatastore;
 pub use self::query::{Query, Order, Filter, FilterTy, FilterOp};
 
 /// Abstraction over any struct that can store `(key, value)` pairs.
-pub trait Datastore {
+pub trait Datastore<T> {
 	/// Sets the value of a key.
-    fn put(&self, key: Cow<str>, value: Vec<u8>);
+	fn put(&self, key: Cow<str>, value: T);
 
 	/// Returns the value corresponding to this key.
 	// TODO: use higher-kinded stuff once stable to provide a more generic "accessor" for the data
-    fn get(&self, key: &str) -> Option<Vec<u8>>;
+	fn get(&self, key: &str) -> Option<T>;
 
 	/// Returns true if the datastore contains the given key.
 	fn has(&self, key: &str) -> bool;
@@ -56,6 +57,6 @@ pub trait Datastore {
 	/// responsibility to pick the right implementation for the right job.
 	fn query<'a>(
 		&'a self,
-        query: Query,
+		query: Query<T>,
-    ) -> Box<Stream<Item = (String, Vec<u8>), Error = IoError> + 'a>;
+	) -> Box<Stream<Item = (String, T), Error = IoError> + 'a>;
 }

datastore/src/query.rs

@@ -31,11 +31,11 @@ use std::vec::IntoIter as VecIntoIter;
 /// The various modifications of the dataset are applied in the same order as the fields (prefix,
 /// filters, orders, skip, limit).
 #[derive(Debug, Clone)]
-pub struct Query<'a> {
+pub struct Query<'a, T: 'a> {
 	/// Only the keys that start with `prefix` will be returned.
 	pub prefix: Cow<'a, str>,
 	/// Filters to apply on the results.
-    pub filters: Vec<Filter<'a>>,
+	pub filters: Vec<Filter<'a, T>>,
 	/// How to order the keys. Applied sequentially.
 	pub orders: Vec<Order>,
 	/// Number of elements to skip from at the start of the results.
@@ -48,30 +48,29 @@ pub struct Query<'a> {
 
 /// A filter to apply to the results set.
 #[derive(Debug, Clone)]
-pub struct Filter<'a> {
+pub struct Filter<'a, T: 'a> {
 	/// Type of filter and value to compare with.
-    pub ty: FilterTy<'a>,
+	pub ty: FilterTy<'a, T>,
 	/// Comparison operation.
 	pub operation: FilterOp,
 }
 
 /// Type of filter and value to compare with.
 #[derive(Debug, Clone)]
-pub enum FilterTy<'a> {
+pub enum FilterTy<'a, T: 'a> {
 	/// Compare the key with a reference value.
 	KeyCompare(Cow<'a, str>),
 	/// Compare the value with a reference value.
-    ValueCompare(Cow<'a, [u8]>),
+	ValueCompare(&'a T),
 }
 
-/// Filtering operation. Keep in mind that anything else than `Equal` and `NotEqual` is a bit
+/// Filtering operation.
-/// blurry.
 #[derive(Debug, Copy, Clone)]
 pub enum FilterOp {
-    Less,
-    LessOrEqual,
 	Equal,
 	NotEqual,
+	Less,
+	LessOrEqual,
 	Greater,
 	GreaterOrEqual,
 }
@@ -90,9 +89,10 @@ pub enum Order {
 }
 
 /// Naively applies a query on a set of results.
-pub fn naive_apply_query<'a, S>(stream: S, query: Query<'a>)
+pub fn naive_apply_query<'a, S, V>(stream: S, query: Query<'a, V>)
-        -> StreamTake<StreamSkip<NaiveKeysOnlyApply<NaiveApplyOrdered<NaiveFiltersApply<'a, NaivePrefixApply<'a, S>, VecIntoIter<Filter<'a>>>>>>>
+		-> StreamTake<StreamSkip<NaiveKeysOnlyApply<NaiveApplyOrdered<NaiveFiltersApply<'a, NaivePrefixApply<'a, S>, VecIntoIter<Filter<'a, V>>>, V>>>>
-    where S: Stream<Item = (String, Vec<u8>), Error = IoError> + 'a
+	where S: Stream<Item = (String, V), Error = IoError> + 'a,
+		  V: Clone + Ord + Default + 'static
 {
 	let prefixed = naive_apply_prefix(stream, query.prefix);
 	let filtered = naive_apply_filters(prefixed, query.filters.into_iter());
@@ -103,18 +103,16 @@ pub fn naive_apply_query<'a, S>(stream: S, query: Query<'a>)
 
 /// Skips the `skip` first element of a stream and only returns `limit` elements.
 #[inline]
-pub fn naive_apply_skip_limit<S>(stream: S, skip: u64, limit: u64) -> StreamTake<StreamSkip<S>>
+pub fn naive_apply_skip_limit<S, T>(stream: S, skip: u64, limit: u64) -> StreamTake<StreamSkip<S>>
-where
+	where S: Stream<Item = (String, T), Error = IoError>
-    S: Stream<Item = (String, Vec<u8>), Error = IoError>,
 {
 	stream.skip(skip).take(limit)
 }
 
 /// Filters the result of a stream to empty values if `keys_only` is true.
 #[inline]
-pub fn naive_apply_keys_only<S>(stream: S, keys_only: bool) -> NaiveKeysOnlyApply<S>
+pub fn naive_apply_keys_only<S, T>(stream: S, keys_only: bool) -> NaiveKeysOnlyApply<S>
-where
+	where S: Stream<Item = (String, T), Error = IoError>
-    S: Stream<Item = (String, Vec<u8>), Error = IoError>,
 {
 	NaiveKeysOnlyApply {
 		keys_only: keys_only,
@@ -129,18 +127,18 @@ pub struct NaiveKeysOnlyApply<S> {
 	stream: S,
 }
 
-impl<S> Stream for NaiveKeysOnlyApply<S>
+impl<S, T> Stream for NaiveKeysOnlyApply<S>
-where
+	where S: Stream<Item = (String, T), Error = IoError>,
-    S: Stream<Item = (String, Vec<u8>), Error = IoError>,
+		  T: Default
 {
-    type Item = (String, Vec<u8>);
+	type Item = (String, T);
 	type Error = IoError;
 
 	#[inline]
 	fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
 		if self.keys_only {
 			Ok(Async::Ready(try_ready!(self.stream.poll()).map(|mut v| {
-                v.1 = Vec::new();
+				v.1 = Default::default();
 				v
 			})))
 		} else {
@@ -151,14 +149,10 @@ where
 
 /// Filters the result of a stream to only keep the results with a prefix.
 #[inline]
-pub fn naive_apply_prefix<'a, S>(stream: S, prefix: Cow<'a, str>) -> NaivePrefixApply<'a, S>
+pub fn naive_apply_prefix<'a, S, T>(stream: S, prefix: Cow<'a, str>) -> NaivePrefixApply<'a, S>
-where
+	where S: Stream<Item = (String, T), Error = IoError>
-    S: Stream<Item = (String, Vec<u8>), Error = IoError>,
 {
-    NaivePrefixApply {
+	NaivePrefixApply { prefix: prefix, stream: stream }
-        prefix: prefix,
-        stream: stream,
-    }
 }
 
 /// Returned by `naive_apply_prefix`.
@@ -168,11 +162,10 @@ pub struct NaivePrefixApply<'a, S> {
 	stream: S,
 }
 
-impl<'a, S> Stream for NaivePrefixApply<'a, S>
+impl<'a, S, T> Stream for NaivePrefixApply<'a, S>
-where
+	where S: Stream<Item = (String, T), Error = IoError>
-    S: Stream<Item = (String, Vec<u8>), Error = IoError>,
 {
-    type Item = (String, Vec<u8>);
+	type Item = (String, T);
 	type Error = IoError;
 
 	#[inline]
@@ -193,19 +186,18 @@ where
 
 /// Applies orderings on the stream data. Will simply pass data through if the list of orderings
 /// is empty. Otherwise will need to collect.
-pub fn naive_apply_ordered<'a, S, I>(stream: S, orders_iter: I) -> NaiveApplyOrdered<'a, S>
+pub fn naive_apply_ordered<'a, S, I, V>(stream: S, orders_iter: I) -> NaiveApplyOrdered<'a, S, V>
-where
+	where S: Stream<Item = (String, V), Error = IoError> + 'a,
-    S: Stream<Item = (String, Vec<u8>), Error = IoError> + 'a,
 		  I: IntoIterator<Item = Order>,
 		  I::IntoIter: 'a,
+		  V: Ord + 'static
 {
 	let orders_iter = orders_iter.into_iter();
 	if orders_iter.size_hint().1 == Some(0) {
 		return NaiveApplyOrdered { inner: NaiveApplyOrderedInner::PassThrough(stream) };
 	}
 
-    let collected = stream
+	let collected = stream.collect()
-        .collect()
 						  .and_then(move |mut collected| {
 		for order in orders_iter {
 			match order {
@@ -231,20 +223,19 @@ where
 }
 
 /// Returned by `naive_apply_ordered`.
-pub struct NaiveApplyOrdered<'a, S> {
+pub struct NaiveApplyOrdered<'a, S, T> {
-    inner: NaiveApplyOrderedInner<'a, S>,
+	inner: NaiveApplyOrderedInner<'a, S, T>,
 }
 
-enum NaiveApplyOrderedInner<'a, S> {
+enum NaiveApplyOrderedInner<'a, S, T> {
 	PassThrough(S),
-    Collected(Box<Stream<Item = (String, Vec<u8>), Error = IoError> + 'a>),
+	Collected(Box<Stream<Item = (String, T), Error = IoError> + 'a>),
 }
 
-impl<'a, S> Stream for NaiveApplyOrdered<'a, S>
+impl<'a, S, V> Stream for NaiveApplyOrdered<'a, S, V>
-where
+	where S: Stream<Item = (String, V), Error = IoError>
-    S: Stream<Item = (String, Vec<u8>), Error = IoError>,
 {
-    type Item = (String, Vec<u8>);
+	type Item = (String, V);
 	type Error = IoError;
 
 	#[inline]
@@ -258,10 +249,10 @@ where
 
 /// Filters the result of a stream to apply a set of filters.
 #[inline]
-pub fn naive_apply_filters<'a, S, I>(stream: S, filters: I) -> NaiveFiltersApply<'a, S, I>
+pub fn naive_apply_filters<'a, S, I, V>(stream: S, filters: I) -> NaiveFiltersApply<'a, S, I>
-where
+	where S: Stream<Item = (String, V), Error = IoError>,
-    S: Stream<Item = (String, Vec<u8>), Error = IoError>,
+		  I: Iterator<Item = Filter<'a, V>> + Clone,
-    I: Iterator<Item = Filter<'a>> + Clone,
+		  V: 'a
 {
 	NaiveFiltersApply {
 		filters: filters,
@@ -278,15 +269,12 @@ pub struct NaiveFiltersApply<'a, S, I> {
 	marker: PhantomData<&'a ()>,
 }
 
-impl<'a, S, I> Stream for NaiveFiltersApply<'a, S, I>
+impl<'a, S, I, T> Stream for NaiveFiltersApply<'a, S, I>
-where
+	where S: Stream<Item = (String, T), Error = IoError>,
-    S: Stream<
+		  I: Iterator<Item = Filter<'a, T>> + Clone,
-        Item = (String, Vec<u8>),
+		  T: Ord + 'a
-        Error = IoError,
-    >,
-    I: Iterator<Item = Filter<'a>> + Clone,
 {
-    type Item = (String, Vec<u8>);
+	type Item = (String, T);
 	type Error = IoError;
 
 	#[inline]
@@ -309,14 +297,16 @@ where
 }
 
 #[inline]
-fn naive_filter_test(entry: &(String, Vec<u8>), filter: &Filter) -> bool {
+fn naive_filter_test<T>(entry: &(String, T), filter: &Filter<T>) -> bool
+	where T: Ord
+{
 	let (expected_ordering, revert_expected) = match filter.operation {
+		FilterOp::Equal => (Ordering::Equal, false),
+		FilterOp::NotEqual => (Ordering::Equal, true),
 		FilterOp::Less => (Ordering::Less, false),
-        FilterOp::LessOrEqual => (Ordering::Greater, true),
-        FilterOp::Equal => (Ordering::Less, false),
-        FilterOp::NotEqual => (Ordering::Less, true),
-        FilterOp::Greater => (Ordering::Greater, false),
 		FilterOp::GreaterOrEqual => (Ordering::Less, true),
+		FilterOp::Greater => (Ordering::Greater, false),
+		FilterOp::LessOrEqual => (Ordering::Greater, true),
 	};
 
 	match filter.ty {
@@ -324,7 +314,7 @@ fn naive_filter_test(entry: &(String, Vec<u8>), filter: &Filter) -> bool {
 			((&*entry.0).cmp(&**ref_value) == expected_ordering) != revert_expected
 		}
 		FilterTy::ValueCompare(ref ref_value) => {
-            ((&*entry.1).cmp(&**ref_value) == expected_ordering) != revert_expected
+			(entry.1.cmp(&**ref_value) == expected_ordering) != revert_expected
 		}
 	}
 }