fluencelabs/rust-libp2p
1
0
Fork 0
mirror of https://github.com/fluencelabs/rust-libp2p synced 2025-05-11 18:37:13 +00:00
Code Issues Projects Releases Wiki Activity

Merge pull request #36 from tomaka/datastore-arbitrary

Browse Source 
Allow datastore to store values of arbitrary type
This commit is contained in:
Fredrik Harrysson 2017-11-16 10:37:35 +01:00 committed by GitHub
commit a62f9f801d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 458 additions and 459 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
datastore/Cargo.toml
View File

@ -7,5 +7,6 @@ authors = ["Parity Technologies <admin@parity.io>"]
base64 = "0.7"
futures = "0.1"
parking_lot = "0.4"
serde = "1.0"
serde_json = "1.0"
tempfile = "2.2"

119
datastore/src/json_file.rs
View File

@ -19,43 +19,47 @@
// DEALINGS IN THE SOFTWARE.
use Datastore;
use base64;
use futures::Future;
use futures::stream::{Stream, iter_ok};
use parking_lot::Mutex;
use query::{Query, naive_apply_query};
use serde_json::{from_reader, to_writer};
use serde_json::map::Map;
use serde::Serialize;
use serde::de::DeserializeOwned;
use serde_json::{Map, from_value, to_value, from_reader, to_writer};
use serde_json::value::Value;
use std::borrow::Cow;
use std::collections::HashMap;
use std::fs;
use std::io::Cursor;
use std::io::Error as IoError;
use std::io::ErrorKind as IoErrorKind;
use std::io::Read;
use std::path::PathBuf;
use parking_lot::Mutex;
use tempfile::NamedTempFile;
/// Implementation of `Datastore` that uses a single plain JSON file.
pub struct JsonFileDatastore {
pub struct JsonFileDatastore<T>
where T: Serialize + DeserializeOwned
{
path: PathBuf,
content: Mutex<Map<String, Value>>,
content: Mutex<HashMap<String, T>>,
}
impl JsonFileDatastore {
impl<T> JsonFileDatastore<T>
where T: Serialize + DeserializeOwned
{
/// Opens or creates the datastore. If the path refers to an existing path, then this function
/// will attempt to load an existing set of values from it (which can result in an error).
/// Otherwise if the path doesn't exist, a new empty datastore will be created.
pub fn new<P>(path: P) -> Result<JsonFileDatastore, IoError>
where
P: Into<PathBuf>,
pub fn new<P>(path: P) -> Result<JsonFileDatastore<T>, IoError>
where P: Into<PathBuf>
{
let path = path.into();
if !path.exists() {
return Ok(JsonFileDatastore {
path: path,
content: Mutex::new(Map::new()),
content: Mutex::new(HashMap::new()),
});
}
@ -70,18 +74,27 @@ impl JsonFileDatastore {
let mut first_byte = [0];
if file.read(&mut first_byte)? == 0 {
// File is empty.
Map::new()
HashMap::new()
} else {
match from_reader::<_, Value>(Cursor::new(first_byte).chain(file)) {
Ok(Value::Null) => Map::new(),
Ok(Value::Object(map)) => map,
Ok(_) => {
return Err(IoError::new(
IoErrorKind::InvalidData,
"expected JSON object",
));
}
Ok(Value::Null) => HashMap::new(),
Ok(Value::Object(map)) => {
let mut out = HashMap::with_capacity(map.len());
for (key, value) in map.into_iter() {
let value = match from_value(value) {
Ok(v) => v,
Err(err) => return Err(IoError::new(IoErrorKind::InvalidData, err)),
};
out.insert(key, value);
}
out
}
Ok(_) => {
return Err(IoError::new(IoErrorKind::InvalidData, "expected JSON object"));
}
Err(err) => {
return Err(IoError::new(IoErrorKind::InvalidData, err));
}
}
}
};
@ -100,14 +113,19 @@ impl JsonFileDatastore {
pub fn flush(&self) -> Result<(), IoError> {
// Create a temporary file in the same directory as the destination, which avoids the
// problem of having a file cleaner delete our file while we use it.
let self_path_parent = self.path.parent().ok_or(IoError::new(
let self_path_parent = self.path
.parent()
.ok_or(IoError::new(
IoErrorKind::Other,
"couldn't get parent directory of destination",
))?;
let mut temporary_file = NamedTempFile::new_in(self_path_parent)?;
let content = self.content.lock();
to_writer(&mut temporary_file, &*content)?;
to_writer(
&mut temporary_file,
&content.iter().map(|(k, v)| (k.clone(), to_value(v).unwrap())).collect::<Map<_, _>>(),
)?; // TODO: panic!
temporary_file.sync_data()?;
// Note that `persist` will fail if we try to persist across filesystems. However that
@ -118,19 +136,19 @@ impl JsonFileDatastore {
}
}
impl Datastore for JsonFileDatastore {
fn put(&self, key: Cow<str>, value: Vec<u8>) {
impl<T> Datastore<T> for JsonFileDatastore<T>
where T: Clone + Serialize + DeserializeOwned + Default + Ord + 'static
{
#[inline]
fn put(&self, key: Cow<str>, value: T) {
let mut content = self.content.lock();
content.insert(key.into_owned(), Value::String(base64::encode(&value)));
content.insert(key.into_owned(), value);
}
fn get(&self, key: &str) -> Option<Vec<u8>> {
fn get(&self, key: &str) -> Option<T> {
let content = self.content.lock();
// If the JSON is malformed, we just ignore the value.
content.get(key).and_then(|val| match val {
&Value::String(ref s) => base64::decode(s).ok(),
_ => None,
})
content.get(key).cloned()
}
fn has(&self, key: &str) -> bool {
@ -145,27 +163,15 @@ impl Datastore for JsonFileDatastore {
fn query<'a>(
&'a self,
query: Query,
) -> Box<Stream<Item = (String, Vec<u8>), Error = IoError> + 'a> {
query: Query<T>,
) -> Box<Stream<Item = (String, T), Error = IoError> + 'a> {
let content = self.content.lock();
let keys_only = query.keys_only;
let content_stream = iter_ok(content.iter().filter_map(|(key, value)| {
// Skip values that are malformed.
let value = if keys_only {
Vec::with_capacity(0)
} else {
match value {
&Value::String(ref s) => {
match base64::decode(s) {
Ok(s) => s,
Err(_) => return None,
}
}
_ => return None,
}
};
let value = if keys_only { Default::default() } else { value.clone() };
Some((key.clone(), value))
}));
@ -182,7 +188,9 @@ impl Datastore for JsonFileDatastore {
}
}
impl Drop for JsonFileDatastore {
impl<T> Drop for JsonFileDatastore<T>
where T: Serialize + DeserializeOwned
{
#[inline]
fn drop(&mut self) {
// Unfortunately there's not much we can do here in case of an error, as panicking would be
@ -206,7 +214,7 @@ mod tests {
#[test]
fn open_and_flush() {
let temp_file = NamedTempFile::new().unwrap();
let datastore = JsonFileDatastore::new(temp_file.path()).unwrap();
let datastore = JsonFileDatastore::<Vec<u8>>::new(temp_file.path()).unwrap();
datastore.flush().unwrap();
}
@ -214,13 +222,13 @@ mod tests {
fn values_store_and_reload() {
let temp_file = NamedTempFile::new().unwrap();
let datastore = JsonFileDatastore::new(temp_file.path()).unwrap();
let datastore = JsonFileDatastore::<Vec<u8>>::new(temp_file.path()).unwrap();
datastore.put("foo".into(), vec![1, 2, 3]);
datastore.put("bar".into(), vec![0, 255, 127]);
datastore.flush().unwrap();
drop(datastore);
let reload = JsonFileDatastore::new(temp_file.path()).unwrap();
let reload = JsonFileDatastore::<Vec<u8>>::new(temp_file.path()).unwrap();
assert_eq!(reload.get("bar").unwrap(), &[0, 255, 127]);
assert_eq!(reload.get("foo").unwrap(), &[1, 2, 3]);
}
@ -229,22 +237,21 @@ mod tests {
fn query_basic() {
let temp_file = NamedTempFile::new().unwrap();
let datastore = JsonFileDatastore::new(temp_file.path()).unwrap();
datastore.put("foo1".into(), vec![1, 2, 3]);
datastore.put("foo2".into(), vec![4, 5, 6]);
let datastore = JsonFileDatastore::<Vec<u8>>::new(temp_file.path()).unwrap();
datastore.put("foo1".into(), vec![6, 7, 8]);
datastore.put("foo2".into(), vec![6, 7, 8]);
datastore.put("foo3".into(), vec![7, 8, 9]);
datastore.put("foo4".into(), vec![10, 11, 12]);
datastore.put("foo5".into(), vec![13, 14, 15]);
datastore.put("bar1".into(), vec![0, 255, 127]);
datastore.flush().unwrap();
let query = datastore
.query(Query {
let query = datastore.query(Query {
prefix: "fo".into(),
filters: vec![
Filter {
ty: FilterTy::ValueCompare(vec![6, 7, 8].into()),
operation: FilterOp::Greater,
ty: FilterTy::ValueCompare(&vec![6, 7, 8].into()),
operation: FilterOp::NotEqual,
},
],
orders: vec![Order::ByKeyDesc],

11
datastore/src/lib.rs
View File

@ -22,6 +22,7 @@ extern crate base64;
#[macro_use]
extern crate futures;
extern crate parking_lot;
extern crate serde;
extern crate serde_json;
extern crate tempfile;
@ -36,13 +37,13 @@ pub use self::json_file::JsonFileDatastore;
pub use self::query::{Query, Order, Filter, FilterTy, FilterOp};
/// Abstraction over any struct that can store `(key, value)` pairs.
pub trait Datastore {
pub trait Datastore<T> {
/// Sets the value of a key.
fn put(&self, key: Cow<str>, value: Vec<u8>);
fn put(&self, key: Cow<str>, value: T);
/// Returns the value corresponding to this key.
// TODO: use higher-kinded stuff once stable to provide a more generic "accessor" for the data
fn get(&self, key: &str) -> Option<Vec<u8>>;
fn get(&self, key: &str) -> Option<T>;
/// Returns true if the datastore contains the given key.
fn has(&self, key: &str) -> bool;
@ -56,6 +57,6 @@ pub trait Datastore {
/// responsibility to pick the right implementation for the right job.
fn query<'a>(
&'a self,
query: Query,
) -> Box<Stream<Item = (String, Vec<u8>), Error = IoError> + 'a>;
query: Query<T>,
) -> Box<Stream<Item = (String, T), Error = IoError> + 'a>;
}

122
datastore/src/query.rs
View File

@ -31,11 +31,11 @@ use std::vec::IntoIter as VecIntoIter;
/// The various modifications of the dataset are applied in the same order as the fields (prefix,
/// filters, orders, skip, limit).
#[derive(Debug, Clone)]
pub struct Query<'a> {
pub struct Query<'a, T: 'a> {
/// Only the keys that start with `prefix` will be returned.
pub prefix: Cow<'a, str>,
/// Filters to apply on the results.
pub filters: Vec<Filter<'a>>,
pub filters: Vec<Filter<'a, T>>,
/// How to order the keys. Applied sequentially.
pub orders: Vec<Order>,
/// Number of elements to skip from at the start of the results.
@ -48,30 +48,29 @@ pub struct Query<'a> {
/// A filter to apply to the results set.
#[derive(Debug, Clone)]
pub struct Filter<'a> {
pub struct Filter<'a, T: 'a> {
/// Type of filter and value to compare with.
pub ty: FilterTy<'a>,
pub ty: FilterTy<'a, T>,
/// Comparison operation.
pub operation: FilterOp,
}
/// Type of filter and value to compare with.
#[derive(Debug, Clone)]
pub enum FilterTy<'a> {
pub enum FilterTy<'a, T: 'a> {
/// Compare the key with a reference value.
KeyCompare(Cow<'a, str>),
/// Compare the value with a reference value.
ValueCompare(Cow<'a, [u8]>),
ValueCompare(&'a T),
}
/// Filtering operation. Keep in mind that anything else than `Equal` and `NotEqual` is a bit
/// blurry.
/// Filtering operation.
#[derive(Debug, Copy, Clone)]
pub enum FilterOp {
Less,
LessOrEqual,
Equal,
NotEqual,
Less,
LessOrEqual,
Greater,
GreaterOrEqual,
}
@ -90,9 +89,10 @@ pub enum Order {
}
/// Naively applies a query on a set of results.
pub fn naive_apply_query<'a, S>(stream: S, query: Query<'a>)
-> StreamTake<StreamSkip<NaiveKeysOnlyApply<NaiveApplyOrdered<NaiveFiltersApply<'a, NaivePrefixApply<'a, S>, VecIntoIter<Filter<'a>>>>>>>
where S: Stream<Item = (String, Vec<u8>), Error = IoError> + 'a
pub fn naive_apply_query<'a, S, V>(stream: S, query: Query<'a, V>)
-> StreamTake<StreamSkip<NaiveKeysOnlyApply<NaiveApplyOrdered<NaiveFiltersApply<'a, NaivePrefixApply<'a, S>, VecIntoIter<Filter<'a, V>>>, V>>>>
where S: Stream<Item = (String, V), Error = IoError> + 'a,
V: Clone + Ord + Default + 'static
{
let prefixed = naive_apply_prefix(stream, query.prefix);
let filtered = naive_apply_filters(prefixed, query.filters.into_iter());
@ -103,18 +103,16 @@ pub fn naive_apply_query<'a, S>(stream: S, query: Query<'a>)
/// Skips the `skip` first element of a stream and only returns `limit` elements.
#[inline]
pub fn naive_apply_skip_limit<S>(stream: S, skip: u64, limit: u64) -> StreamTake<StreamSkip<S>>
where
S: Stream<Item = (String, Vec<u8>), Error = IoError>,
pub fn naive_apply_skip_limit<S, T>(stream: S, skip: u64, limit: u64) -> StreamTake<StreamSkip<S>>
where S: Stream<Item = (String, T), Error = IoError>
{
stream.skip(skip).take(limit)
}
/// Filters the result of a stream to empty values if `keys_only` is true.
#[inline]
pub fn naive_apply_keys_only<S>(stream: S, keys_only: bool) -> NaiveKeysOnlyApply<S>
where
S: Stream<Item = (String, Vec<u8>), Error = IoError>,
pub fn naive_apply_keys_only<S, T>(stream: S, keys_only: bool) -> NaiveKeysOnlyApply<S>
where S: Stream<Item = (String, T), Error = IoError>
{
NaiveKeysOnlyApply {
keys_only: keys_only,
@ -129,18 +127,18 @@ pub struct NaiveKeysOnlyApply<S> {
stream: S,
}
impl<S> Stream for NaiveKeysOnlyApply<S>
where
S: Stream<Item = (String, Vec<u8>), Error = IoError>,
impl<S, T> Stream for NaiveKeysOnlyApply<S>
where S: Stream<Item = (String, T), Error = IoError>,
T: Default
{
type Item = (String, Vec<u8>);
type Item = (String, T);
type Error = IoError;
#[inline]
fn poll(&mut self) -> Poll<Option<Self::Item>, Self::Error> {
if self.keys_only {
Ok(Async::Ready(try_ready!(self.stream.poll()).map(|mut v| {
v.1 = Vec::new();
v.1 = Default::default();
v
})))
} else {
@ -151,14 +149,10 @@ where
/// Filters the result of a stream to only keep the results with a prefix.
#[inline]
pub fn naive_apply_prefix<'a, S>(stream: S, prefix: Cow<'a, str>) -> NaivePrefixApply<'a, S>
where
S: Stream<Item = (String, Vec<u8>), Error = IoError>,
pub fn naive_apply_prefix<'a, S, T>(stream: S, prefix: Cow<'a, str>) -> NaivePrefixApply<'a, S>
where S: Stream<Item = (String, T), Error = IoError>
{
NaivePrefixApply {
prefix: prefix,
stream: stream,
}
NaivePrefixApply { prefix: prefix, stream: stream }
}
/// Returned by `naive_apply_prefix`.
@ -168,11 +162,10 @@ pub struct NaivePrefixApply<'a, S> {
stream: S,
}
impl<'a, S> Stream for NaivePrefixApply<'a, S>
where
S: Stream<Item = (String, Vec<u8>), Error = IoError>,
impl<'a, S, T> Stream for NaivePrefixApply<'a, S>
where S: Stream<Item = (String, T), Error = IoError>
{
type Item = (String, Vec<u8>);
type Item = (String, T);
type Error = IoError;
#[inline]
@ -193,19 +186,18 @@ where
/// Applies orderings on the stream data. Will simply pass data through if the list of orderings
/// is empty. Otherwise will need to collect.
pub fn naive_apply_ordered<'a, S, I>(stream: S, orders_iter: I) -> NaiveApplyOrdered<'a, S>
where
S: Stream<Item = (String, Vec<u8>), Error = IoError> + 'a,
pub fn naive_apply_ordered<'a, S, I, V>(stream: S, orders_iter: I) -> NaiveApplyOrdered<'a, S, V>
where S: Stream<Item = (String, V), Error = IoError> + 'a,
I: IntoIterator<Item = Order>,
I::IntoIter: 'a,
V: Ord + 'static
{
let orders_iter = orders_iter.into_iter();
if orders_iter.size_hint().1 == Some(0) {
return NaiveApplyOrdered { inner: NaiveApplyOrderedInner::PassThrough(stream) };
}
let collected = stream
.collect()
let collected = stream.collect()
.and_then(move |mut collected| {
for order in orders_iter {
match order {
@ -231,20 +223,19 @@ where
}
/// Returned by `naive_apply_ordered`.
pub struct NaiveApplyOrdered<'a, S> {
inner: NaiveApplyOrderedInner<'a, S>,
pub struct NaiveApplyOrdered<'a, S, T> {
inner: NaiveApplyOrderedInner<'a, S, T>,
}
enum NaiveApplyOrderedInner<'a, S> {
enum NaiveApplyOrderedInner<'a, S, T> {
PassThrough(S),
Collected(Box<Stream<Item = (String, Vec<u8>), Error = IoError> + 'a>),
Collected(Box<Stream<Item = (String, T), Error = IoError> + 'a>),
}
impl<'a, S> Stream for NaiveApplyOrdered<'a, S>
where
S: Stream<Item = (String, Vec<u8>), Error = IoError>,
impl<'a, S, V> Stream for NaiveApplyOrdered<'a, S, V>
where S: Stream<Item = (String, V), Error = IoError>
{
type Item = (String, Vec<u8>);
type Item = (String, V);
type Error = IoError;
#[inline]
@ -258,10 +249,10 @@ where
/// Filters the result of a stream to apply a set of filters.
#[inline]
pub fn naive_apply_filters<'a, S, I>(stream: S, filters: I) -> NaiveFiltersApply<'a, S, I>
where
S: Stream<Item = (String, Vec<u8>), Error = IoError>,
I: Iterator<Item = Filter<'a>> + Clone,
pub fn naive_apply_filters<'a, S, I, V>(stream: S, filters: I) -> NaiveFiltersApply<'a, S, I>
where S: Stream<Item = (String, V), Error = IoError>,
I: Iterator<Item = Filter<'a, V>> + Clone,
V: 'a
{
NaiveFiltersApply {
filters: filters,
@ -278,15 +269,12 @@ pub struct NaiveFiltersApply<'a, S, I> {
marker: PhantomData<&'a ()>,
}
impl<'a, S, I> Stream for NaiveFiltersApply<'a, S, I>
where
S: Stream<
Item = (String, Vec<u8>),
Error = IoError,
>,
I: Iterator<Item = Filter<'a>> + Clone,
impl<'a, S, I, T> Stream for NaiveFiltersApply<'a, S, I>
where S: Stream<Item = (String, T), Error = IoError>,
I: Iterator<Item = Filter<'a, T>> + Clone,
T: Ord + 'a
{
type Item = (String, Vec<u8>);
type Item = (String, T);
type Error = IoError;
#[inline]
@ -309,14 +297,16 @@ where
}
#[inline]
fn naive_filter_test(entry: &(String, Vec<u8>), filter: &Filter) -> bool {
fn naive_filter_test<T>(entry: &(String, T), filter: &Filter<T>) -> bool
where T: Ord
{
let (expected_ordering, revert_expected) = match filter.operation {
FilterOp::Equal => (Ordering::Equal, false),
FilterOp::NotEqual => (Ordering::Equal, true),
FilterOp::Less => (Ordering::Less, false),
FilterOp::LessOrEqual => (Ordering::Greater, true),
FilterOp::Equal => (Ordering::Less, false),
FilterOp::NotEqual => (Ordering::Less, true),
FilterOp::Greater => (Ordering::Greater, false),
FilterOp::GreaterOrEqual => (Ordering::Less, true),
FilterOp::Greater => (Ordering::Greater, false),
FilterOp::LessOrEqual => (Ordering::Greater, true),
};
match filter.ty {
@ -324,7 +314,7 @@ fn naive_filter_test(entry: &(String, Vec<u8>), filter: &Filter) -> bool {
((&*entry.0).cmp(&**ref_value) == expected_ordering) != revert_expected
}
FilterTy::ValueCompare(ref ref_value) => {
((&*entry.1).cmp(&**ref_value) == expected_ordering) != revert_expected
(entry.1.cmp(&**ref_value) == expected_ordering) != revert_expected
}
}
}