fluencelabs/rust-libp2p
1
0
Fork 0
mirror of https://github.com/fluencelabs/rust-libp2p synced 2025-06-29 09:41:34 +00:00
Code Issues Projects Releases Wiki Activity

protocols/gossipsub: Add Gossipsub v1.1 support

Browse Source 
This commit upgrades the current gossipsub implementation to support the [v1.1
spec](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md).

It adds a number of features, bug fixes and performance improvements. 

Besides support for all new 1.1 features, other improvements that are of particular note: 

- Improved duplicate LRU-time cache (this was previously a severe bottleneck for
  large message throughput topics)
- Extended message validation configuration options
- Arbitrary topics (users can now implement their own hashing schemes)
- Improved message validation handling - Invalid messages are no longer dropped
  but sent to the behaviour for application-level processing (including scoring)
- Support for floodsub, gossipsub v1 and gossipsub v2
- Protobuf encoding has been shifted into the behaviour. This has permitted two
  improvements:
     1. Message size verification during publishing (report to the user if the
        message is too large before attempting to send).
     2. Message fragmentation. If an RPC is too large it is fragmented into its
        sub components and sent in smaller chunks.

Additional Notes

The peer eXchange protocol defined in the v1.1 spec is inactive in its current
form. The current implementation permits sending `PeerId` in `PRUNE` messages,
however a `PeerId` is not sufficient to form a new connection to a peer. A
`Signed Address Record` is required to safely transmit peer identity
information. Once these are confirmed (https://github.com/libp2p/specs/pull/217)
a future PR will implement these and make PX usable.

Co-authored-by: Max Inden <mail@max-inden.de>
Co-authored-by: Rüdiger Klaehn <rklaehn@protonmail.com>
Co-authored-by: blacktemplar <blacktemplar@a1.net>
Co-authored-by: Rüdiger Klaehn <rklaehn@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: Roman S. Borschel <roman@parity.io>
Co-authored-by: Roman Borschel <romanb@users.noreply.github.com>
Co-authored-by: David Craven <david@craven.ch>
This commit is contained in:
Age Manning
2021-01-07 18:19:31 +11:00
committed by GitHub
parent d918e9a79d
commit df7e73ec47
26 changed files with 12071 additions and 1385 deletions
Download Patch File Download Diff File Expand all files Collapse all files

206
protocols/gossipsub/src/mcache.rs
View File

@ -18,26 +18,31 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
use crate::protocol::{GossipsubMessage, MessageId};
use crate::topic::TopicHash;
use crate::types::{MessageId, RawGossipsubMessage};
use libp2p_core::PeerId;
use log::debug;
use std::fmt::Debug;
use std::{collections::HashMap, fmt};
/// CacheEntry stored in the history.
#[derive(Debug, Clone, PartialEq, Eq, Hash)]
pub struct CacheEntry {
mid: MessageId,
topics: Vec<TopicHash>,
topic: TopicHash,
}
/// MessageCache struct holding history of messages.
#[derive(Clone)]
pub struct MessageCache {
msgs: HashMap<MessageId, GossipsubMessage>,
msgs: HashMap<MessageId, RawGossipsubMessage>,
/// For every message and peer the number of times this peer asked for the message
iwant_counts: HashMap<MessageId, HashMap<PeerId, u32>>,
history: Vec<Vec<CacheEntry>>,
/// The number of indices in the cache history used for gossipping. That means that a message
/// won't get gossipped anymore when shift got called `gossip` many times after inserting the
/// message in the cache.
gossip: usize,
msg_id: fn(&GossipsubMessage) -> MessageId,
}
impl fmt::Debug for MessageCache {
@ -52,30 +57,30 @@ impl fmt::Debug for MessageCache {
/// Implementation of the MessageCache.
impl MessageCache {
pub fn new(
gossip: usize,
history_capacity: usize,
msg_id: fn(&GossipsubMessage) -> MessageId,
) -> MessageCache {
pub fn new(gossip: usize, history_capacity: usize) -> Self {
MessageCache {
gossip,
msgs: HashMap::default(),
iwant_counts: HashMap::default(),
history: vec![Vec::new(); history_capacity],
msg_id,
}
}
/// Put a message into the memory cache.
///
/// Returns the message if it already exists.
pub fn put(&mut self, msg: GossipsubMessage) -> Option<GossipsubMessage> {
let message_id = (self.msg_id)(&msg);
pub fn put(
&mut self,
message_id: &MessageId,
msg: RawGossipsubMessage,
) -> Option<RawGossipsubMessage> {
debug!("Put message {:?} in mcache", message_id);
let cache_entry = CacheEntry {
mid: message_id.clone(),
topics: msg.topics.clone(),
topic: msg.topic.clone(),
};
let seen_message = self.msgs.insert(message_id, msg);
let seen_message = self.msgs.insert(message_id.clone(), msg);
if seen_message.is_none() {
// Don't add duplicate entries to the cache.
self.history[0].push(cache_entry);
@ -84,20 +89,46 @@ impl MessageCache {
}
/// Get a message with `message_id`
pub fn get(&self, message_id: &MessageId) -> Option<&GossipsubMessage> {
#[cfg(test)]
pub fn get(&self, message_id: &MessageId) -> Option<&RawGossipsubMessage> {
self.msgs.get(message_id)
}
/// Gets and validates a message with `message_id`.
pub fn validate(&mut self, message_id: &MessageId) -> Option<&GossipsubMessage> {
/// Increases the iwant count for the given message by one and returns the message together
/// with the iwant if the message exists.
pub fn get_with_iwant_counts(
&mut self,
message_id: &MessageId,
peer: &PeerId,
) -> Option<(&RawGossipsubMessage, u32)> {
let iwant_counts = &mut self.iwant_counts;
self.msgs.get(message_id).and_then(|message| {
if !message.validated {
None
} else {
Some((message, {
let count = iwant_counts
.entry(message_id.clone())
.or_default()
.entry(peer.clone())
.or_default();
*count += 1;
*count
}))
}
})
}
/// Gets a message with [`MessageId`] and tags it as validated.
pub fn validate(&mut self, message_id: &MessageId) -> Option<&RawGossipsubMessage> {
self.msgs.get_mut(message_id).map(|message| {
message.validated = true;
&*message
})
}
/// Get a list of GossipIds for a given topic
pub fn get_gossip_ids(&self, topic: &TopicHash) -> Vec<MessageId> {
/// Get a list of [`MessageId`]s for a given topic.
pub fn get_gossip_message_ids(&self, topic: &TopicHash) -> Vec<MessageId> {
self.history[..self.gossip]
.iter()
.fold(vec![], |mut current_entries, entries| {
@ -105,7 +136,7 @@ impl MessageCache {
let mut found_entries: Vec<MessageId> = entries
.iter()
.filter_map(|entry| {
if entry.topics.iter().any(|t| t == topic) {
if &entry.topic == topic {
let mid = &entry.mid;
// Only gossip validated messages
if let Some(true) = self.msgs.get(mid).map(|msg| msg.validated) {
@ -126,50 +157,74 @@ impl MessageCache {
}
/// Shift the history array down one and delete messages associated with the
/// last entry
/// last entry.
pub fn shift(&mut self) {
for entry in self.history.pop().expect("history is always > 1") {
self.msgs.remove(&entry.mid);
if let Some(msg) = self.msgs.remove(&entry.mid) {
if !msg.validated {
// If GossipsubConfig::validate_messages is true, the implementing
// application has to ensure that Gossipsub::validate_message gets called for
// each received message within the cache timeout time."
debug!(
"The message with id {} got removed from the cache without being validated.",
&entry.mid
);
}
}
debug!("Remove message from the cache: {}", &entry.mid);
self.iwant_counts.remove(&entry.mid);
}
// Insert an empty vec in position 0
self.history.insert(0, Vec::new());
}
/// Removes a message from the cache and returns it if existent
pub fn remove(&mut self, message_id: &MessageId) -> Option<RawGossipsubMessage> {
//We only remove the message from msgs and iwant_count and keep the message_id in the
// history vector. Zhe id in the history vector will simply be ignored on popping.
self.iwant_counts.remove(message_id);
self.msgs.remove(message_id)
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::{Topic, TopicHash};
use crate::types::RawGossipsubMessage;
use crate::{IdentTopic as Topic, TopicHash};
use libp2p_core::PeerId;
fn gen_testm(x: u64, topics: Vec<TopicHash>) -> GossipsubMessage {
let u8x: u8 = x as u8;
let source = Some(PeerId::random());
let data: Vec<u8> = vec![u8x];
let sequence_number = Some(x);
let m = GossipsubMessage {
source,
data,
sequence_number,
topics,
signature: None,
key: None,
validated: true,
};
m
}
fn new_cache(gossip_size: usize, history: usize) -> MessageCache {
let default_id = |message: &GossipsubMessage| {
fn gen_testm(x: u64, topic: TopicHash) -> (MessageId, RawGossipsubMessage) {
let default_id = |message: &RawGossipsubMessage| {
// default message id is: source + sequence number
let mut source_string = message.source.as_ref().unwrap().to_base58();
source_string.push_str(&message.sequence_number.unwrap().to_string());
MessageId::from(source_string)
};
let u8x: u8 = x as u8;
let source = Some(PeerId::random());
let data: Vec<u8> = vec![u8x];
let sequence_number = Some(x);
MessageCache::new(gossip_size, history, default_id)
let m = RawGossipsubMessage {
source,
data,
sequence_number,
topic,
signature: None,
key: None,
validated: false,
};
let id = default_id(&m);
(id, m)
}
fn new_cache(gossip_size: usize, history: usize) -> MessageCache {
MessageCache::new(gossip_size, history)
}
#[test]
@ -186,16 +241,14 @@ mod tests {
fn test_put_get_one() {
let mut mc = new_cache(10, 15);
let topic1_hash = Topic::new("topic1".into()).no_hash().clone();
let topic2_hash = Topic::new("topic2".into()).no_hash().clone();
let topic1_hash = Topic::new("topic1").hash().clone();
let (id, m) = gen_testm(10, topic1_hash);
let m = gen_testm(10, vec![topic1_hash, topic2_hash]);
mc.put(m.clone());
mc.put(&id, m.clone());
assert!(mc.history[0].len() == 1);
let fetched = mc.get(&(mc.msg_id)(&m));
let fetched = mc.get(&id);
assert_eq!(fetched.is_none(), false);
assert_eq!(fetched.is_some(), true);
@ -212,12 +265,10 @@ mod tests {
fn test_get_wrong() {
let mut mc = new_cache(10, 15);
let topic1_hash = Topic::new("topic1".into()).no_hash().clone();
let topic2_hash = Topic::new("topic2".into()).no_hash().clone();
let topic1_hash = Topic::new("topic1").hash().clone();
let (id, m) = gen_testm(10, topic1_hash);
let m = gen_testm(10, vec![topic1_hash, topic2_hash]);
mc.put(m.clone());
mc.put(&id, m.clone());
// Try to get an incorrect ID
let wrong_id = MessageId::new(b"wrongid");
@ -236,36 +287,17 @@ mod tests {
assert_eq!(fetched.is_none(), true);
}
#[test]
/// Test adding a message with no topics.
fn test_no_topic_put() {
let mut mc = new_cache(3, 5);
// Build the message
let m = gen_testm(1, vec![]);
mc.put(m.clone());
let fetched = mc.get(&(mc.msg_id)(&m));
// Make sure it is the same fetched message
match fetched {
Some(x) => assert_eq!(*x, m),
_ => assert!(false),
}
}
#[test]
/// Test shift mechanism.
fn test_shift() {
let mut mc = new_cache(1, 5);
let topic1_hash = Topic::new("topic1".into()).no_hash().clone();
let topic2_hash = Topic::new("topic2".into()).no_hash().clone();
let topic1_hash = Topic::new("topic1").hash().clone();
// Build the message
for i in 0..10 {
let m = gen_testm(i, vec![topic1_hash.clone(), topic2_hash.clone()]);
mc.put(m.clone());
let (id, m) = gen_testm(i, topic1_hash.clone());
mc.put(&id, m.clone());
}
mc.shift();
@ -283,12 +315,12 @@ mod tests {
fn test_empty_shift() {
let mut mc = new_cache(1, 5);
let topic1_hash = Topic::new("topic1".into()).no_hash().clone();
let topic2_hash = Topic::new("topic2".into()).no_hash().clone();
let topic1_hash = Topic::new("topic1").hash().clone();
// Build the message
for i in 0..10 {
let m = gen_testm(i, vec![topic1_hash.clone(), topic2_hash.clone()]);
mc.put(m.clone());
let (id, m) = gen_testm(i, topic1_hash.clone());
mc.put(&id, m.clone());
}
mc.shift();
@ -309,12 +341,12 @@ mod tests {
fn test_remove_last_from_shift() {
let mut mc = new_cache(4, 5);
let topic1_hash = Topic::new("topic1".into()).no_hash().clone();
let topic2_hash = Topic::new("topic2".into()).no_hash().clone();
let topic1_hash = Topic::new("topic1").hash().clone();
// Build the message
for i in 0..10 {
let m = gen_testm(i, vec![topic1_hash.clone(), topic2_hash.clone()]);
mc.put(m.clone());
let (id, m) = gen_testm(i, topic1_hash.clone());
mc.put(&id, m.clone());
}
// Shift right until deleting messages