fluencelabs/rust-libp2p
1
0
Fork 0
mirror of https://github.com/fluencelabs/rust-libp2p synced 2025-06-27 16:51:34 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into weighted_bucket

Browse Source 
# Conflicts:
#	protocols/kad/src/behaviour.rs
#	protocols/kad/src/behaviour/test.rs
This commit is contained in:
folex
2020-03-25 17:52:38 +03:00
parent 5ff311d38d 28ea62d1a9
commit 53b3c34de5
19 changed files with 253 additions and 309 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
protocols/kad/src/behaviour.rs
View File

@ -24,10 +24,10 @@ mod test;
use crate::K_VALUE;
use crate::addresses::{Addresses, Remove};
use crate::handler::{KademliaHandler, KademliaRequestId, KademliaHandlerEvent, KademliaHandlerIn};
use crate::handler::{KademliaHandler, KademliaHandlerConfig, KademliaRequestId, KademliaHandlerEvent, KademliaHandlerIn};
use crate::jobs::*;
use crate::kbucket::{self, KBucketsTable, NodeStatus};
use crate::protocol::{KadConnectionType, KadPeer};
use crate::protocol::{KademliaProtocolConfig, KadConnectionType, KadPeer};
use crate::query::{Query, QueryId, QueryPool, QueryConfig, QueryPoolState};
use crate::record::{self, store::{self, RecordStore}, Record, ProviderRecord};
use crate::contact::Contact;
@ -50,16 +50,13 @@ use wasm_timer::Instant;
use libp2p_core::identity::ed25519::{Keypair, PublicKey};
use trust_graph::TrustGraph;
// TODO: how Kademlia knows hers PeerId? It's stored in KBucketsTable
// TODO: add there hers PublicKey, and exchange it on the network
/// Network behaviour that handles Kademlia.
pub struct Kademlia<TStore> {
/// The Kademlia routing table.
kbuckets: KBucketsTable<kbucket::Key<PeerId>, Contact>,
/// An optional protocol name override to segregate DHTs in the network.
protocol_name_override: Option<Cow<'static, [u8]>>,
/// Configuration of the wire protocol.
protocol_config: KademliaProtocolConfig,
/// The currently active (i.e. in-progress) queries.
queries: QueryPool<QueryInner>,
@ -83,6 +80,9 @@ pub struct Kademlia<TStore> {
/// The TTL of provider records.
provider_record_ttl: Option<Duration>,
/// How long to keep connections alive when they're idle.
connection_idle_timeout: Duration,
/// Queued events to return when the behaviour is being polled.
queued_events: VecDeque<NetworkBehaviourAction<KademliaHandlerIn<QueryId>, KademliaEvent>>,
@ -99,12 +99,13 @@ pub struct Kademlia<TStore> {
pub struct KademliaConfig {
kbucket_pending_timeout: Duration,
query_config: QueryConfig,
protocol_name_override: Option<Cow<'static, [u8]>>,
protocol_config: KademliaProtocolConfig,
record_ttl: Option<Duration>,
record_replication_interval: Option<Duration>,
record_publication_interval: Option<Duration>,
provider_record_ttl: Option<Duration>,
provider_publication_interval: Option<Duration>,
connection_idle_timeout: Duration,
}
impl Default for KademliaConfig {
@ -112,12 +113,13 @@ impl Default for KademliaConfig {
KademliaConfig {
kbucket_pending_timeout: Duration::from_secs(60),
query_config: QueryConfig::default(),
protocol_name_override: None,
protocol_config: Default::default(),
record_ttl: Some(Duration::from_secs(36 * 60 * 60)),
record_replication_interval: Some(Duration::from_secs(60 * 60)),
record_publication_interval: Some(Duration::from_secs(24 * 60 * 60)),
provider_publication_interval: Some(Duration::from_secs(12 * 60 * 60)),
provider_record_ttl: Some(Duration::from_secs(24 * 60 * 60)),
connection_idle_timeout: Duration::from_secs(10),
}
}
}
@ -128,7 +130,7 @@ impl KademliaConfig {
/// Kademlia nodes only communicate with other nodes using the same protocol name. Using a
/// custom name therefore allows to segregate the DHT from others, if that is desired.
pub fn set_protocol_name(&mut self, name: impl Into<Cow<'static, [u8]>>) -> &mut Self {
self.protocol_name_override = Some(name.into());
self.protocol_config.set_protocol_name(name);
self
}
@ -225,6 +227,20 @@ impl KademliaConfig {
self.provider_publication_interval = interval;
self
}
/// Sets the amount of time to keep connections alive when they're idle.
pub fn set_connection_idle_timeout(&mut self, duration: Duration) -> &mut Self {
self.connection_idle_timeout = duration;
self
}
/// Modifies the maximum allowed size of individual Kademlia packets.
///
/// It might be necessary to increase this value if trying to put large records.
pub fn set_max_packet_size(&mut self, size: usize) -> &mut Self {
self.protocol_config.set_max_packet_size(size);
self
}
}
impl<TStore> Kademlia<TStore>
@ -238,9 +254,7 @@ where
/// Get the protocol name of this kademlia instance.
pub fn protocol_name(&self) -> &[u8] {
self.protocol_name_override
.as_ref()
.map_or(crate::protocol::DEFAULT_PROTO_NAME.as_ref(), AsRef::as_ref)
self.protocol_config.protocol_name()
}
/// Creates a new `Kademlia` network behaviour with the given configuration.
@ -264,7 +278,7 @@ where
Kademlia {
store,
kbuckets: KBucketsTable::new(kp, local_key, config.kbucket_pending_timeout),
protocol_name_override: config.protocol_name_override,
protocol_config: config.protocol_config,
queued_events: VecDeque::with_capacity(config.query_config.replication_factor.get()),
queries: QueryPool::new(config.query_config),
connected_peers: Default::default(),
@ -272,7 +286,8 @@ where
put_record_job,
record_ttl: config.record_ttl,
provider_record_ttl: config.provider_record_ttl,
trust
connection_idle_timeout: config.connection_idle_timeout,
trust,
}
}
@ -977,9 +992,7 @@ where
let num_between = self.kbuckets.count_nodes_between(&target);
let k = self.queries.config().replication_factor.get();
let num_beyond_k = (usize::max(k, num_between) - k) as u32;
let expiration = self.record_ttl.map(|ttl|
now + Duration::from_secs(ttl.as_secs() >> num_beyond_k)
);
let expiration = self.record_ttl.map(|ttl| now + exp_decrease(ttl, num_beyond_k));
// The smaller TTL prevails. Only if neither TTL is set is the record
// stored "forever".
record.expires = record.expires.or(expiration).min(expiration);
@ -1002,31 +1015,41 @@ where
// overridden as it avoids having to load the existing record in the
// first place.
// The record is cloned because of the weird libp2p protocol requirement
// to send back the value in the response, although this is a waste of
// resources.
match self.store.put(record.clone()) {
Ok(()) => {
debug!("Record stored: {:?}; {} bytes", record.key, record.value.len());
self.queued_events.push_back(NetworkBehaviourAction::NotifyHandler {
peer_id: source,
handler: NotifyHandler::One(connection),
event: KademliaHandlerIn::PutRecordRes {
key: record.key,
value: record.value,
request_id,
},
})
}
Err(e) => {
info!("Record not stored: {:?}", e);
self.queued_events.push_back(NetworkBehaviourAction::NotifyHandler {
peer_id: source,
handler: NotifyHandler::One(connection),
event: KademliaHandlerIn::Reset(request_id)
})
if !record.is_expired(now) {
// The record is cloned because of the weird libp2p protocol
// requirement to send back the value in the response, although this
// is a waste of resources.
match self.store.put(record.clone()) {
Ok(()) => debug!("Record stored: {:?}; {} bytes", record.key, record.value.len()),
Err(e) => {
info!("Record not stored: {:?}", e);
self.queued_events.push_back(NetworkBehaviourAction::NotifyHandler {
peer_id: source,
handler: NotifyHandler::One(connection),
event: KademliaHandlerIn::Reset(request_id)
});
return
}
}
}
// The remote receives a [`KademliaHandlerIn::PutRecordRes`] even in the
// case where the record is discarded due to being expired. Given that
// the remote sent the local node a [`KademliaHandlerEvent::PutRecord`]
// request, the remote perceives the local node as one node among the k
// closest nodes to the target. In addition returning
// [`KademliaHandlerIn::PutRecordRes`] does not reveal any internal
// information to a possibly malicious remote node.
self.queued_events.push_back(NetworkBehaviourAction::NotifyHandler {
peer_id: source,
handler: NotifyHandler::One(connection),
event: KademliaHandlerIn::PutRecordRes {
key: record.key,
value: record.value,
request_id,
},
})
}
/// Processes a provider record received from a peer.
@ -1053,6 +1076,11 @@ where
}
}
/// Exponentially decrease the given duration (base 2).
fn exp_decrease(ttl: Duration, exp: u32) -> Duration {
Duration::from_secs(ttl.as_secs().checked_shr(exp).unwrap_or(0))
}
impl<TStore> NetworkBehaviour for Kademlia<TStore>
where
for<'a> TStore: RecordStore<'a>,
@ -1062,11 +1090,11 @@ where
type OutEvent = KademliaEvent;
fn new_handler(&mut self) -> Self::ProtocolsHandler {
let mut handler = KademliaHandler::dial_and_listen();
if let Some(name) = self.protocol_name_override.as_ref() {
handler = handler.with_protocol_name(name.clone());
}
handler
KademliaHandler::new(KademliaHandlerConfig {
protocol_config: self.protocol_config.clone(),
allow_listening: true,
idle_timeout: self.connection_idle_timeout,
})
}
fn addresses_of_peer(&mut self, peer_id: &PeerId) -> Vec<Multiaddr> {
@ -1356,7 +1384,7 @@ where
fn poll(&mut self, cx: &mut Context, parameters: &mut impl PollParameters) -> Poll<
NetworkBehaviourAction<
<Self::ProtocolsHandler as ProtocolsHandler>::InEvent,
<KademliaHandler<QueryId> as ProtocolsHandler>::InEvent,
Self::OutEvent,
>,
> {
@ -1967,4 +1995,3 @@ impl QueryInfo {
}
}
}

14
protocols/kad/src/behaviour/test.rs
View File

@ -18,7 +18,6 @@
// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
// DEALINGS IN THE SOFTWARE.
/*
#![cfg(test)]
use super::*;
@ -683,4 +682,15 @@ fn exceed_jobs_max_queries() {
})
)
}
*/
#[test]
fn exp_decr_expiration_overflow() {
fn prop_no_panic(ttl: Duration, factor: u32) {
exp_decrease(ttl, factor);
}
// Right shifting a u64 by >63 results in a panic.
prop_no_panic(KademliaConfig::default().record_ttl.unwrap(), 64);
quickcheck(prop_no_panic as fn(_, _))
}

70
protocols/kad/src/handler.rs
View File

@ -37,7 +37,7 @@ use libp2p_core::{
upgrade::{self, InboundUpgrade, OutboundUpgrade}
};
use log::trace;
use std::{borrow::Cow, error, fmt, io, pin::Pin, task::Context, task::Poll, time::Duration};
use std::{error, fmt, io, pin::Pin, task::Context, task::Poll, time::Duration};
use wasm_timer::Instant;
/// Protocol handler that handles Kademlia communications with the remote.
@ -48,10 +48,7 @@ use wasm_timer::Instant;
/// It also handles requests made by the remote.
pub struct KademliaHandler<TUserData> {
/// Configuration for the Kademlia protocol.
config: KademliaProtocolConfig,
/// If false, we always refuse incoming Kademlia substreams.
allow_listening: bool,
config: KademliaHandlerConfig,
/// Next unique ID of a connection.
next_connec_unique_id: UniqueConnecId,
@ -63,6 +60,19 @@ pub struct KademliaHandler<TUserData> {
keep_alive: KeepAlive,
}
/// Configuration of a [`KademliaHandler`].
#[derive(Debug, Clone)]
pub struct KademliaHandlerConfig {
/// Configuration of the wire protocol.
pub protocol_config: KademliaProtocolConfig,
/// If false, we deny incoming requests.
pub allow_listening: bool,
/// Time after which we close an idle connection.
pub idle_timeout: Duration,
}
/// State of an active substream, opened either by us or by the remote.
enum SubstreamState<TUserData> {
/// We haven't started opening the outgoing substream yet.
@ -369,42 +379,22 @@ pub struct KademliaRequestId {
struct UniqueConnecId(u64);
impl<TUserData> KademliaHandler<TUserData> {
/// Create a `KademliaHandler` that only allows sending messages to the remote but denying
/// incoming connections.
pub fn dial_only() -> Self {
KademliaHandler::with_allow_listening(false)
}
/// Create a [`KademliaHandler`] using the given configuration.
pub fn new(config: KademliaHandlerConfig) -> Self {
let keep_alive = KeepAlive::Until(Instant::now() + config.idle_timeout);
/// Create a `KademliaHandler` that only allows sending messages but also receive incoming
/// requests.
///
/// The `Default` trait implementation wraps around this function.
pub fn dial_and_listen() -> Self {
KademliaHandler::with_allow_listening(true)
}
fn with_allow_listening(allow_listening: bool) -> Self {
KademliaHandler {
config: Default::default(),
allow_listening,
config,
next_connec_unique_id: UniqueConnecId(0),
substreams: Vec::new(),
keep_alive: KeepAlive::Until(Instant::now() + Duration::from_secs(10)),
keep_alive,
}
}
/// Modifies the protocol name used on the wire. Can be used to create incompatibilities
/// between networks on purpose.
pub fn with_protocol_name(mut self, name: impl Into<Cow<'static, [u8]>>) -> Self {
self.config = self.config.with_protocol_name(name);
self
}
}
impl<TUserData> Default for KademliaHandler<TUserData> {
#[inline]
fn default() -> Self {
KademliaHandler::dial_and_listen()
KademliaHandler::new(Default::default())
}
}
@ -422,8 +412,8 @@ where
#[inline]
fn listen_protocol(&self) -> SubstreamProtocol<Self::InboundProtocol> {
if self.allow_listening {
SubstreamProtocol::new(self.config.clone()).map_upgrade(upgrade::EitherUpgrade::A)
if self.config.allow_listening {
SubstreamProtocol::new(self.config.protocol_config.clone()).map_upgrade(upgrade::EitherUpgrade::A)
} else {
SubstreamProtocol::new(upgrade::EitherUpgrade::B(upgrade::DeniedUpgrade))
}
@ -449,7 +439,7 @@ where
EitherOutput::Second(p) => void::unreachable(p),
};
debug_assert!(self.allow_listening);
debug_assert!(self.config.allow_listening);
let connec_unique_id = self.next_connec_unique_id;
self.next_connec_unique_id.0 += 1;
self.substreams
@ -635,7 +625,7 @@ where
let mut substream = self.substreams.swap_remove(n);
loop {
match advance_substream(substream, self.config.clone(), cx) {
match advance_substream(substream, self.config.protocol_config.clone(), cx) {
(Some(new_state), Some(event), _) => {
self.substreams.push(new_state);
return Poll::Ready(event);
@ -672,6 +662,16 @@ where
}
}
impl Default for KademliaHandlerConfig {
fn default() -> Self {
KademliaHandlerConfig {
protocol_config: Default::default(),
allow_listening: true,
idle_timeout: Duration::from_secs(10),
}
}
}
/// Advances one substream.
///
/// Returns the new state for that substream, an event to generate, and whether the substream

22
protocols/kad/src/protocol.rs
View File

@ -153,21 +153,33 @@ impl Into<proto::message::Peer> for KadPeer {
#[derive(Debug, Clone)]
pub struct KademliaProtocolConfig {
protocol_name: Cow<'static, [u8]>,
/// Maximum allowed size of a packet.
max_packet_size: usize,
}
impl KademliaProtocolConfig {
/// Returns the configured protocol name.
pub fn protocol_name(&self) -> &[u8] {
&self.protocol_name
}
/// Modifies the protocol name used on the wire. Can be used to create incompatibilities
/// between networks on purpose.
pub fn with_protocol_name(mut self, name: impl Into<Cow<'static, [u8]>>) -> Self {
pub fn set_protocol_name(&mut self, name: impl Into<Cow<'static, [u8]>>) {
self.protocol_name = name.into();
self
}
/// Modifies the maximum allowed size of a single Kademlia packet.
pub fn set_max_packet_size(&mut self, size: usize) {
self.max_packet_size = size;
}
}
impl Default for KademliaProtocolConfig {
fn default() -> Self {
KademliaProtocolConfig {
protocol_name: Cow::Borrowed(DEFAULT_PROTO_NAME)
protocol_name: Cow::Borrowed(DEFAULT_PROTO_NAME),
max_packet_size: 4096,
}
}
}
@ -191,7 +203,7 @@ where
fn upgrade_inbound(self, incoming: C, _: Self::Info) -> Self::Future {
let mut codec = UviBytes::default();
codec.set_max_len(4096);
codec.set_max_len(self.max_packet_size);
future::ok(
Framed::new(incoming, codec)
@ -223,7 +235,7 @@ where
fn upgrade_outbound(self, incoming: C, _: Self::Info) -> Self::Future {
let mut codec = UviBytes::default();
codec.set_max_len(4096);
codec.set_max_len(self.max_packet_size);
future::ok(
Framed::new(incoming, codec)

2
protocols/kad/src/record/store.rs
View File

@ -20,7 +20,7 @@
mod memory;
pub use memory::MemoryStore;
pub use memory::{MemoryStore, MemoryStoreConfig};
use crate::K_VALUE;
use super::*;

2
protocols/mdns/src/service.rs
View File

@ -127,7 +127,7 @@ impl MdnsService {
Self::new_inner(false)
}
/// Same as `new`, but we don't send automatically send queries on the network.
/// Same as `new`, but we don't automatically send queries on the network.
pub fn silent() -> io::Result<MdnsService> {
Self::new_inner(true)
}