diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e708f594..44715bfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: container: image: rust env: - CC: clang-9 + CC: clang-10 steps: - uses: actions/checkout@v1 - name: Install Rust @@ -50,9 +50,9 @@ jobs: - name: Install a recent version of clang run: | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - - echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list + echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" >> /etc/apt/sources.list apt-get update - apt-get install -y clang-9 + apt-get install -y clang-10 - name: Install CMake run: apt-get install -y cmake - name: Cache cargo registry diff --git a/CHANGELOG.md b/CHANGELOG.md index c49535ac..f2e73fe1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,64 @@ # Version ??? + +# Version 0.19.0 (2020-05-18) + +- `libp2p-core`, `libp2p-swarm`: Added support for multiple dialing + attempts per peer, with a configurable limit. + [PR 1506](https://github.com/libp2p/rust-libp2p/pull/1506) + +- `libp2p-core`: `PeerId`s that use the identity hashing will now be properly + displayed using the string representation of an identity multihash, rather + than the canonical SHA 256 representation. + [PR 1576](https://github.com/libp2p/rust-libp2p/pull/1576) + +- `libp2p-core`: Updated to multihash 0.11.0. + [PR 1566](https://github.com/libp2p/rust-libp2p/pull/1566) + +- `libp2p-core`: Make the number of events buffered to/from tasks configurable. + [PR 1574](https://github.com/libp2p/rust-libp2p/pull/1574) + +- `libp2p-dns`, `parity-multiaddr`: Added support for the `/dns` multiaddr + protocol. Additionally, the `multiaddr::from_url` function will now use + `/dns` instead of `/dns4`. + [PR 1575](https://github.com/libp2p/rust-libp2p/pull/1575) + +- `libp2p-noise`: Added the `X25519Spec` protocol suite which uses + libp2p-noise-spec compliant signatures on static keys as well as the + `/noise` protocol upgrade, hence providing a libp2p-noise-spec compliant + `XX` handshake. `IK` and `IX` are still supported with `X25519Spec` + though not guaranteed to be interoperable with other libp2p + implementations as these handshake patterns are not currently + included in the libp2p-noise-spec. The `X25519Spec` implementation + will eventually replace the current `X25519` implementation, with + the former being removed. To upgrade without interruptions, you may + temporarily include `NoiseConfig`s for both implementations as + alternatives in your transport upgrade pipeline. + - `libp2p-kad`: Consider fixed (K_VALUE) amount of peers at closest query initialization. Unless `KademliaConfig::set_replication_factor` is used change has no effect. [PR 1536](https://github.com/libp2p/rust-libp2p/pull/1536) +- `libp2p-kad`: Provide more insight into, and control of, the execution of + queries. All query results are now wrapped in `KademliaEvent::QueryResult`. + As a side-effect of these changes and for as long as the record storage + API is not asynchronous, local storage errors on `put_record` are reported + synchronously in a `Result`, instead of being reported asynchronously by + an event. + [PR 1567](https://github.com/libp2p/rust-libp2p/pull/1567) + +- `libp2p-tcp`, `libp2p`: Made the `libp2p-tcp/async-std` feature flag + disabled by default, and split the `libp2p/tcp` feature in two: + `tcp-async-std` and `tcp-tokio`. `tcp-async-std` is still enabled by default. + [PR 1471](https://github.com/libp2p/rust-libp2p/pull/1471) + +- `libp2p-tcp`: On listeners started with an IPv6 multi-address the socket + option `IPV6_V6ONLY` is set to true. Instead of relying on IPv4-mapped IPv6 + address support, two listeners can be started if IPv4 and IPv6 should both + be supported. IPv4 listener addresses are not affected by this change. + [PR 1555](https://github.com/libp2p/rust-libp2p/pull/1555) + # Version 0.18.1 (2020-04-17) - `libp2p-swarm`: Make sure inject_dial_failure is called in all situations. diff --git a/Cargo.toml b/Cargo.toml index 3c5618ac..d0eb8507 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p" edition = "2018" description = "Peer-to-peer networking library" -version = "0.18.1" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -25,7 +25,7 @@ default = [ "pnet", "secio", "secp256k1", - "tcp", + "tcp-async-std", "uds", "wasm-ext", "websocket", @@ -44,7 +44,8 @@ ping = ["libp2p-ping"] plaintext = ["libp2p-plaintext"] pnet = ["libp2p-pnet"] secio = ["libp2p-secio"] -tcp = ["libp2p-tcp"] +tcp-async-std = ["libp2p-tcp", "libp2p-tcp/async-std"] +tcp-tokio = ["libp2p-tcp", "libp2p-tcp/tokio"] uds = ["libp2p-uds"] wasm-ext = ["libp2p-wasm-ext"] websocket = ["libp2p-websocket"] @@ -54,36 +55,36 @@ secp256k1 = ["libp2p-core/secp256k1", "libp2p-secio/secp256k1"] [dependencies] bytes = "0.5" futures = "0.3.1" -multiaddr = { package = "parity-multiaddr", version = "0.8.0", path = "misc/multiaddr" } -multihash = "0.10" +multiaddr = { package = "parity-multiaddr", version = "0.9.0", path = "misc/multiaddr" } +multihash = "0.11.0" lazy_static = "1.2" -libp2p-mplex = { version = "0.18.0", path = "muxers/mplex", optional = true } -libp2p-identify = { version = "0.18.0", path = "protocols/identify", optional = true } -libp2p-kad = { version = "0.18.0", path = "protocols/kad", optional = true } -libp2p-floodsub = { version = "0.18.0", path = "protocols/floodsub", optional = true } -libp2p-gossipsub = { version = "0.18.0", path = "./protocols/gossipsub", optional = true } -libp2p-ping = { version = "0.18.0", path = "protocols/ping", optional = true } -libp2p-plaintext = { version = "0.18.0", path = "protocols/plaintext", optional = true } -libp2p-pnet = { version = "0.18.0", path = "protocols/pnet", optional = true } -libp2p-core = { version = "0.18.0", path = "core" } -libp2p-core-derive = { version = "0.18.0", path = "misc/core-derive" } -libp2p-secio = { version = "0.18.0", path = "protocols/secio", default-features = false, optional = true } -libp2p-swarm = { version = "0.18.1", path = "swarm" } -libp2p-uds = { version = "0.18.0", path = "transports/uds", optional = true } -libp2p-wasm-ext = { version = "0.18.0", path = "transports/wasm-ext", optional = true } -libp2p-yamux = { version = "0.18.0", path = "muxers/yamux", optional = true } -libp2p-noise = { version = "0.18.0", path = "protocols/noise", optional = true } +libp2p-mplex = { version = "0.19.0", path = "muxers/mplex", optional = true } +libp2p-identify = { version = "0.19.0", path = "protocols/identify", optional = true } +libp2p-kad = { version = "0.19.0", path = "protocols/kad", optional = true } +libp2p-floodsub = { version = "0.19.0", path = "protocols/floodsub", optional = true } +libp2p-gossipsub = { version = "0.19.0", path = "./protocols/gossipsub", optional = true } +libp2p-ping = { version = "0.19.0", path = "protocols/ping", optional = true } +libp2p-plaintext = { version = "0.19.0", path = "protocols/plaintext", optional = true } +libp2p-pnet = { version = "0.19.0", path = "protocols/pnet", optional = true } +libp2p-core = { version = "0.19.0", path = "core" } +libp2p-core-derive = { version = "0.19.0", path = "misc/core-derive" } +libp2p-secio = { version = "0.19.0", path = "protocols/secio", default-features = false, optional = true } +libp2p-swarm = { version = "0.19.0", path = "swarm" } +libp2p-uds = { version = "0.19.0", path = "transports/uds", optional = true } +libp2p-wasm-ext = { version = "0.19.0", path = "transports/wasm-ext", optional = true } +libp2p-yamux = { version = "0.19.0", path = "muxers/yamux", optional = true } +libp2p-noise = { version = "0.19.0", path = "protocols/noise", optional = true } parking_lot = "0.10.0" pin-project = "0.4.6" smallvec = "1.0" wasm-timer = "0.2.4" [target.'cfg(not(any(target_os = "emscripten", target_os = "unknown")))'.dependencies] -libp2p-deflate = { version = "0.18.0", path = "protocols/deflate", optional = true } -libp2p-dns = { version = "0.18.0", path = "transports/dns", optional = true } -libp2p-mdns = { version = "0.18.0", path = "protocols/mdns", optional = true } -libp2p-tcp = { version = "0.18.0", path = "transports/tcp", optional = true } -libp2p-websocket = { version = "0.18.0", path = "transports/websocket", optional = true } +libp2p-deflate = { version = "0.19.0", path = "protocols/deflate", optional = true } +libp2p-dns = { version = "0.19.0", path = "transports/dns", optional = true } +libp2p-mdns = { version = "0.19.0", path = "protocols/mdns", optional = true } +libp2p-tcp = { version = "0.19.0", path = "transports/tcp", optional = true } +libp2p-websocket = { version = "0.19.0", path = "transports/websocket", optional = true } [dev-dependencies] async-std = "1.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index dd7f0461..c2f8166b 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-core" edition = "2018" description = "Core traits and structs of libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -20,8 +20,8 @@ futures-timer = "3" lazy_static = "1.2" libsecp256k1 = { version = "0.3.1", optional = true } log = "0.4" -multiaddr = { package = "parity-multiaddr", version = "0.8.0", path = "../misc/multiaddr" } -multihash = "0.10" +multiaddr = { package = "parity-multiaddr", version = "0.9.0", path = "../misc/multiaddr" } +multihash = "0.11.0" multistream-select = { version = "0.8.0", path = "../misc/multistream-select" } parking_lot = "0.10.0" pin-project = "0.4.6" @@ -40,9 +40,9 @@ ring = { version = "0.16.9", features = ["alloc", "std"], default-features = fal [dev-dependencies] async-std = "1.0" -libp2p-mplex = { version = "0.18.0", path = "../muxers/mplex" } -libp2p-secio = { version = "0.18.0", path = "../protocols/secio" } -libp2p-tcp = { version = "0.18.0", path = "../transports/tcp" } +libp2p-mplex = { version = "0.19.0", path = "../muxers/mplex" } +libp2p-secio = { version = "0.19.0", path = "../protocols/secio" } +libp2p-tcp = { version = "0.19.0", path = "../transports/tcp" } quickcheck = "0.9.0" wasm-timer = "0.2" diff --git a/core/src/connection/manager.rs b/core/src/connection/manager.rs index c366a071..9dbb644a 100644 --- a/core/src/connection/manager.rs +++ b/core/src/connection/manager.rs @@ -99,6 +99,9 @@ pub struct Manager { /// Next available identifier for a new connection / task. next_task_id: TaskId, + /// Size of the task command buffer (per task). + task_command_buffer_size: usize, + /// The executor to use for running the background tasks. If `None`, /// the tasks are kept in `local_spawns` instead and polled on the /// current thread when the manager is polled for new events. @@ -127,6 +130,32 @@ where } } +/// Configuration options when creating a [`Manager`]. +/// +/// The default configuration specifies no dedicated task executor, a +/// task event buffer size of 32, and a task command buffer size of 7. +#[non_exhaustive] +pub struct ManagerConfig { + /// Executor to use to spawn tasks. + pub executor: Option>, + + /// Size of the task command buffer (per task). + pub task_command_buffer_size: usize, + + /// Size of the task event buffer (for all tasks). + pub task_event_buffer_size: usize, +} + +impl Default for ManagerConfig { + fn default() -> Self { + ManagerConfig { + executor: None, + task_event_buffer_size: 32, + task_command_buffer_size: 7, + } + } +} + /// Internal information about a running task. /// /// Contains the sender to deliver event messages to the task, and @@ -196,12 +225,13 @@ pub enum Event<'a, I, O, H, TE, HE, C> { impl Manager { /// Creates a new connection manager. - pub fn new(executor: Option>) -> Self { - let (tx, rx) = mpsc::channel(1); + pub fn new(config: ManagerConfig) -> Self { + let (tx, rx) = mpsc::channel(config.task_event_buffer_size); Self { tasks: FnvHashMap::default(), next_task_id: TaskId(0), - executor, + task_command_buffer_size: config.task_command_buffer_size, + executor: config.executor, local_spawns: FuturesUnordered::new(), events_tx: tx, events_rx: rx @@ -234,7 +264,7 @@ impl Manager { let task_id = self.next_task_id; self.next_task_id.0 += 1; - let (tx, rx) = mpsc::channel(4); + let (tx, rx) = mpsc::channel(self.task_command_buffer_size); self.tasks.insert(task_id, TaskInfo { sender: tx, state: TaskState::Pending }); let task = Box::pin(Task::pending(task_id, self.events_tx.clone(), rx, future, handler)); @@ -269,7 +299,7 @@ impl Manager { let task_id = self.next_task_id; self.next_task_id.0 += 1; - let (tx, rx) = mpsc::channel(4); + let (tx, rx) = mpsc::channel(self.task_command_buffer_size); self.tasks.insert(task_id, TaskInfo { sender: tx, state: TaskState::Established(info) }); diff --git a/core/src/connection/pool.rs b/core/src/connection/pool.rs index f8c9e1a7..b319ca76 100644 --- a/core/src/connection/pool.rs +++ b/core/src/connection/pool.rs @@ -19,7 +19,6 @@ // DEALINGS IN THE SOFTWARE. use crate::{ - Executor, ConnectedPoint, PeerId, connection::{ @@ -36,7 +35,7 @@ use crate::{ OutgoingInfo, Substream, PendingConnectionError, - manager::{self, Manager}, + manager::{self, Manager, ManagerConfig}, }, muxing::StreamMuxer, }; @@ -175,13 +174,13 @@ where /// Creates a new empty `Pool`. pub fn new( local_id: TPeerId, - executor: Option>, + manager_config: ManagerConfig, limits: PoolLimits ) -> Self { Pool { local_id, limits, - manager: Manager::new(executor), + manager: Manager::new(manager_config), established: Default::default(), pending: Default::default(), } @@ -225,12 +224,7 @@ where TPeerId: Clone + Send + 'static, { let endpoint = info.to_connected_point(); - if let Some(limit) = self.limits.max_incoming { - let current = self.iter_pending_incoming().count(); - if current >= limit { - return Err(ConnectionLimit { limit, current }) - } - } + self.limits.check_incoming(|| self.iter_pending_incoming().count())?; Ok(self.add_pending(future, handler, endpoint, None)) } @@ -267,6 +261,11 @@ where TPeerId: Clone + Send + 'static, { self.limits.check_outgoing(|| self.iter_pending_outgoing().count())?; + + if let Some(peer) = &info.peer_id { + self.limits.check_outgoing_per_peer(|| self.num_peer_outgoing(peer))?; + } + let endpoint = info.to_connected_point(); Ok(self.add_pending(future, handler, endpoint, info.peer_id.cloned())) } @@ -465,6 +464,13 @@ where self.established.get(peer).map_or(0, |conns| conns.len()) } + /// Counts the number of pending outgoing connections to the given peer. + pub fn num_peer_outgoing(&self, peer: &TPeerId) -> usize { + self.iter_pending_outgoing() + .filter(|info| info.peer_id == Some(peer)) + .count() + } + /// Returns an iterator over all established connections of `peer`. pub fn iter_peer_established<'a>(&'a mut self, peer: &TPeerId) -> EstablishedConnectionIter<'a, @@ -837,6 +843,7 @@ pub struct PoolLimits { pub max_outgoing: Option, pub max_incoming: Option, pub max_established_per_peer: Option, + pub max_outgoing_per_peer: Option, } impl PoolLimits { @@ -854,6 +861,20 @@ impl PoolLimits { Self::check(current, self.max_outgoing) } + fn check_incoming(&self, current: F) -> Result<(), ConnectionLimit> + where + F: FnOnce() -> usize + { + Self::check(current, self.max_incoming) + } + + fn check_outgoing_per_peer(&self, current: F) -> Result<(), ConnectionLimit> + where + F: FnOnce() -> usize + { + Self::check(current, self.max_outgoing_per_peer) + } + fn check(current: F, limit: Option) -> Result<(), ConnectionLimit> where F: FnOnce() -> usize diff --git a/core/src/network.rs b/core/src/network.rs index ccb21e84..1e89dcd7 100644 --- a/core/src/network.rs +++ b/core/src/network.rs @@ -43,6 +43,7 @@ use crate::{ ListenersStream, PendingConnectionError, Substream, + manager::ManagerConfig, pool::{Pool, PoolEvent, PoolLimits}, }, muxing::StreamMuxer, @@ -50,12 +51,14 @@ use crate::{ }; use fnv::{FnvHashMap}; use futures::{prelude::*, future}; +use smallvec::SmallVec; use std::{ collections::hash_map, convert::TryFrom as _, error, fmt, hash::Hash, + num::NonZeroUsize, pin::Pin, task::{Context, Poll}, }; @@ -78,21 +81,17 @@ where /// The ongoing dialing attempts. /// - /// The `Network` enforces a single ongoing dialing attempt per peer, - /// even if multiple (established) connections per peer are allowed. - /// However, a single dialing attempt operates on a list of addresses - /// to connect to, which can be extended with new addresses while - /// the connection attempt is still in progress. Thereby each - /// dialing attempt is associated with a new connection and hence a new - /// connection ID. + /// There may be multiple ongoing dialing attempts to the same peer. + /// Each dialing attempt is associated with a new connection and hence + /// a new connection ID. /// /// > **Note**: `dialing` must be consistent with the pending outgoing /// > connections in `pool`. That is, for every entry in `dialing` /// > there must exist a pending outgoing connection in `pool` with /// > the same connection ID. This is ensured by the implementation of /// > `Network` (see `dial_peer_impl` and `on_connection_failed`) - /// > together with the implementation of `DialingConnection::abort`. - dialing: FnvHashMap, + /// > together with the implementation of `DialingAttempt::abort`. + dialing: FnvHashMap>, } impl fmt::Debug for @@ -157,7 +156,7 @@ where Network { local_peer_id, listeners: ListenersStream::new(transport), - pool: Pool::new(pool_local_id, config.executor, config.pool_limits), + pool: Pool::new(pool_local_id, config.manager_config, config.pool_limits), dialing: Default::default(), } } @@ -381,8 +380,11 @@ where Poll::Pending => return Poll::Pending, Poll::Ready(PoolEvent::ConnectionEstablished { connection, num_established }) => { match self.dialing.entry(connection.peer_id().clone()) { - hash_map::Entry::Occupied(e) if e.get().id == connection.id() => { - e.remove(); + hash_map::Entry::Occupied(mut e) => { + e.get_mut().retain(|s| s.current.0 != connection.id()); + if e.get().is_empty() { + e.remove(); + } }, _ => {} } @@ -453,7 +455,7 @@ fn dial_peer_impl::Error, TConnInfo, TPeerId>, - dialing: &mut FnvHashMap, + dialing: &mut FnvHashMap>, opts: DialingOpts ) -> Result where @@ -489,14 +491,12 @@ where }; if let Ok(id) = &result { - let former = dialing.insert(opts.peer, - peer::DialingAttempt { - id: *id, - current: opts.address, - next: opts.remaining, + dialing.entry(opts.peer).or_default().push( + peer::DialingState { + current: (*id, opts.address), + remaining: opts.remaining, }, ); - debug_assert!(former.is_none()); } result @@ -508,7 +508,7 @@ where /// If the failed connection attempt was a dialing attempt and there /// are more addresses to try, new `DialingOpts` are returned. fn on_connection_failed<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId>( - dialing: &mut FnvHashMap, + dialing: &mut FnvHashMap>, id: ConnectionId, endpoint: ConnectedPoint, error: PendingConnectionError, @@ -521,27 +521,34 @@ where TPeerId: Eq + Hash + Clone, { // Check if the failed connection is associated with a dialing attempt. - // TODO: could be more optimal than iterating over everything - let dialing_peer = dialing.iter() // (1) - .find(|(_, a)| a.id == id) - .map(|(p, _)| p.clone()); + let dialing_failed = dialing.iter_mut() + .find_map(|(peer, attempts)| { + if let Some(pos) = attempts.iter().position(|s| s.current.0 == id) { + let attempt = attempts.remove(pos); + let last = attempts.is_empty(); + Some((peer.clone(), attempt, last)) + } else { + None + } + }); - if let Some(peer_id) = dialing_peer { - // A pending outgoing connection to a known peer failed. - let mut attempt = dialing.remove(&peer_id).expect("by (1)"); + if let Some((peer_id, mut attempt, last)) = dialing_failed { + if last { + dialing.remove(&peer_id); + } - let num_remain = u32::try_from(attempt.next.len()).unwrap(); - let failed_addr = attempt.current.clone(); + let num_remain = u32::try_from(attempt.remaining.len()).unwrap(); + let failed_addr = attempt.current.1.clone(); let (opts, attempts_remaining) = if num_remain > 0 { if let Some(handler) = handler { - let next_attempt = attempt.next.remove(0); + let next_attempt = attempt.remaining.remove(0); let opts = DialingOpts { peer: peer_id.clone(), handler, address: next_attempt, - remaining: attempt.next + remaining: attempt.remaining }; (Some(opts), num_remain) } else { @@ -581,25 +588,33 @@ where /// Information about the network obtained by [`Network::info()`]. #[derive(Clone, Debug)] pub struct NetworkInfo { + /// The total number of connected peers. pub num_peers: usize, + /// The total number of connections, both established and pending. pub num_connections: usize, + /// The total number of pending connections, both incoming and outgoing. pub num_connections_pending: usize, + /// The total number of established connections. pub num_connections_established: usize, } /// The (optional) configuration for a [`Network`]. /// -/// The default configuration specifies no dedicated task executor -/// and no connection limits. +/// The default configuration specifies no dedicated task executor, no +/// connection limits, a connection event buffer size of 32, and a +/// `notify_handler` buffer size of 8. #[derive(Default)] pub struct NetworkConfig { - executor: Option>, + /// Note that the `ManagerConfig`s task command buffer always provides + /// one "free" slot per task. Thus the given total `notify_handler_buffer_size` + /// exposed for configuration on the `Network` is reduced by one. + manager_config: ManagerConfig, pool_limits: PoolLimits, } impl NetworkConfig { pub fn set_executor(&mut self, e: Box) -> &mut Self { - self.executor = Some(e); + self.manager_config.executor = Some(e); self } @@ -616,7 +631,30 @@ impl NetworkConfig { } pub fn executor(&self) -> Option<&Box> { - self.executor.as_ref() + self.manager_config.executor.as_ref() + } + + /// Sets the maximum number of events sent to a connection's background task + /// that may be buffered, if the task cannot keep up with their consumption and + /// delivery to the connection handler. + /// + /// When the buffer for a particular connection is full, `notify_handler` will no + /// longer be able to deliver events to the associated `ConnectionHandler`, + /// thus exerting back-pressure on the connection and peer API. + pub fn set_notify_handler_buffer_size(&mut self, n: NonZeroUsize) -> &mut Self { + self.manager_config.task_command_buffer_size = n.get() - 1; + self + } + + /// Sets the maximum number of buffered connection events (beyond a guaranteed + /// buffer of 1 event per connection). + /// + /// When the buffer is full, the background tasks of all connections will stall. + /// In this way, the consumers of network events exert back-pressure on + /// the network connection I/O. + pub fn set_connection_event_buffer_size(&mut self, n: usize) -> &mut Self { + self.manager_config.task_event_buffer_size = n; + self } pub fn set_incoming_limit(&mut self, n: usize) -> &mut Self { @@ -633,4 +671,9 @@ impl NetworkConfig { self.pool_limits.max_established_per_peer = Some(n); self } + + pub fn set_outgoing_per_peer_limit(&mut self, n: usize) -> &mut Self { + self.pool_limits.max_outgoing_per_peer = Some(n); + self + } } diff --git a/core/src/network/peer.rs b/core/src/network/peer.rs index b06be772..8f9dd099 100644 --- a/core/src/network/peer.rs +++ b/core/src/network/peer.rs @@ -35,8 +35,11 @@ use crate::{ IntoConnectionHandler, PendingConnection, Substream, + pool::Pool, }, }; +use fnv::FnvHashMap; +use smallvec::SmallVec; use std::{ collections::hash_map, error, @@ -47,6 +50,10 @@ use super::{Network, DialingOpts}; /// The possible representations of a peer in a [`Network`], as /// seen by the local node. +/// +/// > **Note**: In any state there may always be a pending incoming +/// > connection attempt from the peer, however, the remote identity +/// > of a peer is only known once a connection is fully established. pub enum Peer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> where TTrans: Transport, @@ -63,10 +70,6 @@ where /// There exists no established connection to the peer and there is /// currently no ongoing dialing (i.e. outgoing connection) attempt /// in progress. - /// - /// > **Note**: In this state there may always be a pending incoming - /// > connection attempt from the peer, however, the remote identity - /// > of a peer is only known once a connection is fully established. Disconnected(DisconnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId>), /// The peer represents the local node. @@ -82,20 +85,20 @@ where TPeerId: fmt::Debug + Eq + Hash, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - match *self { - Peer::Connected(ConnectedPeer { ref peer_id, .. }) => { + match self { + Peer::Connected(p) => { f.debug_struct("Connected") - .field("peer_id", peer_id) + .field("peer", &p) .finish() } - Peer::Dialing(DialingPeer { ref peer_id, .. } ) => { - f.debug_struct("DialingPeer") - .field("peer_id", peer_id) + Peer::Dialing(p) => { + f.debug_struct("Dialing") + .field("peer", &p) .finish() } - Peer::Disconnected(DisconnectedPeer { ref peer_id, .. }) => { + Peer::Disconnected(p) => { f.debug_struct("Disconnected") - .field("peer_id", peer_id) + .field("peer", &p) .finish() } Peer::Local => { @@ -164,12 +167,11 @@ where TTrans::Dial: Send + 'static, TMuxer: StreamMuxer + Send + Sync + 'static, TMuxer::OutboundSubstream: Send, - TMuxer::Substream: Send, TInEvent: Send + 'static, TOutEvent: Send + 'static, THandler: IntoConnectionHandler + Send + 'static, - THandler::Handler: ConnectionHandler, InEvent = TInEvent, OutEvent = TOutEvent> + Send + 'static, - ::OutboundOpenInfo: Send + 'static, // TODO: shouldn't be necessary + THandler::Handler: ConnectionHandler, InEvent = TInEvent, OutEvent = TOutEvent> + Send, + ::OutboundOpenInfo: Send, ::Error: error::Error + Send + 'static, TConnInfo: fmt::Debug + ConnectionInfo + Send + 'static, TPeerId: Eq + Hash + Clone + Send + 'static, @@ -208,7 +210,41 @@ where } } - /// Converts the peer into a `ConnectedPeer`, if there an established connection exists. + /// Initiates a new dialing attempt to this peer using the given addresses. + /// + /// The connection ID of the first connection attempt, i.e. to `address`, + /// is returned, together with a [`DialingPeer`] for further use. The + /// `remaining` addresses are tried in order in subsequent connection + /// attempts in the context of the same dialing attempt, if the connection + /// attempt to the first address fails. + pub fn dial(self, address: Multiaddr, remaining: I, handler: THandler) + -> Result< + (ConnectionId, DialingPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId>), + ConnectionLimit + > + where + I: IntoIterator, + { + let (peer_id, network) = match self { + Peer::Connected(p) => (p.peer_id, p.network), + Peer::Dialing(p) => (p.peer_id, p.network), + Peer::Disconnected(p) => (p.peer_id, p.network), + Peer::Local => return Err(ConnectionLimit { current: 0, limit: 0 }) + }; + + let id = network.dial_peer(DialingOpts { + peer: peer_id.clone(), + handler, + address, + remaining: remaining.into_iter().collect(), + })?; + + Ok((id, DialingPeer { network, peer_id })) + } + + /// Converts the peer into a `ConnectedPeer`, if an established connection exists. + /// + /// Succeeds if the there is at least one established connection to the peer. pub fn into_connected(self) -> Option< ConnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> > { @@ -221,6 +257,8 @@ where } /// Converts the peer into a `DialingPeer`, if a dialing attempt exists. + /// + /// Succeeds if the there is at least one pending outgoing connection to the peer. pub fn into_dialing(self) -> Option< DialingPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> > { @@ -245,7 +283,8 @@ where } /// The representation of a peer in a [`Network`] to whom at least -/// one established connection exists. +/// one established connection exists. There may also be additional ongoing +/// dialing attempts to the peer. pub struct ConnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> where TTrans: Transport, @@ -267,57 +306,12 @@ where &self.peer_id } - /// Attempts to establish a new connection to this peer using the given addresses, - /// if there is currently no ongoing dialing attempt. - /// - /// Existing established connections are not affected. - /// - /// > **Note**: If there is an ongoing dialing attempt, a `DialingPeer` - /// > is returned with the given addresses and handler being ignored. - /// > You may want to check [`ConnectedPeer::is_dialing`] first. - pub fn connect(self, address: Multiaddr, remaining: I, handler: THandler) - -> Result, - ConnectionLimit> - where - I: IntoIterator, - THandler: Send + 'static, - THandler::Handler: Send, - ::Error: error::Error + Send, - ::OutboundOpenInfo: Send, - THandler::Handler: ConnectionHandler, InEvent = TInEvent, OutEvent = TOutEvent> + Send, - TTrans: Transport + Clone, - TTrans::Error: Send + 'static, - TTrans::Dial: Send + 'static, - TMuxer: StreamMuxer + Send + Sync + 'static, - TMuxer::OutboundSubstream: Send, - TMuxer::Substream: Send, - TConnInfo: fmt::Debug + Send + 'static, - TPeerId: Eq + Hash + Clone + Send + 'static, - TInEvent: Send + 'static, - TOutEvent: Send + 'static, - - { - if self.network.dialing.contains_key(&self.peer_id) { - let peer = DialingPeer { - network: self.network, - peer_id: self.peer_id - }; - Ok(peer) - } else { - self.network.dial_peer(DialingOpts { - peer: self.peer_id.clone(), - handler, - address, - remaining: remaining.into_iter().collect(), - })?; - Ok(DialingPeer { - network: self.network, - peer_id: self.peer_id, - }) - } + /// Returns the `ConnectedPeer` into a `Peer`. + pub fn into_peer(self) -> Peer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> { + Peer::Connected(self) } - /// Obtains an existing connection to the peer. + /// Obtains an established connection to the peer by ID. pub fn connection<'b>(&'b mut self, id: ConnectionId) -> Option> { @@ -348,7 +342,7 @@ where } } - /// Gets an iterator over all established connections of the peer. + /// Gets an iterator over all established connections to the peer. pub fn connections<'b>(&'b mut self) -> EstablishedConnectionIter<'b, impl Iterator, @@ -386,11 +380,13 @@ impl<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> fmt::Debug f where TTrans: Transport, THandler: IntoConnectionHandler, - TPeerId: fmt::Debug, + TPeerId: Eq + Hash + fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { f.debug_struct("ConnectedPeer") .field("peer_id", &self.peer_id) + .field("established", &self.network.pool.iter_peer_established_info(&self.peer_id)) + .field("attempts", &self.network.dialing.get(&self.peer_id)) .finish() } } @@ -419,8 +415,16 @@ where &self.peer_id } - /// Disconnects from this peer, closing all pending connections. - pub fn disconnect(self) -> DisconnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> { + /// Returns the `DialingPeer` into a `Peer`. + pub fn into_peer(self) -> Peer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> { + Peer::Dialing(self) + } + + /// Disconnects from this peer, closing all established connections and + /// aborting all dialing attempts. + pub fn disconnect(self) + -> DisconnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> + { self.network.disconnect(&self.peer_id); DisconnectedPeer { network: self.network, peer_id: self.peer_id } } @@ -443,20 +447,50 @@ where } } - /// Obtains the connection that is currently being established. - pub fn connection<'b>(&'b mut self) -> DialingConnection<'b, TInEvent, TConnInfo, TPeerId> { - let attempt = match self.network.dialing.entry(self.peer_id.clone()) { - hash_map::Entry::Occupied(e) => e, - _ => unreachable!("By `Peer::new` and the definition of `DialingPeer`.") - }; - - let inner = self.network.pool - .get_outgoing(attempt.get().id) - .expect("By consistency of `network.pool` with `network.dialing`."); - - DialingConnection { - inner, dialing: attempt, peer_id: &self.peer_id + /// Obtains a dialing attempt to the peer by connection ID of + /// the current connection attempt. + pub fn attempt<'b>(&'b mut self, id: ConnectionId) + -> Option> + { + if let hash_map::Entry::Occupied(attempts) = self.network.dialing.entry(self.peer_id.clone()) { + if let Some(pos) = attempts.get().iter().position(|s| s.current.0 == id) { + if let Some(inner) = self.network.pool.get_outgoing(id) { + return Some(DialingAttempt { pos, inner, attempts }) + } + } } + None + } + + /// The number of ongoing dialing attempts, i.e. pending outgoing connections + /// to this peer. + pub fn num_attempts(&self) -> usize { + self.network.pool.num_peer_outgoing(&self.peer_id) + } + + /// Gets an iterator over all dialing (i.e. pending outgoing) connections to the peer. + pub fn attempts<'b>(&'b mut self) + -> DialingAttemptIter<'b, + TInEvent, + TOutEvent, + THandler, + TTrans::Error, + ::Error, + TConnInfo, + TPeerId> + { + DialingAttemptIter::new(&self.peer_id, &mut self.network.pool, &mut self.network.dialing) + } + + /// Obtains some dialing connection to the peer. + /// + /// At least one dialing connection is guaranteed to exist on a `DialingPeer`. + pub fn some_attempt<'b>(&'b mut self) + -> DialingAttempt<'b, TInEvent, TConnInfo, TPeerId> + { + self.attempts() + .into_first() + .expect("By `Peer::new` and the definition of `DialingPeer`.") } } @@ -465,11 +499,13 @@ impl<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> fmt::Debug f where TTrans: Transport, THandler: IntoConnectionHandler, - TPeerId: fmt::Debug, + TPeerId: Eq + Hash + fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { f.debug_struct("DialingPeer") .field("peer_id", &self.peer_id) + .field("established", &self.network.pool.iter_peer_established_info(&self.peer_id)) + .field("attempts", &self.network.dialing.get(&self.peer_id)) .finish() } } @@ -500,46 +536,19 @@ where } } -impl<'a, TTrans, TInEvent, TOutEvent, TMuxer, THandler, TConnInfo, TPeerId> +impl<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> DisconnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> where - TTrans: Transport + Clone, - TTrans::Error: Send + 'static, - TTrans::Dial: Send + 'static, - TMuxer: StreamMuxer + Send + Sync + 'static, - TMuxer::OutboundSubstream: Send, - TMuxer::Substream: Send, - THandler: IntoConnectionHandler + Send + 'static, - THandler::Handler: ConnectionHandler, InEvent = TInEvent, OutEvent = TOutEvent> + Send, - ::OutboundOpenInfo: Send, - ::Error: error::Error + Send, - TInEvent: Send + 'static, - TOutEvent: Send + 'static, + TTrans: Transport, + THandler: IntoConnectionHandler, { pub fn id(&self) -> &TPeerId { &self.peer_id } - /// Attempts to connect to this peer using the given addresses. - pub fn connect(self, first: Multiaddr, rest: TIter, handler: THandler) - -> Result, - ConnectionLimit> - where - TIter: IntoIterator, - TConnInfo: fmt::Debug + ConnectionInfo + Send + 'static, - TPeerId: Eq + Hash + Clone + Send + 'static, - { - self.network.dial_peer(DialingOpts { - peer: self.peer_id.clone(), - handler, - address: first, - remaining: rest.into_iter().collect(), - })?; - Ok(DialingPeer { - network: self.network, - peer_id: self.peer_id, - }) - + /// Returns the `DisconnectedPeer` into a `Peer`. + pub fn into_peer(self) -> Peer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> { + Peer::Disconnected(self) } /// Moves the peer into a connected state by supplying an existing @@ -550,8 +559,7 @@ where /// # Panics /// /// Panics if `connected.peer_id()` does not identify the current peer. - /// - pub fn set_connected( + pub fn set_connected( self, connected: Connected, connection: Connection, @@ -559,8 +567,17 @@ where ConnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId>, ConnectionLimit > where + TInEvent: Send + 'static, + TOutEvent: Send + 'static, + THandler: Send + 'static, + TTrans::Error: Send + 'static, + THandler::Handler: ConnectionHandler, InEvent = TInEvent, OutEvent = TOutEvent> + Send, + ::OutboundOpenInfo: Send, + ::Error: error::Error + Send + 'static, TConnInfo: fmt::Debug + ConnectionInfo + Clone + Send + 'static, - TPeerId: Eq + Hash + Clone + fmt::Debug, + TPeerId: Eq + Hash + Clone + Send + fmt::Debug + 'static, + TMuxer: StreamMuxer + Send + Sync + 'static, + TMuxer::OutboundSubstream: Send, { if connected.peer_id() != &self.peer_id { panic!("Invalid peer ID given: {:?}. Expected: {:?}", connected.peer_id(), self.peer_id) @@ -574,71 +591,142 @@ where } } -/// Attempt to reach a peer. +/// The (internal) state of a `DialingAttempt`, tracking the +/// current connection attempt as well as remaining addresses. #[derive(Debug, Clone)] -pub(super) struct DialingAttempt { - /// Identifier for the reach attempt. - pub(super) id: ConnectionId, - /// Multiaddr currently being attempted. - pub(super) current: Multiaddr, +pub(super) struct DialingState { + /// The ID and (remote) address of the current connection attempt. + pub(super) current: (ConnectionId, Multiaddr), /// Multiaddresses to attempt if the current one fails. - pub(super) next: Vec, + pub(super) remaining: Vec, } -/// A `DialingConnection` is a [`PendingConnection`] where the local peer -/// has the role of the dialer (i.e. initiator) and the (expected) remote -/// peer ID is known. -pub struct DialingConnection<'a, TInEvent, TConnInfo, TPeerId> { - peer_id: &'a TPeerId, +/// A `DialingAttempt` is an ongoing outgoing connection attempt to +/// a known / expected remote peer ID and a list of alternative addresses +/// to connect to, if the current connection attempt fails. +pub struct DialingAttempt<'a, TInEvent, TConnInfo, TPeerId> { + /// The underlying pending connection in the `Pool`. inner: PendingConnection<'a, TInEvent, TConnInfo, TPeerId>, - dialing: hash_map::OccupiedEntry<'a, TPeerId, DialingAttempt>, + /// All current dialing attempts of the peer. + attempts: hash_map::OccupiedEntry<'a, TPeerId, SmallVec<[DialingState; 10]>>, + /// The position of the current `DialingState` of this connection in the `attempts`. + pos: usize, } impl<'a, TInEvent, TConnInfo, TPeerId> - DialingConnection<'a, TInEvent, TConnInfo, TPeerId> + DialingAttempt<'a, TInEvent, TConnInfo, TPeerId> { - /// Returns the local connection ID. + /// Returns the ID of the current connection attempt. pub fn id(&self) -> ConnectionId { self.inner.id() } - /// Returns the (expected) peer ID of the ongoing connection attempt. + /// Returns the (expected) peer ID of the dialing attempt. pub fn peer_id(&self) -> &TPeerId { - self.peer_id + self.attempts.key() } - /// Returns information about this endpoint of the connection attempt. - pub fn endpoint(&self) -> &ConnectedPoint { - self.inner.endpoint() - } - - /// Aborts the connection attempt. - pub fn abort(self) - where - TPeerId: Eq + Hash + Clone, - { - self.dialing.remove(); - self.inner.abort(); - } - - /// Adds new candidate addresses to the end of the addresses used - /// in the ongoing dialing process. - /// - /// Duplicates are ignored. - pub fn add_addresses(&mut self, addrs: impl IntoIterator) { - for addr in addrs { - self.add_address(addr); + /// Returns the remote address of the current connection attempt. + pub fn address(&self) -> &Multiaddr { + match self.inner.endpoint() { + ConnectedPoint::Dialer { address } => address, + ConnectedPoint::Listener { .. } => unreachable!("by definition of a `DialingAttempt`.") } } - /// Adds an address to the end of the addresses used in the ongoing - /// dialing process. + /// Aborts the dialing attempt. /// - /// Duplicates are ignored. + /// Aborting a dialing attempt involves aborting the current connection + /// attempt and dropping any remaining addresses given to [`Peer::dial()`] + /// that have not yet been tried. + pub fn abort(mut self) { + self.attempts.get_mut().remove(self.pos); + if self.attempts.get().is_empty() { + self.attempts.remove(); + } + self.inner.abort(); + } + + /// Adds an address to the end of the remaining addresses + /// for this dialing attempt. Duplicates are ignored. pub fn add_address(&mut self, addr: Multiaddr) { - if self.dialing.get().next.iter().all(|a| a != &addr) { - self.dialing.get_mut().next.push(addr); + let remaining = &mut self.attempts.get_mut()[self.pos].remaining; + if remaining.iter().all(|a| a != &addr) { + remaining.push(addr); } } } +/// An iterator over the ongoing dialing attempts to a peer. +pub struct DialingAttemptIter<'a, TInEvent, TOutEvent, THandler, TTransErr, THandlerErr, TConnInfo, TPeerId> { + /// The peer whose dialing attempts are being iterated. + peer_id: &'a TPeerId, + /// The underlying connection `Pool` of the `Network`. + pool: &'a mut Pool, + /// The state of all current dialing attempts known to the `Network`. + /// + /// Ownership of the `OccupiedEntry` for `peer_id` containing all attempts must be + /// borrowed to each `DialingAttempt` in order for it to remove the entry if the + /// last dialing attempt is aborted. + dialing: &'a mut FnvHashMap>, + /// The current position of the iterator in `dialing[peer_id]`. + pos: usize, + /// The total number of elements in `dialing[peer_id]` to iterate over. + end: usize, +} + +// Note: Ideally this would be an implementation of `Iterator`, but that +// requires GATs (cf. https://github.com/rust-lang/rust/issues/44265) and +// a different definition of `Iterator`. +impl<'a, TInEvent, TOutEvent, THandler, TTransErr, THandlerErr, TConnInfo, TPeerId> + DialingAttemptIter<'a, TInEvent, TOutEvent, THandler, TTransErr, THandlerErr, TConnInfo, TPeerId> +where + TConnInfo: ConnectionInfo, + TPeerId: Eq + Hash + Clone, +{ + fn new( + peer_id: &'a TPeerId, + pool: &'a mut Pool, + dialing: &'a mut FnvHashMap>, + ) -> Self { + let end = dialing.get(peer_id).map_or(0, |conns| conns.len()); + Self { pos: 0, end, pool, dialing, peer_id } + } + + /// Obtains the next dialing connection, if any. + pub fn next<'b>(&'b mut self) -> Option> { + if self.pos == self.end { + return None + } + + if let hash_map::Entry::Occupied(attempts) = self.dialing.entry(self.peer_id.clone()) { + let id = attempts.get()[self.pos].current.0; + if let Some(inner) = self.pool.get_outgoing(id) { + let conn = DialingAttempt { pos: self.pos, inner, attempts }; + self.pos += 1; + return Some(conn) + } + } + + None + } + + /// Returns the first connection, if any, consuming the iterator. + pub fn into_first<'b>(self) + -> Option> + where 'a: 'b + { + if self.pos == self.end { + return None + } + + if let hash_map::Entry::Occupied(attempts) = self.dialing.entry(self.peer_id.clone()) { + let id = attempts.get()[self.pos].current.0; + if let Some(inner) = self.pool.get_outgoing(id) { + return Some(DialingAttempt { pos: self.pos, inner, attempts }) + } + } + + None + } +} diff --git a/core/src/peer_id.rs b/core/src/peer_id.rs index 9ad57086..827a0e97 100644 --- a/core/src/peer_id.rs +++ b/core/src/peer_id.rs @@ -77,10 +77,9 @@ impl PeerId { }; let canonical = canonical_algorithm.map(|alg| - alg.hasher().expect("SHA2-256 hasher is always supported").digest(&key_enc)); + alg.digest(&key_enc)); - let multihash = hash_algorithm.hasher() - .expect("Identity and SHA-256 hasher are always supported").digest(&key_enc); + let multihash = hash_algorithm.digest(&key_enc); PeerId { multihash, canonical } } @@ -148,7 +147,7 @@ impl PeerId { /// Returns a base-58 encoded string of this `PeerId`. pub fn to_base58(&self) -> String { - bs58::encode(self.borrow() as &[u8]).into_string() + bs58::encode(self.as_bytes()).into_string() } /// Checks whether the public key passed as parameter matches the public key of this `PeerId`. @@ -158,7 +157,7 @@ impl PeerId { pub fn is_public_key(&self, public_key: &PublicKey) -> Option { let alg = self.multihash.algorithm(); let enc = public_key.clone().into_protobuf_encoding(); - Some(alg.hasher()?.digest(&enc) == self.multihash) + Some(alg.digest(&enc) == self.multihash) } /// Returns public key if it was inlined in this `PeerId`. @@ -336,8 +335,8 @@ mod tests { } fn property(data: Vec, algo1: HashAlgo, algo2: HashAlgo) -> bool { - let a = PeerId::try_from(algo1.0.hasher().unwrap().digest(&data)).unwrap(); - let b = PeerId::try_from(algo2.0.hasher().unwrap().digest(&data)).unwrap(); + let a = PeerId::try_from(algo1.0.digest(&data)).unwrap(); + let b = PeerId::try_from(algo2.0.digest(&data)).unwrap(); if algo1 == algo2 || algo1.0 == Code::Identity || algo2.0 == Code::Identity { a == b diff --git a/core/src/translation.rs b/core/src/translation.rs index 5dab82a0..70700ca6 100644 --- a/core/src/translation.rs +++ b/core/src/translation.rs @@ -37,9 +37,14 @@ use multiaddr::{Multiaddr, Protocol}; /// If the first [`Protocol`]s are not IP addresses, `None` is returned instead. pub fn address_translation(original: &Multiaddr, observed: &Multiaddr) -> Option { original.replace(0, move |proto| match proto { - Protocol::Ip4(_) | Protocol::Ip6(_) | Protocol::Dns4(_) | Protocol::Dns6(_) => match observed.iter().next() { + Protocol::Ip4(_) + | Protocol::Ip6(_) + | Protocol::Dns(_) + | Protocol::Dns4(_) + | Protocol::Dns6(_) => match observed.iter().next() { x @ Some(Protocol::Ip4(_)) => x, x @ Some(Protocol::Ip6(_)) => x, + x @ Some(Protocol::Dns(_)) => x, x @ Some(Protocol::Dns4(_)) => x, x @ Some(Protocol::Dns6(_)) => x, _ => None, diff --git a/core/tests/network_dial_error.rs b/core/tests/network_dial_error.rs index c01cebdd..630eccc0 100644 --- a/core/tests/network_dial_error.rs +++ b/core/tests/network_dial_error.rs @@ -22,47 +22,60 @@ mod util; use futures::prelude::*; use libp2p_core::identity; -use libp2p_core::multiaddr::multiaddr; +use libp2p_core::multiaddr::{multiaddr, Multiaddr}; use libp2p_core::{ Network, PeerId, Transport, connection::PendingConnectionError, muxing::StreamMuxerBox, - network::NetworkEvent, + network::{NetworkEvent, NetworkConfig}, + transport, upgrade, }; +use rand::Rng; use rand::seq::SliceRandom; -use std::{io, task::Poll}; +use std::{io, error::Error, fmt, task::Poll}; use util::TestHandler; -type TestNetwork = Network; +type TestNetwork = Network; +type TestTransport = transport::boxed::Boxed<(PeerId, StreamMuxerBox), BoxError>; + +#[derive(Debug)] +struct BoxError(Box); + +impl Error for BoxError {} + +impl fmt::Display for BoxError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Transport error: {}", self.0) + } +} + +fn new_network(cfg: NetworkConfig) -> TestNetwork { + let local_key = identity::Keypair::generate_ed25519(); + let local_public_key = local_key.public(); + let transport: TestTransport = libp2p_tcp::TcpConfig::new() + .upgrade(upgrade::Version::V1) + .authenticate(libp2p_secio::SecioConfig::new(local_key)) + .multiplex(libp2p_mplex::MplexConfig::new()) + .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))) + .and_then(|(peer, mplex), _| { + // Gracefully close the connection to allow protocol + // negotiation to complete. + util::CloseMuxer::new(mplex).map_ok(move |mplex| (peer, mplex)) + }) + .map_err(|e| BoxError(Box::new(e))) + .boxed(); + TestNetwork::new(transport, local_public_key.into(), cfg) +} #[test] fn deny_incoming_connec() { // Checks whether refusing an incoming connection on a swarm triggers the correct events. - let mut swarm1 = { - let local_key = identity::Keypair::generate_ed25519(); - let local_public_key = local_key.public(); - let transport = libp2p_tcp::TcpConfig::new() - .upgrade(upgrade::Version::V1) - .authenticate(libp2p_secio::SecioConfig::new(local_key)) - .multiplex(libp2p_mplex::MplexConfig::new()) - .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))); - TestNetwork::new(transport, local_public_key.into(), Default::default()) - }; - - let mut swarm2 = { - let local_key = identity::Keypair::generate_ed25519(); - let local_public_key = local_key.public(); - let transport = libp2p_tcp::TcpConfig::new() - .upgrade(upgrade::Version::V1) - .authenticate(libp2p_secio::SecioConfig::new(local_key)) - .multiplex(libp2p_mplex::MplexConfig::new()) - .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))); - TestNetwork::new(transport, local_public_key.into(), Default::default()) - }; + let mut swarm1 = new_network(NetworkConfig::default()); + let mut swarm2 = new_network(NetworkConfig::default()); swarm1.listen_on("/ip4/127.0.0.1/tcp/0".parse().unwrap()).unwrap(); @@ -76,8 +89,7 @@ fn deny_incoming_connec() { swarm2 .peer(swarm1.local_peer_id().clone()) - .into_disconnected().unwrap() - .connect(address.clone(), Vec::new(), TestHandler()) + .dial(address.clone(), Vec::new(), TestHandler()) .unwrap(); async_std::task::block_on(future::poll_fn(|cx| -> Poll> { @@ -119,22 +131,7 @@ fn dial_self() { // // The last two can happen in any order. - let mut swarm = { - let local_key = identity::Keypair::generate_ed25519(); - let local_public_key = local_key.public(); - let transport = libp2p_tcp::TcpConfig::new() - .upgrade(upgrade::Version::V1) - .authenticate(libp2p_secio::SecioConfig::new(local_key)) - .multiplex(libp2p_mplex::MplexConfig::new()) - .and_then(|(peer, mplex), _| { - // Gracefully close the connection to allow protocol - // negotiation to complete. - util::CloseMuxer::new(mplex).map_ok(move |mplex| (peer, mplex)) - }) - .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))); - TestNetwork::new(transport, local_public_key.into(), Default::default()) - }; - + let mut swarm = new_network(NetworkConfig::default()); swarm.listen_on("/ip4/127.0.0.1/tcp/0".parse().unwrap()).unwrap(); let (local_address, mut swarm) = async_std::task::block_on( @@ -193,36 +190,16 @@ fn dial_self() { fn dial_self_by_id() { // Trying to dial self by passing the same `PeerId` shouldn't even be possible in the first // place. - - let mut swarm = { - let local_key = identity::Keypair::generate_ed25519(); - let local_public_key = local_key.public(); - let transport = libp2p_tcp::TcpConfig::new() - .upgrade(upgrade::Version::V1) - .authenticate(libp2p_secio::SecioConfig::new(local_key)) - .multiplex(libp2p_mplex::MplexConfig::new()) - .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))); - TestNetwork::new(transport, local_public_key.into(), Default::default()) - }; - + let mut swarm = new_network(NetworkConfig::default()); let peer_id = swarm.local_peer_id().clone(); assert!(swarm.peer(peer_id).into_disconnected().is_none()); } #[test] fn multiple_addresses_err() { - // Tries dialing multiple addresses, and makes sure there's one dialing error per addresses. + // Tries dialing multiple addresses, and makes sure there's one dialing error per address. - let mut swarm = { - let local_key = identity::Keypair::generate_ed25519(); - let local_public_key = local_key.public(); - let transport = libp2p_tcp::TcpConfig::new() - .upgrade(upgrade::Version::V1) - .authenticate(libp2p_secio::SecioConfig::new(local_key)) - .multiplex(libp2p_mplex::MplexConfig::new()) - .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))); - TestNetwork::new(transport, local_public_key.into(), Default::default()) - }; + let mut swarm = new_network(NetworkConfig::default()); let mut addresses = Vec::new(); for _ in 0 .. 3 { @@ -238,8 +215,7 @@ fn multiple_addresses_err() { let target = PeerId::random(); swarm.peer(target.clone()) - .into_disconnected().unwrap() - .connect(first, rest, TestHandler()) + .dial(first, rest, TestHandler()) .unwrap(); async_std::task::block_on(future::poll_fn(|cx| -> Poll> { @@ -267,3 +243,44 @@ fn multiple_addresses_err() { } })).unwrap(); } + +#[test] +fn connection_limit() { + let outgoing_per_peer_limit = rand::thread_rng().gen_range(1, 10); + let outgoing_limit = 2 * outgoing_per_peer_limit; + + let mut cfg = NetworkConfig::default(); + cfg.set_outgoing_per_peer_limit(outgoing_per_peer_limit); + cfg.set_outgoing_limit(outgoing_limit); + let mut network = new_network(cfg); + + let target = PeerId::random(); + for _ in 0 .. outgoing_per_peer_limit { + network.peer(target.clone()) + .dial(Multiaddr::empty(), Vec::new(), TestHandler()) + .ok() + .expect("Unexpected connection limit."); + } + + let err = network.peer(target) + .dial(Multiaddr::empty(), Vec::new(), TestHandler()) + .expect_err("Unexpected dialing success."); + + assert_eq!(err.current, outgoing_per_peer_limit); + assert_eq!(err.limit, outgoing_per_peer_limit); + + let target2 = PeerId::random(); + for _ in outgoing_per_peer_limit .. outgoing_limit { + network.peer(target2.clone()) + .dial(Multiaddr::empty(), Vec::new(), TestHandler()) + .ok() + .expect("Unexpected connection limit."); + } + + let err = network.peer(target2) + .dial(Multiaddr::empty(), Vec::new(), TestHandler()) + .expect_err("Unexpected dialing success."); + + assert_eq!(err.current, outgoing_limit); + assert_eq!(err.limit, outgoing_limit); +} diff --git a/examples/distributed-key-value-store.rs b/examples/distributed-key-value-store.rs index 7166f098..014418d3 100644 --- a/examples/distributed-key-value-store.rs +++ b/examples/distributed-key-value-store.rs @@ -1,201 +1,208 @@ -// // Copyright 20l9 Parity Technologies (UK) Ltd. -// // -// // Permission is hereby granted, free of charge, to any person obtaining a -// // copy of this software and associated documentation files (the "Software"), -// // to deal in the Software without restriction, including without limitation -// // the rights to use, copy, modify, merge, publish, distribute, sublicense, -// // and/or sell copies of the Software, and to permit persons to whom the -// // Software is furnished to do so, subject to the following conditions: -// // -// // The above copyright notice and this permission notice shall be included in -// // all copies or substantial portions of the Software. -// // -// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// // DEALINGS IN THE SOFTWARE. +// Copyright 20l9 Parity Technologies (UK) Ltd. // -// //! A basic key value store demonstrating libp2p and the mDNS and Kademlia protocols. -// //! -// //! 1. Using two terminal windows, start two instances. If you local network -// //! allows mDNS, they will automatically connect. -// //! -// //! 2. Type `PUT my-key my-value` in terminal one and hit return. -// //! -// //! 3. Type `GET my-key` in terminal two and hit return. -// //! -// //! 4. Close with Ctrl-c. +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: // -// use async_std::{io, task}; -// use futures::prelude::*; -// use libp2p::identity::Keypair; -// use libp2p::kad::record::store::MemoryStore; -// use libp2p::kad::{record::Key, Kademlia, KademliaEvent, PutRecordOk, Quorum, Record}; -// use libp2p::{ -// build_development_transport, identity, -// mdns::{Mdns, MdnsEvent}, -// swarm::NetworkBehaviourEventProcess, -// NetworkBehaviour, PeerId, Swarm, -// }; -// use std::{ -// error::Error, -// task::{Context, Poll}, -// }; +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. // -// fn main() -> Result<(), Box> { -// env_logger::init(); -// -// // Create a random key for ourselves. -// let local_key = identity::Keypair::generate_ed25519(); -// let local_peer_id = PeerId::from(local_key.public()); -// -// // Set up a an encrypted DNS-enabled TCP Transport over the Mplex protocol. -// let transport = build_development_transport(local_key.clone())?; -// -// // We create a custom network behaviour that combines Kademlia and mDNS. -// #[derive(NetworkBehaviour)] -// struct MyBehaviour { -// kademlia: Kademlia, -// mdns: Mdns, -// } -// -// impl NetworkBehaviourEventProcess for MyBehaviour { -// // Called when `mdns` produces an event. -// fn inject_event(&mut self, event: MdnsEvent) { -// if let MdnsEvent::Discovered(list) = event { -// for (peer_id, multiaddr) in list { -// self.kademlia.add_address(&peer_id, multiaddr); -// } -// } -// } -// } -// -// impl NetworkBehaviourEventProcess for MyBehaviour { -// // Called when `kademlia` produces an event. -// fn inject_event(&mut self, message: KademliaEvent) { -// match message { -// KademliaEvent::GetRecordResult(Ok(result)) => { -// for Record { key, value, .. } in result.records { -// println!( -// "Got record {:?} {:?}", -// std::str::from_utf8(key.as_ref()).unwrap(), -// std::str::from_utf8(&value).unwrap(), -// ); -// } -// } -// KademliaEvent::GetRecordResult(Err(err)) => { -// eprintln!("Failed to get record: {:?}", err); -// } -// KademliaEvent::PutRecordResult(Ok(PutRecordOk { key })) => { -// println!( -// "Successfully put record {:?}", -// std::str::from_utf8(key.as_ref()).unwrap() -// ); -// } -// KademliaEvent::PutRecordResult(Err(err)) => { -// eprintln!("Failed to put record: {:?}", err); -// } -// _ => {} -// } -// } -// } -// -// // Create a swarm to manage peers and events. -// let mut swarm = { -// // Create a Kademlia behaviour. -// let store = MemoryStore::new(local_peer_id.clone()); -// let Keypair::Ed25519(local_key) = local_key; -// let kademlia = Kademlia::new(local_key, local_peer_id.clone(), store); -// let mdns = Mdns::new()?; -// let behaviour = MyBehaviour { kademlia, mdns }; -// Swarm::new(transport, behaviour, local_peer_id) -// }; -// -// // Read full lines from stdin -// let mut stdin = io::BufReader::new(io::stdin()).lines(); -// -// // Listen on all interfaces and whatever port the OS assigns. -// Swarm::listen_on(&mut swarm, "/ip4/0.0.0.0/tcp/0".parse()?)?; -// -// // Kick it off. -// let mut listening = false; -// task::block_on(future::poll_fn(move |cx: &mut Context| { -// loop { -// match stdin.try_poll_next_unpin(cx)? { -// Poll::Ready(Some(line)) => handle_input_line(&mut swarm.kademlia, line), -// Poll::Ready(None) => panic!("Stdin closed"), -// Poll::Pending => break, -// } -// } -// loop { -// match swarm.poll_next_unpin(cx) { -// Poll::Ready(Some(event)) => println!("{:?}", event), -// Poll::Ready(None) => return Poll::Ready(Ok(())), -// Poll::Pending => { -// if !listening { -// if let Some(a) = Swarm::listeners(&swarm).next() { -// println!("Listening on {:?}", a); -// listening = true; -// } -// } -// break; -// } -// } -// } -// Poll::Pending -// })) -// } -// -// fn handle_input_line(kademlia: &mut Kademlia, line: String) { -// let mut args = line.split(" "); -// -// match args.next() { -// Some("GET") => { -// let key = { -// match args.next() { -// Some(key) => Key::new(&key), -// None => { -// eprintln!("Expected key"); -// return; -// } -// } -// }; -// kademlia.get_record(&key, Quorum::One); -// } -// Some("PUT") => { -// let key = { -// match args.next() { -// Some(key) => Key::new(&key), -// None => { -// eprintln!("Expected key"); -// return; -// } -// } -// }; -// let value = { -// match args.next() { -// Some(value) => value.as_bytes().to_vec(), -// None => { -// eprintln!("Expected value"); -// return; -// } -// } -// }; -// let record = Record { -// key, -// value, -// publisher: None, -// expires: None, -// }; -// kademlia.put_record(record, Quorum::One); -// } -// _ => { -// eprintln!("expected GET or PUT"); -// } -// } -// } +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. -fn main() {} +//! A basic key value store demonstrating libp2p and the mDNS and Kademlia protocols. +//! +//! 1. Using two terminal windows, start two instances. If you local network +//! allows mDNS, they will automatically connect. +//! +//! 2. Type `PUT my-key my-value` in terminal one and hit return. +//! +//! 3. Type `GET my-key` in terminal two and hit return. +//! +//! 4. Close with Ctrl-c. + +use async_std::{io, task}; +use futures::prelude::*; +use libp2p::kad::record::store::MemoryStore; +use libp2p::kad::{ + record::Key, + Kademlia, + KademliaEvent, + PutRecordOk, + QueryResult, + Quorum, + Record +}; +use libp2p::{ + NetworkBehaviour, + PeerId, + Swarm, + build_development_transport, + identity, + mdns::{Mdns, MdnsEvent}, + swarm::NetworkBehaviourEventProcess +}; +use std::{error::Error, task::{Context, Poll}}; + +fn main() -> Result<(), Box> { + env_logger::init(); + + // Create a random key for ourselves. + let local_key = identity::Keypair::generate_ed25519(); + let local_peer_id = PeerId::from(local_key.public()); + + // Set up a an encrypted DNS-enabled TCP Transport over the Mplex protocol. + let transport = build_development_transport(local_key)?; + + // We create a custom network behaviour that combines Kademlia and mDNS. + #[derive(NetworkBehaviour)] + struct MyBehaviour { + kademlia: Kademlia, + mdns: Mdns + } + + impl NetworkBehaviourEventProcess for MyBehaviour { + // Called when `mdns` produces an event. + fn inject_event(&mut self, event: MdnsEvent) { + if let MdnsEvent::Discovered(list) = event { + for (peer_id, multiaddr) in list { + self.kademlia.add_address(&peer_id, multiaddr); + } + } + } + } + + impl NetworkBehaviourEventProcess for MyBehaviour { + // Called when `kademlia` produces an event. + fn inject_event(&mut self, message: KademliaEvent) { + match message { + KademliaEvent::QueryResult { result, .. } => match result { + QueryResult::GetRecord(Ok(ok)) => { + for Record { key, value, .. } in ok.records { + println!( + "Got record {:?} {:?}", + std::str::from_utf8(key.as_ref()).unwrap(), + std::str::from_utf8(&value).unwrap(), + ); + } + } + QueryResult::GetRecord(Err(err)) => { + eprintln!("Failed to get record: {:?}", err); + } + QueryResult::PutRecord(Ok(PutRecordOk { key })) => { + println!( + "Successfully put record {:?}", + std::str::from_utf8(key.as_ref()).unwrap() + ); + } + QueryResult::PutRecord(Err(err)) => { + eprintln!("Failed to put record: {:?}", err); + } + _ => {} + } + _ => {} + } + } + } + + // Create a swarm to manage peers and events. + let mut swarm = { + // Create a Kademlia behaviour. + let store = MemoryStore::new(local_peer_id.clone()); + let kademlia = Kademlia::new(local_peer_id.clone(), store); + let mdns = Mdns::new()?; + let behaviour = MyBehaviour { kademlia, mdns }; + Swarm::new(transport, behaviour, local_peer_id) + }; + + // Read full lines from stdin + let mut stdin = io::BufReader::new(io::stdin()).lines(); + + // Listen on all interfaces and whatever port the OS assigns. + Swarm::listen_on(&mut swarm, "/ip4/0.0.0.0/tcp/0".parse()?)?; + + // Kick it off. + let mut listening = false; + task::block_on(future::poll_fn(move |cx: &mut Context| { + loop { + match stdin.try_poll_next_unpin(cx)? { + Poll::Ready(Some(line)) => handle_input_line(&mut swarm.kademlia, line), + Poll::Ready(None) => panic!("Stdin closed"), + Poll::Pending => break + } + } + loop { + match swarm.poll_next_unpin(cx) { + Poll::Ready(Some(event)) => println!("{:?}", event), + Poll::Ready(None) => return Poll::Ready(Ok(())), + Poll::Pending => { + if !listening { + if let Some(a) = Swarm::listeners(&swarm).next() { + println!("Listening on {:?}", a); + listening = true; + } + } + break + } + } + } + Poll::Pending + })) +} + +fn handle_input_line(kademlia: &mut Kademlia, line: String) { + let mut args = line.split(" "); + + match args.next() { + Some("GET") => { + let key = { + match args.next() { + Some(key) => Key::new(&key), + None => { + eprintln!("Expected key"); + return; + } + } + }; + kademlia.get_record(&key, Quorum::One); + } + Some("PUT") => { + let key = { + match args.next() { + Some(key) => Key::new(&key), + None => { + eprintln!("Expected key"); + return; + } + } + }; + let value = { + match args.next() { + Some(value) => value.as_bytes().to_vec(), + None => { + eprintln!("Expected value"); + return; + } + } + }; + let record = Record { + key, + value, + publisher: None, + expires: None, + }; + kademlia.put_record(record, Quorum::One).expect("Failed to store record locally."); + } + _ => { + eprintln!("expected GET or PUT"); + } + } +} diff --git a/examples/ipfs-kad.rs b/examples/ipfs-kad.rs index 56c26955..ec48435d 100644 --- a/examples/ipfs-kad.rs +++ b/examples/ipfs-kad.rs @@ -1,123 +1,129 @@ -// // Copyright 2018 Parity Technologies (UK) Ltd. -// // -// // Permission is hereby granted, free of charge, to any person obtaining a -// // copy of this software and associated documentation files (the "Software"), -// // to deal in the Software without restriction, including without limitation -// // the rights to use, copy, modify, merge, publish, distribute, sublicense, -// // and/or sell copies of the Software, and to permit persons to whom the -// // Software is furnished to do so, subject to the following conditions: -// // -// // The above copyright notice and this permission notice shall be included in -// // all copies or substantial portions of the Software. -// // -// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// // DEALINGS IN THE SOFTWARE. +// Copyright 2018 Parity Technologies (UK) Ltd. // -// //! Demonstrates how to perform Kademlia queries on the IPFS network. -// //! -// //! You can pass as parameter a base58 peer ID to search for. If you don't pass any parameter, a -// //! peer ID will be generated randomly. +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: // -// use async_std::task; -// use libp2p::identity::Keypair; -// use libp2p::kad::record::store::MemoryStore; -// use libp2p::kad::{GetClosestPeersError, Kademlia, KademliaConfig, KademliaEvent}; -// use libp2p::{build_development_transport, identity, PeerId, Swarm}; -// use std::{env, error::Error, time::Duration}; +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. // -// fn main() -> Result<(), Box> { -// env_logger::init(); -// -// // Create a random key for ourselves. -// let local_key = identity::Keypair::generate_ed25519(); -// let local_peer_id = PeerId::from(local_key.public()); -// -// // Set up a an encrypted DNS-enabled TCP Transport over the Mplex protocol -// let transport = build_development_transport(local_key.clone())?; -// -// // Create a swarm to manage peers and events. -// let mut swarm = { -// // Create a Kademlia behaviour. -// let mut cfg = KademliaConfig::default(); -// cfg.set_query_timeout(Duration::from_secs(5 * 60)); -// let store = MemoryStore::new(local_peer_id.clone()); -// let mut behaviour = Kademlia::with_config(local_key, local_peer_id.clone(), store, cfg); -// -// // TODO: the /dnsaddr/ scheme is not supported (https://github.com/libp2p/rust-libp2p/issues/967) -// /*behaviour.add_address(&"QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); -// behaviour.add_address(&"QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); -// behaviour.add_address(&"QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); -// behaviour.add_address(&"QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap());*/ -// -// // The only address that currently works. -// behaviour.add_address( -// &"QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ".parse()?, -// "/ip4/104.131.131.82/tcp/4001".parse()?, -// ); -// -// // The following addresses always fail signature verification, possibly due to -// // RSA keys with < 2048 bits. -// // behaviour.add_address(&"QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM".parse().unwrap(), "/ip4/104.236.179.241/tcp/4001".parse().unwrap()); -// // behaviour.add_address(&"QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu".parse().unwrap(), "/ip4/128.199.219.111/tcp/4001".parse().unwrap()); -// // behaviour.add_address(&"QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64".parse().unwrap(), "/ip4/104.236.76.40/tcp/4001".parse().unwrap()); -// // behaviour.add_address(&"QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd".parse().unwrap(), "/ip4/178.62.158.247/tcp/4001".parse().unwrap()); -// -// // The following addresses are permanently unreachable: -// // Other(Other(A(Transport(A(Underlying(Os { code: 101, kind: Other, message: "Network is unreachable" })))))) -// // behaviour.add_address(&"QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM".parse().unwrap(), "/ip6/2604:a880:1:20::203:d001/tcp/4001".parse().unwrap()); -// // behaviour.add_address(&"QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu".parse().unwrap(), "/ip6/2400:6180:0:d0::151:6001/tcp/4001".parse().unwrap()); -// // behaviour.add_address(&"QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64".parse().unwrap(), "/ip6/2604:a880:800:10::4a:5001/tcp/4001".parse().unwrap()); -// // behaviour.add_address(&"QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd".parse().unwrap(), "/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001".parse().unwrap()); -// Swarm::new(transport, behaviour, local_peer_id) -// }; -// -// // Order Kademlia to search for a peer. -// let to_search: PeerId = if let Some(peer_id) = env::args().nth(1) { -// peer_id.parse()? -// } else { -// identity::Keypair::generate_ed25519().public().into() -// }; -// -// println!("Searching for the closest peers to {:?}", to_search); -// swarm.get_closest_peers(to_search); -// -// // Kick it off! -// task::block_on(async move { -// loop { -// let event = swarm.next().await; -// if let KademliaEvent::GetClosestPeersResult(result) = event { -// match result { -// Ok(ok) => { -// if !ok.peers.is_empty() { -// println!("Query finished with closest peers: {:#?}", ok.peers) -// } else { -// // The example is considered failed as there -// // should always be at least 1 reachable peer. -// println!("Query finished with no closest peers.") -// } -// } -// Err(GetClosestPeersError::Timeout { peers, .. }) => { -// if !peers.is_empty() { -// println!("Query timed out with closest peers: {:#?}", peers) -// } else { -// // The example is considered failed as there -// // should always be at least 1 reachable peer. -// println!("Query timed out with no closest peers."); -// } -// } -// }; -// -// break; -// } -// } -// -// Ok(()) -// }) -// } +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. -fn main() {} +//! Demonstrates how to perform Kademlia queries on the IPFS network. +//! +//! You can pass as parameter a base58 peer ID to search for. If you don't pass any parameter, a +//! peer ID will be generated randomly. + +use async_std::task; +use libp2p::{ + Swarm, + PeerId, + identity, + build_development_transport +}; +use libp2p::kad::{ + Kademlia, + KademliaConfig, + KademliaEvent, + GetClosestPeersError, + QueryResult, +}; +use libp2p::kad::record::store::MemoryStore; +use std::{env, error::Error, time::Duration}; + +fn main() -> Result<(), Box> { + env_logger::init(); + + // Create a random key for ourselves. + let local_key = identity::Keypair::generate_ed25519(); + let local_peer_id = PeerId::from(local_key.public()); + + // Set up a an encrypted DNS-enabled TCP Transport over the Mplex protocol + let transport = build_development_transport(local_key)?; + + // Create a swarm to manage peers and events. + let mut swarm = { + // Create a Kademlia behaviour. + let mut cfg = KademliaConfig::default(); + cfg.set_query_timeout(Duration::from_secs(5 * 60)); + let store = MemoryStore::new(local_peer_id.clone()); + let mut behaviour = Kademlia::with_config(local_peer_id.clone(), store, cfg); + + // TODO: the /dnsaddr/ scheme is not supported (https://github.com/libp2p/rust-libp2p/issues/967) + /*behaviour.add_address(&"QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); + behaviour.add_address(&"QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); + behaviour.add_address(&"QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); + behaviour.add_address(&"QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap());*/ + + // The only address that currently works. + behaviour.add_address(&"QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ".parse()?, "/ip4/104.131.131.82/tcp/4001".parse()?); + + // The following addresses always fail signature verification, possibly due to + // RSA keys with < 2048 bits. + // behaviour.add_address(&"QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM".parse().unwrap(), "/ip4/104.236.179.241/tcp/4001".parse().unwrap()); + // behaviour.add_address(&"QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu".parse().unwrap(), "/ip4/128.199.219.111/tcp/4001".parse().unwrap()); + // behaviour.add_address(&"QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64".parse().unwrap(), "/ip4/104.236.76.40/tcp/4001".parse().unwrap()); + // behaviour.add_address(&"QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd".parse().unwrap(), "/ip4/178.62.158.247/tcp/4001".parse().unwrap()); + + // The following addresses are permanently unreachable: + // Other(Other(A(Transport(A(Underlying(Os { code: 101, kind: Other, message: "Network is unreachable" })))))) + // behaviour.add_address(&"QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM".parse().unwrap(), "/ip6/2604:a880:1:20::203:d001/tcp/4001".parse().unwrap()); + // behaviour.add_address(&"QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu".parse().unwrap(), "/ip6/2400:6180:0:d0::151:6001/tcp/4001".parse().unwrap()); + // behaviour.add_address(&"QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64".parse().unwrap(), "/ip6/2604:a880:800:10::4a:5001/tcp/4001".parse().unwrap()); + // behaviour.add_address(&"QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd".parse().unwrap(), "/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001".parse().unwrap()); + Swarm::new(transport, behaviour, local_peer_id) + }; + + // Order Kademlia to search for a peer. + let to_search: PeerId = if let Some(peer_id) = env::args().nth(1) { + peer_id.parse()? + } else { + identity::Keypair::generate_ed25519().public().into() + }; + + println!("Searching for the closest peers to {:?}", to_search); + swarm.get_closest_peers(to_search); + + // Kick it off! + task::block_on(async move { + loop { + let event = swarm.next().await; + if let KademliaEvent::QueryResult { + result: QueryResult::GetClosestPeers(result), + .. + } = event { + match result { + Ok(ok) => + if !ok.peers.is_empty() { + println!("Query finished with closest peers: {:#?}", ok.peers) + } else { + // The example is considered failed as there + // should always be at least 1 reachable peer. + println!("Query finished with no closest peers.") + } + Err(GetClosestPeersError::Timeout { peers, .. }) => + if !peers.is_empty() { + println!("Query timed out with closest peers: {:#?}", peers) + } else { + // The example is considered failed as there + // should always be at least 1 reachable peer. + println!("Query timed out with no closest peers."); + } + }; + + break; + } + } + + Ok(()) + }) +} diff --git a/misc/core-derive/Cargo.toml b/misc/core-derive/Cargo.toml index 166156cd..b5521838 100644 --- a/misc/core-derive/Cargo.toml +++ b/misc/core-derive/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-core-derive" edition = "2018" description = "Procedural macros of libp2p-core" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -17,4 +17,4 @@ syn = { version = "1.0.8", default-features = false, features = ["clone-impls", quote = "1.0" [dev-dependencies] -libp2p = { version = "0.18.0", path = "../.." } +libp2p = { version = "0.19.0", path = "../.." } diff --git a/misc/core-derive/src/lib.rs b/misc/core-derive/src/lib.rs index d7d28f15..ba9b0411 100644 --- a/misc/core-derive/src/lib.rs +++ b/misc/core-derive/src/lib.rs @@ -288,7 +288,7 @@ fn build_struct(ast: &DeriveInput, data_struct: &DataStruct) -> TokenStream { quote!{ ev } }; - for _ in 0 .. data_struct.fields.iter().filter(|f| !is_ignored(f)).count() - 1 - field_n { + for _ in 0 .. data_struct.fields.iter().filter(|f| !is_ignored(f)).count() - 1 - enum_n { elem = quote!{ #either_ident::First(#elem) }; } @@ -378,7 +378,7 @@ fn build_struct(ast: &DeriveInput, data_struct: &DataStruct) -> TokenStream { } else { quote!{ event } }; - for _ in 0 .. data_struct.fields.iter().filter(|f| !is_ignored(f)).count() - 1 - field_n { + for _ in 0 .. data_struct.fields.iter().filter(|f| !is_ignored(f)).count() - 1 - enum_n { wrapped_event = quote!{ #either_ident::First(#wrapped_event) }; } diff --git a/misc/core-derive/tests/test.rs b/misc/core-derive/tests/test.rs index b65fb16e..6181de8e 100644 --- a/misc/core-derive/tests/test.rs +++ b/misc/core-derive/tests/test.rs @@ -109,6 +109,33 @@ fn three_fields() { } } +#[test] +fn three_fields_non_last_ignored() { + #[allow(dead_code)] + #[derive(NetworkBehaviour)] + struct Foo { + ping: libp2p::ping::Ping, + #[behaviour(ignore)] + identify: String, + kad: libp2p::kad::Kademlia, + } + + impl libp2p::swarm::NetworkBehaviourEventProcess for Foo { + fn inject_event(&mut self, _: libp2p::ping::PingEvent) { + } + } + + impl libp2p::swarm::NetworkBehaviourEventProcess for Foo { + fn inject_event(&mut self, _: libp2p::kad::KademliaEvent) { + } + } + + #[allow(dead_code)] + fn foo() { + require_net_behaviour::(); + } +} + #[test] fn custom_polling() { #[allow(dead_code)] diff --git a/misc/multiaddr/Cargo.toml b/misc/multiaddr/Cargo.toml index 239407ae..38f920e6 100644 --- a/misc/multiaddr/Cargo.toml +++ b/misc/multiaddr/Cargo.toml @@ -6,14 +6,14 @@ description = "Implementation of the multiaddr format" homepage = "https://github.com/libp2p/rust-libp2p" keywords = ["multiaddr", "ipfs"] license = "MIT" -version = "0.8.0" +version = "0.9.0" [dependencies] arrayref = "0.3" bs58 = "0.3.0" byteorder = "1.3.1" data-encoding = "2.1" -multihash = "0.10" +multihash = "0.11.0" percent-encoding = "2.1.0" serde = "1.0.70" static_assertions = "1.1" diff --git a/misc/multiaddr/src/from_url.rs b/misc/multiaddr/src/from_url.rs index 454672da..738ed20b 100644 --- a/misc/multiaddr/src/from_url.rs +++ b/misc/multiaddr/src/from_url.rs @@ -70,7 +70,7 @@ fn from_url_inner_http_ws(url: url::Url, lossy: bool) -> std::result::Result() { Protocol::from(ip) } else { - Protocol::Dns4(hostname.into()) + Protocol::Dns(hostname.into()) } } else { return Err(FromUrlErr::BadUrl); @@ -185,31 +185,31 @@ mod tests { #[test] fn dns_addr_ws() { let addr = from_url("ws://example.com").unwrap(); - assert_eq!(addr, "/dns4/example.com/tcp/80/ws".parse().unwrap()); + assert_eq!(addr, "/dns/example.com/tcp/80/ws".parse().unwrap()); } #[test] fn dns_addr_http() { let addr = from_url("http://example.com").unwrap(); - assert_eq!(addr, "/dns4/example.com/tcp/80/http".parse().unwrap()); + assert_eq!(addr, "/dns/example.com/tcp/80/http".parse().unwrap()); } #[test] fn dns_addr_wss() { let addr = from_url("wss://example.com").unwrap(); - assert_eq!(addr, "/dns4/example.com/tcp/443/wss".parse().unwrap()); + assert_eq!(addr, "/dns/example.com/tcp/443/wss".parse().unwrap()); } #[test] fn dns_addr_https() { let addr = from_url("https://example.com").unwrap(); - assert_eq!(addr, "/dns4/example.com/tcp/443/https".parse().unwrap()); + assert_eq!(addr, "/dns/example.com/tcp/443/https".parse().unwrap()); } #[test] fn bad_hostname() { let addr = from_url("wss://127.0.0.1x").unwrap(); - assert_eq!(addr, "/dns4/127.0.0.1x/tcp/443/wss".parse().unwrap()); + assert_eq!(addr, "/dns/127.0.0.1x/tcp/443/wss".parse().unwrap()); } #[test] @@ -223,7 +223,7 @@ mod tests { #[test] fn dns_and_port() { let addr = from_url("http://example.com:1000").unwrap(); - assert_eq!(addr, "/dns4/example.com/tcp/1000/http".parse().unwrap()); + assert_eq!(addr, "/dns/example.com/tcp/1000/http".parse().unwrap()); } #[test] diff --git a/misc/multiaddr/src/protocol.rs b/misc/multiaddr/src/protocol.rs index 6491d0d8..1ff0a8a6 100644 --- a/misc/multiaddr/src/protocol.rs +++ b/misc/multiaddr/src/protocol.rs @@ -17,6 +17,7 @@ use unsigned_varint::{encode, decode}; use crate::onion_addr::Onion3Addr; const DCCP: u32 = 33; +const DNS: u32 = 53; const DNS4: u32 = 54; const DNS6: u32 = 55; const DNSADDR: u32 = 56; @@ -66,6 +67,7 @@ const PATH_SEGMENT_ENCODE_SET: &percent_encoding::AsciiSet = &percent_encoding:: #[derive(PartialEq, Eq, Clone, Debug)] pub enum Protocol<'a> { Dccp(u16), + Dns(Cow<'a, str>), Dns4(Cow<'a, str>), Dns6(Cow<'a, str>), Dnsaddr(Cow<'a, str>), @@ -125,6 +127,10 @@ impl<'a> Protocol<'a> { let s = iter.next().ok_or(Error::InvalidProtocolString)?; Ok(Protocol::Ip6(Ipv6Addr::from_str(s)?)) } + "dns" => { + let s = iter.next().ok_or(Error::InvalidProtocolString)?; + Ok(Protocol::Dns(Cow::Borrowed(s))) + } "dns4" => { let s = iter.next().ok_or(Error::InvalidProtocolString)?; Ok(Protocol::Dns4(Cow::Borrowed(s))) @@ -206,6 +212,11 @@ impl<'a> Protocol<'a> { let num = rdr.read_u16::()?; Ok((Protocol::Dccp(num), rest)) } + DNS => { + let (n, input) = decode::usize(input)?; + let (data, rest) = split_at(n, input)?; + Ok((Protocol::Dns(Cow::Borrowed(str::from_utf8(data)?)), rest)) + } DNS4 => { let (n, input) = decode::usize(input)?; let (data, rest) = split_at(n, input)?; @@ -345,6 +356,12 @@ impl<'a> Protocol<'a> { w.write_all(encode::u32(SCTP, &mut buf))?; w.write_u16::(*port)? } + Protocol::Dns(s) => { + w.write_all(encode::u32(DNS, &mut buf))?; + let bytes = s.as_bytes(); + w.write_all(encode::usize(bytes.len(), &mut encode::usize_buffer()))?; + w.write_all(&bytes)? + } Protocol::Dns4(s) => { w.write_all(encode::u32(DNS4, &mut buf))?; let bytes = s.as_bytes(); @@ -421,6 +438,7 @@ impl<'a> Protocol<'a> { use self::Protocol::*; match self { Dccp(a) => Dccp(a), + Dns(cow) => Dns(Cow::Owned(cow.into_owned())), Dns4(cow) => Dns4(Cow::Owned(cow.into_owned())), Dns6(cow) => Dns6(Cow::Owned(cow.into_owned())), Dnsaddr(cow) => Dnsaddr(Cow::Owned(cow.into_owned())), @@ -454,6 +472,7 @@ impl<'a> fmt::Display for Protocol<'a> { use self::Protocol::*; match self { Dccp(port) => write!(f, "/dccp/{}", port), + Dns(s) => write!(f, "/dns/{}", s), Dns4(s) => write!(f, "/dns4/{}", s), Dns6(s) => write!(f, "/dns6/{}", s), Dnsaddr(s) => write!(f, "/dnsaddr/{}", s), diff --git a/misc/multiaddr/tests/lib.rs b/misc/multiaddr/tests/lib.rs index dafeb215..d0928022 100644 --- a/misc/multiaddr/tests/lib.rs +++ b/misc/multiaddr/tests/lib.rs @@ -76,36 +76,37 @@ struct Proto(Protocol<'static>); impl Arbitrary for Proto { fn arbitrary(g: &mut G) -> Self { use Protocol::*; - match g.gen_range(0, 24) { // TODO: Add Protocol::Quic + match g.gen_range(0, 25) { // TODO: Add Protocol::Quic 0 => Proto(Dccp(g.gen())), - 1 => Proto(Dns4(Cow::Owned(SubString::arbitrary(g).0))), - 2 => Proto(Dns6(Cow::Owned(SubString::arbitrary(g).0))), - 3 => Proto(Http), - 4 => Proto(Https), - 5 => Proto(Ip4(Ipv4Addr::arbitrary(g))), - 6 => Proto(Ip6(Ipv6Addr::arbitrary(g))), - 7 => Proto(P2pWebRtcDirect), - 8 => Proto(P2pWebRtcStar), - 9 => Proto(P2pWebSocketStar), - 10 => Proto(Memory(g.gen())), + 1 => Proto(Dns(Cow::Owned(SubString::arbitrary(g).0))), + 2 => Proto(Dns4(Cow::Owned(SubString::arbitrary(g).0))), + 3 => Proto(Dns6(Cow::Owned(SubString::arbitrary(g).0))), + 4 => Proto(Http), + 5 => Proto(Https), + 6 => Proto(Ip4(Ipv4Addr::arbitrary(g))), + 7 => Proto(Ip6(Ipv6Addr::arbitrary(g))), + 8 => Proto(P2pWebRtcDirect), + 9 => Proto(P2pWebRtcStar), + 10 => Proto(P2pWebSocketStar), + 11 => Proto(Memory(g.gen())), // TODO: impl Arbitrary for Multihash: - 11 => Proto(P2p(multihash("QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC"))), - 12 => Proto(P2pCircuit), - 13 => Proto(Quic), - 14 => Proto(Sctp(g.gen())), - 15 => Proto(Tcp(g.gen())), - 16 => Proto(Udp(g.gen())), - 17 => Proto(Udt), - 18 => Proto(Unix(Cow::Owned(SubString::arbitrary(g).0))), - 19 => Proto(Utp), - 20 => Proto(Ws("/".into())), - 21 => Proto(Wss("/".into())), - 22 => { + 12 => Proto(P2p(multihash("QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC"))), + 13 => Proto(P2pCircuit), + 14 => Proto(Quic), + 15 => Proto(Sctp(g.gen())), + 16 => Proto(Tcp(g.gen())), + 17 => Proto(Udp(g.gen())), + 18 => Proto(Udt), + 19 => Proto(Unix(Cow::Owned(SubString::arbitrary(g).0))), + 20 => Proto(Utp), + 21 => Proto(Ws("/".into())), + 22 => Proto(Wss("/".into())), + 23 => { let mut a = [0; 10]; g.fill(&mut a); Proto(Onion(Cow::Owned(a), g.gen_range(1, std::u16::MAX))) }, - 23 => { + 24 => { let mut a = [0; 35]; g.fill_bytes(&mut a); Proto(Onion3((a, g.gen_range(1, std::u16::MAX)).into())) diff --git a/misc/peer-id-generator/Cargo.toml b/misc/peer-id-generator/Cargo.toml index f3efec74..1813bcbd 100644 --- a/misc/peer-id-generator/Cargo.toml +++ b/misc/peer-id-generator/Cargo.toml @@ -11,5 +11,5 @@ categories = ["network-programming", "asynchronous"] publish = false [dependencies] -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } num_cpus = "1.8" diff --git a/muxers/mplex/Cargo.toml b/muxers/mplex/Cargo.toml index 4be96b7e..23398a67 100644 --- a/muxers/mplex/Cargo.toml +++ b/muxers/mplex/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-mplex" edition = "2018" description = "Mplex multiplexing protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -14,11 +14,11 @@ bytes = "0.5" fnv = "1.0" futures = "0.3.1" futures_codec = "0.3.4" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4" parking_lot = "0.10" unsigned-varint = { version = "0.3", features = ["futures-codec"] } [dev-dependencies] async-std = "1.0" -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } diff --git a/muxers/yamux/Cargo.toml b/muxers/yamux/Cargo.toml index 4255bea4..2901556a 100644 --- a/muxers/yamux/Cargo.toml +++ b/muxers/yamux/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-yamux" edition = "2018" description = "Yamux multiplexing protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,7 +11,7 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } parking_lot = "0.10" thiserror = "1.0" yamux = "0.4.5" diff --git a/protocols/deflate/Cargo.toml b/protocols/deflate/Cargo.toml index 0d2552f5..ab7f0018 100644 --- a/protocols/deflate/Cargo.toml +++ b/protocols/deflate/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-deflate" edition = "2018" description = "Deflate encryption protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,11 +11,11 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } flate2 = "1.0" [dev-dependencies] async-std = "1.0" -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } rand = "0.7" quickcheck = "0.9" diff --git a/protocols/floodsub/Cargo.toml b/protocols/floodsub/Cargo.toml index 4c8fa279..45701703 100644 --- a/protocols/floodsub/Cargo.toml +++ b/protocols/floodsub/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-floodsub" edition = "2018" description = "Floodsub protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -13,8 +13,8 @@ categories = ["network-programming", "asynchronous"] cuckoofilter = "0.3.2" fnv = "1.0" futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } prost = "0.6.1" rand = "0.7" smallvec = "1.0" diff --git a/protocols/gossipsub/Cargo.toml b/protocols/gossipsub/Cargo.toml index 8f1b297d..8d91104d 100644 --- a/protocols/gossipsub/Cargo.toml +++ b/protocols/gossipsub/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-gossipsub" edition = "2018" description = "Gossipsub protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Age Manning "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -10,8 +10,8 @@ keywords = ["peer-to-peer", "libp2p", "networking"] categories = ["network-programming", "asynchronous"] [dependencies] -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } bytes = "0.5.4" byteorder = "1.3.2" fnv = "1.0.6" @@ -30,8 +30,8 @@ prost = "0.6.1" [dev-dependencies] async-std = "1.4.0" env_logger = "0.7.1" -libp2p-plaintext = { version = "0.18.0", path = "../plaintext" } -libp2p-yamux = { version = "0.18.0", path = "../../muxers/yamux" } +libp2p-plaintext = { version = "0.19.0", path = "../plaintext" } +libp2p-yamux = { version = "0.19.0", path = "../../muxers/yamux" } quickcheck = "0.9.2" [build-dependencies] diff --git a/protocols/gossipsub/src/handler.rs b/protocols/gossipsub/src/handler.rs index 495bf305..b9cf9559 100644 --- a/protocols/gossipsub/src/handler.rs +++ b/protocols/gossipsub/src/handler.rs @@ -27,7 +27,7 @@ use libp2p_swarm::protocols_handler::{ KeepAlive, ProtocolsHandler, ProtocolsHandlerEvent, ProtocolsHandlerUpgrErr, SubstreamProtocol, }; use libp2p_swarm::NegotiatedSubstream; -use log::{debug, trace, warn}; +use log::{debug, error, trace, warn}; use smallvec::SmallVec; use std::{ borrow::Cow, @@ -274,7 +274,13 @@ impl ProtocolsHandler for GossipsubHandler { Some(OutboundSubstreamState::PendingFlush(substream)) } Err(e) => { - return Poll::Ready(ProtocolsHandlerEvent::Close(e)); + if let io::ErrorKind::PermissionDenied = e.kind() { + error!("Message over the maximum transmission limit was not sent."); + self.outbound_substream = + Some(OutboundSubstreamState::WaitingOutput(substream)); + } else { + return Poll::Ready(ProtocolsHandlerEvent::Close(e)); + } } } } diff --git a/protocols/identify/Cargo.toml b/protocols/identify/Cargo.toml index 31fb5e9a..77f3dec9 100644 --- a/protocols/identify/Cargo.toml +++ b/protocols/identify/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-identify" edition = "2018" description = "Nodes identifcation protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,8 +11,8 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } log = "0.4.1" prost = "0.6.1" smallvec = "1.0" @@ -20,9 +20,9 @@ wasm-timer = "0.2" [dev-dependencies] async-std = "1.0" -libp2p-mplex = { version = "0.18.0", path = "../../muxers/mplex" } -libp2p-secio = { version = "0.18.0", path = "../../protocols/secio" } -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } +libp2p-mplex = { version = "0.19.0", path = "../../muxers/mplex" } +libp2p-secio = { version = "0.19.0", path = "../../protocols/secio" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } [build-dependencies] prost-build = "0.6" diff --git a/protocols/kad/Cargo.toml b/protocols/kad/Cargo.toml index 85b5d8f9..9f3ccc8d 100644 --- a/protocols/kad/Cargo.toml +++ b/protocols/kad/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-kad" edition = "2018" description = "Kademlia protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -17,9 +17,9 @@ fnv = "1.0" futures_codec = "0.3.4" futures = "0.3.1" log = "0.4" -libp2p-core = { version = "0.18.0", path = "../../core" } -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } -multihash = "0.10" +libp2p-core = { version = "0.19.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } +multihash = "0.11.0" prost = "0.6.1" rand = "0.7.2" sha2 = "0.8.0" @@ -34,8 +34,8 @@ derivative = "2.0.2" trust-graph = { git = "ssh://git@github.com/fluencelabs/arqada.git", branch = "memory_store_set" } [dev-dependencies] -libp2p-secio = { version = "0.18.0", path = "../secio" } -libp2p-yamux = { version = "0.18.0", path = "../../muxers/yamux" } +libp2p-secio = { version = "0.19.0", path = "../secio" } +libp2p-yamux = { version = "0.19.0", path = "../../muxers/yamux" } quickcheck = "0.9.0" [build-dependencies] diff --git a/protocols/kad/src/behaviour.rs b/protocols/kad/src/behaviour.rs index bc948756..fc85b5d0 100644 --- a/protocols/kad/src/behaviour.rs +++ b/protocols/kad/src/behaviour.rs @@ -45,13 +45,17 @@ use log::{info, debug, warn}; use smallvec::SmallVec; use std::{borrow::{Borrow, Cow}, error, iter, time::Duration}; use std::collections::{HashSet, VecDeque}; +use std::fmt; use std::num::NonZeroUsize; use std::task::{Context, Poll}; +use std::vec; use wasm_timer::Instant; use libp2p_core::identity::ed25519::{Keypair, PublicKey}; use trust_graph::TrustGraph; use derivative::Derivative; +pub use crate::query::QueryStats; + /// Network behaviour that handles Kademlia. pub struct Kademlia { /// The Kademlia routing table. @@ -293,6 +297,46 @@ where } } + /// Gets an iterator over immutable references to all running queries. + pub fn iter_queries<'a>(&'a self) -> impl Iterator> { + self.queries.iter().filter_map(|query| + if !query.is_finished() { + Some(QueryRef { query }) + } else { + None + }) + } + + /// Gets an iterator over mutable references to all running queries. + pub fn iter_queries_mut<'a>(&'a mut self) -> impl Iterator> { + self.queries.iter_mut().filter_map(|query| + if !query.is_finished() { + Some(QueryMut { query }) + } else { + None + }) + } + + /// Gets an immutable reference to a running query, if it exists. + pub fn query<'a>(&'a self, id: &QueryId) -> Option> { + self.queries.get(id).and_then(|query| + if !query.is_finished() { + Some(QueryRef { query }) + } else { + None + }) + } + + /// Gets a mutable reference to a running query, if it exists. + pub fn query_mut<'a>(&'a mut self, id: &QueryId) -> Option> { + self.queries.get_mut(id).and_then(|query| + if !query.is_finished() { + Some(QueryMut { query }) + } else { + None + }) + } + /// Adds a known listen address of a peer participating in the DHT to the /// routing table. /// @@ -357,10 +401,11 @@ where self.kbuckets.iter().map(|entry| entry.node.key.preimage()) } - /// Performs a lookup for the closest peers to the given key. + /// Initiates an iterative query for the closest peers to the given key. /// - /// The result of this operation is delivered in [`KademliaEvent::GetClosestPeersResult`]. - pub fn get_closest_peers(&mut self, key: K) + /// The result of the query is delivered in a + /// [`KademliaEvent::QueryResult{QueryResult::GetClosestPeers}`]. + pub fn get_closest_peers(&mut self, key: K) -> QueryId where K: Borrow<[u8]> + Clone { @@ -368,13 +413,14 @@ where let target = kbucket::Key::new(key); let peers = self.kbuckets.closest_keys(&target); let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); + self.queries.add_iter_closest(target.clone(), peers, inner) } /// Performs a lookup for a record in the DHT. /// - /// The result of this operation is delivered in [`KademliaEvent::GetRecordResult`]. - pub fn get_record(&mut self, key: &record::Key, quorum: Quorum) { + /// The result of this operation is delivered in a + /// [`KademliaEvent::QueryResult{QueryResult::GetRecord}`]. + pub fn get_record(&mut self, key: &record::Key, quorum: Quorum) -> QueryId { let quorum = quorum.eval(self.queries.config().replication_factor); let mut records = Vec::with_capacity(quorum.get()); @@ -383,25 +429,30 @@ where self.store.remove(key) } else { records.push(record.into_owned()); - if quorum.get() == 1 { - self.queued_events.push_back(NetworkBehaviourAction::GenerateEvent( - KademliaEvent::GetRecordResult(Ok(GetRecordOk { records })) - )); - return; - } } } + let done = records.len() >= quorum.get(); let target = kbucket::Key::new(key.clone()); let info = QueryInfo::GetRecord { key: key.clone(), records, quorum, cache_at: None }; let peers = self.kbuckets.closest_keys(&target); let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); + let id = self.queries.add_iter_closest(target.clone(), peers, inner); // (*) + + // Instantly finish the query if we already have enough records. + if done { + self.queries.get_mut(&id).expect("by (*)").finish(); + } + + id } /// Stores a record in the DHT. /// - /// The result of this operation is delivered in [`KademliaEvent::PutRecordResult`]. + /// Returns `Ok` if a record has been stored locally, providing the + /// `QueryId` of the initial query that replicates the record in the DHT. + /// The result of the query is eventually reported as a + /// [`KademliaEvent::QueryResult{QueryResult::PutRecord}`]. /// /// The record is always stored locally with the given expiration. If the record's /// expiration is `None`, the common case, it does not expire in local storage @@ -413,28 +464,23 @@ where /// does not update the record's expiration in local storage, thus a given record /// with an explicit expiration will always expire at that instant and until then /// is subject to regular (re-)replication and (re-)publication. - pub fn put_record(&mut self, mut record: Record, quorum: Quorum) { + pub fn put_record(&mut self, mut record: Record, quorum: Quorum) -> Result { record.publisher = Some(self.kbuckets.local_key().preimage().clone()); - if let Err(err) = self.store.put(record.clone()) { - self.queued_events.push_back(NetworkBehaviourAction::GenerateEvent( - KademliaEvent::PutRecordResult(Err( - PutRecordError::LocalStorageError { - key: record.key, - cause: err, - } - )) - )); - } else { - record.expires = record.expires.or_else(|| - self.record_ttl.map(|ttl| Instant::now() + ttl)); - let quorum = quorum.eval(self.queries.config().replication_factor); - let target = kbucket::Key::new(record.key.clone()); - let peers = self.kbuckets.closest_keys(&target); - let context = PutRecordContext::Publish; - let info = QueryInfo::PreparePutRecord { record, quorum, context }; - let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); - } + self.store.put(record.clone())?; + record.expires = record.expires.or_else(|| + self.record_ttl.map(|ttl| Instant::now() + ttl)); + let quorum = quorum.eval(self.queries.config().replication_factor); + let target = kbucket::Key::new(record.key.clone()); + let peers = self.kbuckets.closest_keys(&target); + let context = PutRecordContext::Publish; + let info = QueryInfo::PutRecord { + context, + record, + quorum, + phase: PutRecordPhase::GetClosestPeers + }; + let inner = QueryInner::new(info); + Ok(self.queries.add_iter_closest(target.clone(), peers, inner)) } /// Removes the record with the given key from _local_ storage, @@ -469,18 +515,28 @@ where /// refreshed by initiating an additional bootstrapping query for each such /// bucket with random keys. /// - /// The result(s) of this operation are delivered in [`KademliaEvent::BootstrapResult`], - /// with one event per bootstrapping query. + /// Returns `Ok` if bootstrapping has been initiated with a self-lookup, providing the + /// `QueryId` for the entire bootstrapping process. The progress of bootstrapping is + /// reported via [`KademliaEvent::QueryResult{QueryResult::Bootstrap}`] events, + /// with one such event per bootstrapping query. + /// + /// Returns `Err` if bootstrapping is impossible due an empty routing table. /// /// > **Note**: Bootstrapping requires at least one node of the DHT to be known. /// > See [`Kademlia::add_address`]. - pub fn bootstrap(&mut self) { + pub fn bootstrap(&mut self) -> Result { let local_key = self.kbuckets.local_key().clone(); - let info = QueryInfo::Bootstrap { peer: local_key.preimage().clone() }; + let info = QueryInfo::Bootstrap { + peer: local_key.preimage().clone(), + remaining: None + }; let peers = self.kbuckets.closest_keys(&local_key).collect::>(); - // TODO: Emit error if `peers` is empty? BootstrapError::NoPeers? - let inner = QueryInner::new(info); - self.queries.add_iter_closest(local_key, peers, inner); + if peers.is_empty() { + Err(NoKnownPeers()) + } else { + let inner = QueryInner::new(info); + Ok(self.queries.add_iter_closest(local_key, peers, inner)) + } } /// Establishes the local node as a provider of a value for the given key. @@ -489,6 +545,9 @@ where /// identity of the local node to the peers closest to the key, thus establishing /// the local node as a provider. /// + /// Returns `Ok` if a provider record has been stored locally, providing the + /// `QueryId` of the initial query that announces the local node as a provider. + /// /// The publication of the provider records is periodically repeated as per the /// configured interval, to renew the expiry and account for changes to the DHT /// topology. A provider record may be removed from local storage and @@ -501,31 +560,28 @@ where /// of the libp2p Kademlia provider API. /// /// The results of the (repeated) provider announcements sent by this node are - /// delivered in [`AddProviderResult`]. - pub fn start_providing(&mut self, key: record::Key) { + /// reported via [`KademliaEvent::QueryResult{QueryResult::AddProvider}`]. + pub fn start_providing(&mut self, key: record::Key) -> Result { self.print_bucket_table(); // TODO: calculate weight for self? let record = ProviderRecord::new(key.clone(), self.kbuckets.local_key().preimage().clone()); - if let Err(err) = self.store.add_provider(record) { - log::error!("Error on add_provider to local storage {}: {:?}", bs58::encode(key.as_ref()).into_string(), err); - self.queued_events.push_back(NetworkBehaviourAction::GenerateEvent( - KademliaEvent::StartProvidingResult(Err( - AddProviderError::LocalStorageError { key, cause: err } - )) - )); - } else { - let target = kbucket::Key::new(key.clone()); - debug!( - "start_providing for key {} ; kademlia key {}", - bs58::encode(target.preimage().as_ref()).into_string(), // peer id - bs58::encode(target.as_ref()).into_string(), // sha256 - ); - let peers = self.kbuckets.closest_keys(&target); - let context = AddProviderContext::Publish; - let info = QueryInfo::PrepareAddProvider { key, context }; - let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); - } + self.store.add_provider(record)?; + let target = kbucket::Key::new(key.clone()); + debug!( + "start_providing for key {} ; kademlia key {}", + bs58::encode(target.preimage().as_ref()).into_string(), // peer id + bs58::encode(target.as_ref()).into_string(), // sha256 + ); + let peers = self.kbuckets.closest_keys(&target); + let context = AddProviderContext::Publish; + let info = QueryInfo::AddProvider { + context, + key, + phase: AddProviderPhase::GetClosestPeers + }; + let inner = QueryInner::new(info); + let id = self.queries.add_iter_closest(target.clone(), peers, inner); + Ok(id) } /// Stops the local node from announcing that it is a provider for the given key. @@ -544,8 +600,9 @@ where /// Performs a lookup for providers of a value to the given key. /// - /// The result of this operation is delivered in [`KademliaEvent::GetProvidersResult`]. - pub fn get_providers(&mut self, key: record::Key) { + /// The result of this operation is delivered in a + /// reported via [`KademliaEvent::QueryResult{QueryResult::GetProviders}`]. + pub fn get_providers(&mut self, key: record::Key) -> QueryId { self.print_bucket_table(); let info = QueryInfo::GetProviders { key: key.clone(), @@ -559,7 +616,7 @@ where ); let peers = self.kbuckets.closest_keys(&target); let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); + self.queries.add_iter_closest(target.clone(), peers, inner) } /// Processes discovered peers from a successful request in an iterative `Query`. @@ -636,7 +693,11 @@ where /// Starts an iterative `ADD_PROVIDER` query for the given key. fn start_add_provider(&mut self, key: record::Key, context: AddProviderContext) { - let info = QueryInfo::PrepareAddProvider { key: key.clone(), context }; + let info = QueryInfo::AddProvider { + context, + key: key.clone(), + phase: AddProviderPhase::GetClosestPeers + }; let target = kbucket::Key::new(key); let peers = self.kbuckets.closest_keys(&target); let inner = QueryInner::new(info); @@ -648,7 +709,9 @@ where let quorum = quorum.eval(self.queries.config().replication_factor); let target = kbucket::Key::new(record.key.clone()); let peers = self.kbuckets.closest_keys(&target); - let info = QueryInfo::PreparePutRecord { record, quorum, context }; + let info = QueryInfo::PutRecord { + record, quorum, context, phase: PutRecordPhase::GetClosestPeers + }; let inner = QueryInner::new(info); self.queries.add_iter_closest(target.clone(), peers, inner); } @@ -755,19 +818,22 @@ where fn query_finished(&mut self, q: Query, params: &mut impl PollParameters) -> Option { + let query_id = q.id(); + log::trace!("Query {:?} finished.", query_id); let result = q.into_result(); log::info!("Query {} finished", format!("{:#?}", result.inner.info).lines().take(1).next().unwrap()); match result.inner.info { - QueryInfo::Bootstrap { peer } => { + QueryInfo::Bootstrap { peer, remaining } => { self.print_bucket_table(); let local_key = self.kbuckets.local_key().clone(); - if &peer == local_key.preimage() { + let mut remaining = remaining.unwrap_or_else(|| { + debug_assert_eq!(&peer, local_key.preimage()); // The lookup for the local key finished. To complete the bootstrap process, // a bucket refresh should be performed for every bucket farther away than // the first non-empty bucket (which are most likely no more than the last // few, i.e. farthest, buckets). - let targets = self.kbuckets.buckets() + self.kbuckets.buckets() .skip_while(|b| b.num_entries() == 0) .skip(1) // Skip the bucket with the closest neighbour. .map(|b| { @@ -793,77 +859,117 @@ where target = kbucket::Key::new(PeerId::random()); } target - }).collect::>(); + }).collect::>().into_iter() + }); - for target in targets { - let info = QueryInfo::Bootstrap { peer: target.clone().into_preimage() }; - let peers = self.kbuckets.closest_keys(&target); - let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); - } + let num_remaining = remaining.len().saturating_sub(1) as u32; + + if let Some(target) = remaining.next() { + let info = QueryInfo::Bootstrap { + peer: target.clone().into_preimage(), + remaining: Some(remaining) + }; + let peers = self.kbuckets.closest_keys(&target); + let inner = QueryInner::new(info); + self.queries.continue_iter_closest(query_id, target.clone(), peers, inner); } - Some(KademliaEvent::BootstrapResult(Ok(BootstrapOk { peer }))) + + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::Bootstrap(Ok(BootstrapOk { peer, num_remaining })) + }) } QueryInfo::GetClosestPeers { key, .. } => { - Some(KademliaEvent::GetClosestPeersResult(Ok( - GetClosestPeersOk { key, peers: result.peers.collect() } - ))) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetClosestPeers(Ok( + GetClosestPeersOk { key, peers: result.peers.collect() } + )) + }) } QueryInfo::GetProviders { key, providers } => { - Some(KademliaEvent::GetProvidersResult(Ok( - GetProvidersOk { - key, - providers, - closest_peers: result.peers.collect() - } - ))) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetProviders(Ok( + GetProvidersOk { + key, + providers, + closest_peers: result.peers.collect() + } + )) + }) } - QueryInfo::PrepareAddProvider { key, context } => { - let closest_peers = result.peers.map(kbucket::Key::from); + QueryInfo::AddProvider { + context, + key, + phase: AddProviderPhase::GetClosestPeers + } => { let provider_id = params.local_peer_id().clone(); let external_addresses = params.external_addresses().collect(); let provider_key = self.kbuckets.local_public_key(); let inner = QueryInner::new(QueryInfo::AddProvider { - key, - provider_id, - external_addresses, context, + key, + phase: AddProviderPhase::AddProvider { + provider_id, + external_addresses, + get_closest_peers_stats: result.stats + } provider_key }); - self.queries.add_fixed(closest_peers, inner); + self.queries.continue_fixed(query_id, result.peers, inner); None } - QueryInfo::AddProvider { key, context, .. } => { + QueryInfo::AddProvider { + context, + key, + phase: AddProviderPhase::AddProvider { get_closest_peers_stats, .. } + } => { log::info!("AddProvider finished {:?}!", context); match context { AddProviderContext::Publish => { - Some(KademliaEvent::StartProvidingResult(Ok( - AddProviderOk { key } - ))) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: get_closest_peers_stats.merge(result.stats), + result: QueryResult::StartProviding(Ok(AddProviderOk { key })) + }) } AddProviderContext::Republish => { - Some(KademliaEvent::RepublishProviderResult(Ok( - AddProviderOk { key } - ))) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: get_closest_peers_stats.merge(result.stats), + result: QueryResult::RepublishProvider(Ok(AddProviderOk { key })) + }) } } } QueryInfo::GetRecord { key, records, quorum, cache_at } => { - let result = if records.len() >= quorum.get() { // [not empty] + let results = if records.len() >= quorum.get() { // [not empty] if let Some(cache_key) = cache_at { // Cache the record at the closest node to the key that // did not return the record. let record = records.first().expect("[not empty]").clone(); let quorum = NonZeroUsize::new(1).expect("1 > 0"); let context = PutRecordContext::Cache; - let info = QueryInfo::PutRecord { record, quorum, context, num_results: 0 }; + let info = QueryInfo::PutRecord { + context, + record, + quorum, + phase: PutRecordPhase::PutRecord { + num_results: 0, + get_closest_peers_stats: QueryStats::empty() + } + }; let inner = QueryInner::new(info); - self.queries.add_fixed(iter::once(cache_key), inner); + self.queries.add_fixed(iter::once(cache_key.into_preimage()), inner); } Ok(GetRecordOk { records }) } else if records.is_empty() { @@ -874,19 +980,40 @@ where } else { Err(GetRecordError::QuorumFailed { key, records, quorum }) }; - Some(KademliaEvent::GetRecordResult(result)) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetRecord(results) + }) } - QueryInfo::PreparePutRecord { record, quorum, context } => { - let closest_peers = result.peers.map(kbucket::Key::from); - let info = QueryInfo::PutRecord { record, quorum, context, num_results: 0 }; + QueryInfo::PutRecord { + context, + record, + quorum, + phase: PutRecordPhase::GetClosestPeers + } => { + let info = QueryInfo::PutRecord { + context, + record, + quorum, + phase: PutRecordPhase::PutRecord { + num_results: 0, + get_closest_peers_stats: result.stats + } + }; let inner = QueryInner::new(info); - self.queries.add_fixed(closest_peers, inner); + self.queries.continue_fixed(query_id, result.peers, inner); None } - QueryInfo::PutRecord { record, quorum, num_results, context } => { - let result = |key: record::Key| { + QueryInfo::PutRecord { + context, + record, + quorum, + phase: PutRecordPhase::PutRecord { num_results, get_closest_peers_stats } + } => { + let mk_result = |key: record::Key| { if num_results >= quorum.get() { Ok(PutRecordOk { key }) } else { @@ -895,9 +1022,17 @@ where }; match context { PutRecordContext::Publish => - Some(KademliaEvent::PutRecordResult(result(record.key))), + Some(KademliaEvent::QueryResult { + id: query_id, + stats: get_closest_peers_stats.merge(result.stats), + result: QueryResult::PutRecord(mk_result(record.key)) + }), PutRecordContext::Republish => - Some(KademliaEvent::RepublishRecordResult(result(record.key))), + Some(KademliaEvent::QueryResult { + id: query_id, + stats: get_closest_peers_stats.merge(result.stats), + result: QueryResult::RepublishRecord(mk_result(record.key)) + }), PutRecordContext::Replicate => { debug!("Record replicated: {:?}", record.key); None @@ -912,99 +1047,138 @@ where } /// Handles a query that timed out. - fn query_timeout(&self, query: Query) -> Option { - log::trace!("Query {:?} timed out.", query.id()); + fn query_timeout(&mut self, query: Query) -> Option { + let query_id = query.id(); + log::trace!("Query {:?} timed out.", query_id); let result = query.into_result(); match result.inner.info { - QueryInfo::Bootstrap { peer } => - Some(KademliaEvent::BootstrapResult(Err( - BootstrapError::Timeout { peer }))), + QueryInfo::Bootstrap { peer, mut remaining } => { + let num_remaining = remaining.as_ref().map(|r| r.len().saturating_sub(1) as u32); - QueryInfo::PrepareAddProvider { key, context } => + if let Some(mut remaining) = remaining.take() { + // Continue with the next bootstrap query if `remaining` is not empty. + if let Some(target) = remaining.next() { + let info = QueryInfo::Bootstrap { + peer: target.clone().into_preimage(), + remaining: Some(remaining) + }; + let peers = self.kbuckets.closest_keys(&target); + let inner = QueryInner::new(info); + self.queries.continue_iter_closest(query_id, target.clone(), peers, inner); + } + } + + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::Bootstrap(Err( + BootstrapError::Timeout { peer, num_remaining } + )) + }) + } + + QueryInfo::AddProvider { context, key, .. } => Some(match context { AddProviderContext::Publish => - KademliaEvent::StartProvidingResult(Err( - AddProviderError::Timeout { key })), + KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::StartProviding(Err( + AddProviderError::Timeout { key } + )) + }, AddProviderContext::Republish => - KademliaEvent::RepublishProviderResult(Err( - AddProviderError::Timeout { key })), - }), - - QueryInfo::AddProvider { key, context, .. } => - Some(match context { - AddProviderContext::Publish => - KademliaEvent::StartProvidingResult(Err( - AddProviderError::Timeout { key })), - AddProviderContext::Republish => - KademliaEvent::RepublishProviderResult(Err( - AddProviderError::Timeout { key })), + KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::RepublishProvider(Err( + AddProviderError::Timeout { key } + )) + } }), QueryInfo::GetClosestPeers { key } => { - Some(KademliaEvent::GetClosestPeersResult(Err( - GetClosestPeersError::Timeout { - key, - peers: result.peers.collect() - }))) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetClosestPeers(Err( + GetClosestPeersError::Timeout { + key, + peers: result.peers.collect() + } + )) + }) }, - QueryInfo::PreparePutRecord { record, quorum, context, .. } => { + QueryInfo::PutRecord { record, quorum, context, phase } => { let err = Err(PutRecordError::Timeout { key: record.key, - num_results: 0, - quorum + quorum, + num_results: match phase { + PutRecordPhase::GetClosestPeers => 0, + PutRecordPhase::PutRecord { num_results, .. } => num_results + } }); match context { PutRecordContext::Publish => - Some(KademliaEvent::PutRecordResult(err)), + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::PutRecord(err) + }), PutRecordContext::Republish => - Some(KademliaEvent::RepublishRecordResult(err)), - PutRecordContext::Replicate => { - warn!("Locating closest peers for replication failed: {:?}", err); - None + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::RepublishRecord(err) + }), + PutRecordContext::Replicate => match phase { + PutRecordPhase::GetClosestPeers => { + warn!("Locating closest peers for replication failed: {:?}", err); + None + } + PutRecordPhase::PutRecord { .. } => { + debug!("Replicating record failed: {:?}", err); + None + } } - PutRecordContext::Cache => - // Caching a record at the closest peer to a key that did not return - // a record is never preceded by a lookup for the closest peers, i.e. - // it is a direct query to a single peer. - unreachable!() - } - } - - QueryInfo::PutRecord { record, quorum, num_results, context } => { - let err = Err(PutRecordError::Timeout { - key: record.key, - num_results, - quorum - }); - match context { - PutRecordContext::Publish => - Some(KademliaEvent::PutRecordResult(err)), - PutRecordContext::Republish => - Some(KademliaEvent::RepublishRecordResult(err)), - PutRecordContext::Replicate => { - debug!("Replicatiing record failed: {:?}", err); - None - } - PutRecordContext::Cache => { - debug!("Caching record failed: {:?}", err); - None + PutRecordContext::Cache => match phase { + PutRecordPhase::GetClosestPeers => { + // Caching a record at the closest peer to a key that did not return + // a record is never preceded by a lookup for the closest peers, i.e. + // it is a direct query to a single peer. + unreachable!() + } + PutRecordPhase::PutRecord { .. } => { + debug!("Caching record failed: {:?}", err); + None + } } } } QueryInfo::GetRecord { key, records, quorum, .. } => - Some(KademliaEvent::GetRecordResult(Err( - GetRecordError::Timeout { key, records, quorum }))), + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetRecord(Err( + GetRecordError::Timeout { key, records, quorum } + )) + }), QueryInfo::GetProviders { key, providers } => - Some(KademliaEvent::GetProvidersResult(Err( - GetProvidersError::Timeout { - key, - providers, - closest_peers: result.peers.collect() - }))), - } + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetProviders(Err( + GetProvidersError::Timeout { + key, + providers, + closest_peers: result.peers.collect() + } + )) + }) + } } /// Processes a record received from a peer. @@ -1450,7 +1624,7 @@ where } = &mut query.inner.info { if let Some(record) = record { records.push(record); - if records.len() == quorum.get() { + if records.len() >= quorum.get() { query.finish() } } else if quorum.get() == 1 { @@ -1487,10 +1661,10 @@ where if let Some(query) = self.queries.get_mut(&user_data) { query.on_success(&source, vec![]); if let QueryInfo::PutRecord { - num_results, quorum, .. + phase: PutRecordPhase::PutRecord { num_results, .. }, quorum, .. } = &mut query.inner.info { *num_results += 1; - if *num_results == quorum.get() { + if *num_results >= quorum.get() { query.finish() } } @@ -1582,7 +1756,10 @@ where // better emit an event when the request has been sent (and report // an error if sending fails), instead of immediately reporting // "success" somewhat prematurely here. - if let QueryInfo::AddProvider { .. } = &query.inner.info { + if let QueryInfo::AddProvider { + phase: AddProviderPhase::AddProvider { .. }, + .. + } = &query.inner.info { query.on_success(&peer_id, vec![]) } if self.connected_peers.contains(&peer_id) { @@ -1642,29 +1819,15 @@ impl Quorum { #[derive(Derivative)] #[derivative(Debug)] pub enum KademliaEvent { - /// The result of [`Kademlia::bootstrap`]. - BootstrapResult(BootstrapResult), - - /// The result of [`Kademlia::get_closest_peers`]. - GetClosestPeersResult(GetClosestPeersResult), - - /// The result of [`Kademlia::get_providers`]. - GetProvidersResult(GetProvidersResult), - - /// The result of [`Kademlia::start_providing`]. - StartProvidingResult(AddProviderResult), - - /// The result of a (automatic) republishing of a provider record. - RepublishProviderResult(AddProviderResult), - - /// The result of [`Kademlia::get_record`]. - GetRecordResult(GetRecordResult), - - /// The result of [`Kademlia::put_record`]. - PutRecordResult(PutRecordResult), - - /// The result of a (automatic) republishing of a (value-)record. - RepublishRecordResult(PutRecordResult), + /// A query has produced a result. + QueryResult { + /// The ID of the query that finished. + id: QueryId, + /// The result of the query. + result: QueryResult, + /// Execution statistics from the query. + stats: QueryStats + }, /// A peer has been discovered during a query. Discovered { @@ -1700,6 +1863,34 @@ pub enum KademliaEvent { } } +/// The results of Kademlia queries. +#[derive(Debug)] +pub enum QueryResult { + /// The result of [`Kademlia::bootstrap`]. + Bootstrap(BootstrapResult), + + /// The result of [`Kademlia::get_closest_peers`]. + GetClosestPeers(GetClosestPeersResult), + + /// The result of [`Kademlia::get_providers`]. + GetProviders(GetProvidersResult), + + /// The result of [`Kademlia::start_providing`]. + StartProviding(AddProviderResult), + + /// The result of a (automatic) republishing of a provider record. + RepublishProvider(AddProviderResult), + + /// The result of [`Kademlia::get_record`]. + GetRecord(GetRecordResult), + + /// The result of [`Kademlia::put_record`]. + PutRecord(PutRecordResult), + + /// The result of a (automatic) republishing of a (value-)record. + RepublishRecord(PutRecordResult), +} + /// The result of [`Kademlia::get_record`]. pub type GetRecordResult = Result; @@ -1771,10 +1962,6 @@ pub enum PutRecordError { num_results: usize, quorum: NonZeroUsize }, - LocalStorageError { - key: record::Key, - cause: store::Error - } } impl PutRecordError { @@ -1783,7 +1970,6 @@ impl PutRecordError { match self { PutRecordError::QuorumFailed { key, .. } => key, PutRecordError::Timeout { key, .. } => key, - PutRecordError::LocalStorageError { key, .. } => key } } @@ -1793,7 +1979,6 @@ impl PutRecordError { match self { PutRecordError::QuorumFailed { key, .. } => key, PutRecordError::Timeout { key, .. } => key, - PutRecordError::LocalStorageError { key, .. } => key, } } } @@ -1804,13 +1989,17 @@ pub type BootstrapResult = Result; /// The successful result of [`Kademlia::bootstrap`]. #[derive(Debug, Clone)] pub struct BootstrapOk { - pub peer: PeerId + pub peer: PeerId, + pub num_remaining: u32, } /// The error result of [`Kademlia::bootstrap`]. #[derive(Debug, Clone)] pub enum BootstrapError { - Timeout { peer: PeerId } + Timeout { + peer: PeerId, + num_remaining: Option, + } } /// The result of [`Kademlia::get_closest_peers`]. @@ -1903,11 +2092,6 @@ pub enum AddProviderError { Timeout { key: record::Key, }, - /// The provider record could not be stored. - LocalStorageError { - key: record::Key, - cause: store::Error - } } impl AddProviderError { @@ -1915,16 +2099,13 @@ impl AddProviderError { pub fn key(&self) -> &record::Key { match self { AddProviderError::Timeout { key, .. } => key, - AddProviderError::LocalStorageError { key, .. } => key, } } /// Extracts the key for which the operation failed, - /// consuming the error. pub fn into_key(self) -> record::Key { match self { AddProviderError::Timeout { key, .. } => key, - AddProviderError::LocalStorageError { key, .. } => key, } } } @@ -1984,33 +2165,42 @@ impl QueryInner { } } +/// The context of a [`QueryInfo::AddProvider`] query. #[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum AddProviderContext { +pub enum AddProviderContext { Publish, Republish, } +/// The context of a [`QueryInfo::PutRecord`] query. #[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum PutRecordContext { +pub enum PutRecordContext { Publish, Republish, Replicate, Cache, } -/// The internal query state. -#[derive(Debug, Clone, PartialEq, Eq)] -enum QueryInfo { - /// A bootstrapping query. +/// Information about a running query. +#[derive(Debug, Clone)] +pub enum QueryInfo { + /// A query initiated by [`Kademlia::bootstrap`]. Bootstrap { /// The targeted peer ID. peer: PeerId, + /// The remaining random peer IDs to query, one per + /// bucket that still needs refreshing. + /// + /// This is `None` if the initial self-lookup has not + /// yet completed and `Some` with an exhausted iterator + /// if bootstrapping is complete. + remaining: Option>> }, - /// A query to find the closest peers to a key. + /// A query initiated by [`Kademlia::get_closest_peers`]. GetClosestPeers { key: Vec }, - /// A query for the providers of a key. + /// A query initiated by [`Kademlia::get_providers`]. GetProviders { /// The key for which to search for providers. key: record::Key, @@ -2018,50 +2208,40 @@ enum QueryInfo { providers: HashSet, }, - /// A query that searches for the closest closest nodes to a key to be - /// used in a subsequent `AddProvider` query. - PrepareAddProvider { - key: record::Key, - context: AddProviderContext, - }, - - /// A query that advertises the local node as a provider for a key. + /// A (repeated) query initiated by [`Kademlia::start_providing`]. AddProvider { + /// The record key. key: record::Key, - provider_id: PeerId, - external_addresses: Vec, + /// The current phase of the query. + phase: AddProviderPhase, + /// The execution context of the query. context: AddProviderContext, provider_key: PublicKey }, - /// A query that searches for the closest closest nodes to a key to be used - /// in a subsequent `PutValue` query. - PreparePutRecord { - record: Record, - quorum: NonZeroUsize, - context: PutRecordContext, - }, - - /// A query that replicates a record to other nodes. + /// A (repeated) query initiated by [`Kademlia::put_record`]. PutRecord { record: Record, + /// The expected quorum of responses w.r.t. the replication factor. quorum: NonZeroUsize, - num_results: usize, + /// The current phase of the query. + phase: PutRecordPhase, + /// The execution context of the query. context: PutRecordContext, }, - /// A query that searches for values for a key. + /// A query initiated by [`Kademlia::get_record`]. GetRecord { /// The key to look for. key: record::Key, - /// The records found. + /// The records found so far. records: Vec, /// The number of records to look for. quorum: NonZeroUsize, /// The closest peer to `key` that did not return a record. /// /// When a record is found in a standard Kademlia query (quorum == 1), - /// it is cached at this peer. + /// it is cached at this peer as soon as a record is found. cache_at: Option>, }, } @@ -2071,7 +2251,7 @@ impl QueryInfo { /// context of a query. fn to_request(&self, query_id: QueryId) -> KademliaHandlerIn { match &self { - QueryInfo::Bootstrap { peer } => KademliaHandlerIn::FindNodeReq { + QueryInfo::Bootstrap { peer, .. } => KademliaHandlerIn::FindNodeReq { key: peer.clone().into_bytes(), user_data: query_id, }, @@ -2083,37 +2263,140 @@ impl QueryInfo { key: key.clone(), user_data: query_id, }, - QueryInfo::PrepareAddProvider { key, .. } => KademliaHandlerIn::FindNodeReq { - key: key.to_vec(), - user_data: query_id, - }, - QueryInfo::AddProvider { - key, + QueryInfo::AddProvider { key, phase, .. } => match phase { + AddProviderPhase::GetClosestPeers => KademliaHandlerIn::FindNodeReq { + key: key.to_vec(), + user_data: query_id, + }, + AddProviderPhase::AddProvider { provider_id, external_addresses, provider_key, - .. - } => KademliaHandlerIn::AddProvider { + ..} => { + KademliaHandlerIn::AddProvider { key: key.clone(), provider: crate::protocol::KadPeer { public_key: provider_key.clone(), - node_id: provider_id.clone(), - multiaddrs: external_addresses.clone(), - connection_ty: crate::protocol::KadConnectionType::Connected, + node_id: provider_id.clone(), + multiaddrs: external_addresses.clone(), + connection_ty: crate::protocol::KadConnectionType::Connected, + } + } } }, QueryInfo::GetRecord { key, .. } => KademliaHandlerIn::GetRecord { key: key.clone(), user_data: query_id, }, - QueryInfo::PreparePutRecord { record, .. } => KademliaHandlerIn::FindNodeReq { - key: record.key.to_vec(), - user_data: query_id, - }, - QueryInfo::PutRecord { record, .. } => KademliaHandlerIn::PutRecord { - record: record.clone(), - user_data: query_id + QueryInfo::PutRecord { record, phase, .. } => match phase { + PutRecordPhase::GetClosestPeers => KademliaHandlerIn::FindNodeReq { + key: record.key.to_vec(), + user_data: query_id, + }, + PutRecordPhase::PutRecord { .. } => KademliaHandlerIn::PutRecord { + record: record.clone(), + user_data: query_id + } } } } } + +/// The phases of a [`QueryInfo::AddProvider`] query. +#[derive(Debug, Clone)] +pub enum AddProviderPhase { + /// The query is searching for the closest nodes to the record key. + GetClosestPeers, + + /// The query advertises the local node as a provider for the key to + /// the closest nodes to the key. + AddProvider { + /// The local peer ID that is advertised as a provider. + provider_id: PeerId, + /// The external addresses of the provider being advertised. + external_addresses: Vec, + /// Query statistics from the finished `GetClosestPeers` phase. + get_closest_peers_stats: QueryStats, + }, +} + +/// The phases of a [`QueryInfo::PutRecord`] query. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PutRecordPhase { + /// The query is searching for the closest nodes to the record key. + GetClosestPeers, + + /// The query is replicating the record to the closest nodes to the key. + PutRecord { + /// The number of successful replication requests so far. + num_results: usize, + /// Query statistics from the finished `GetClosestPeers` phase. + get_closest_peers_stats: QueryStats, + }, +} + +/// A mutable reference to a running query. +pub struct QueryMut<'a> { + query: &'a mut Query, +} + +impl<'a> QueryMut<'a> { + pub fn id(&self) -> QueryId { + self.query.id() + } + + /// Gets information about the type and state of the query. + pub fn info(&self) -> &QueryInfo { + &self.query.inner.info + } + + /// Gets execution statistics about the query. + /// + /// For a multi-phase query such as `put_record`, these are the + /// statistics of the current phase. + pub fn stats(&self) -> &QueryStats { + self.query.stats() + } + + /// Finishes the query asap, without waiting for the + /// regular termination conditions. + pub fn finish(&mut self) { + self.query.finish() + } +} + +/// An immutable reference to a running query. +pub struct QueryRef<'a> { + query: &'a Query, +} + +impl<'a> QueryRef<'a> { + pub fn id(&self) -> QueryId { + self.query.id() + } + + /// Gets information about the type and state of the query. + pub fn info(&self) -> &QueryInfo { + &self.query.inner.info + } + + /// Gets execution statistics about the query. + /// + /// For a multi-phase query such as `put_record`, these are the + /// statistics of the current phase. + pub fn stats(&self) -> &QueryStats { + self.query.stats() + } +} + +/// An operation failed to due no known peers in the routing table. +#[derive(Debug, Clone)] +pub struct NoKnownPeers(); + +impl fmt::Display for NoKnownPeers { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "No known peers.") + } +} + +impl std::error::Error for NoKnownPeers {} diff --git a/protocols/kad/src/behaviour/test.rs b/protocols/kad/src/behaviour/test.rs index 55ede105..e84e705c 100644 --- a/protocols/kad/src/behaviour/test.rs +++ b/protocols/kad/src/behaviour/test.rs @@ -152,10 +152,11 @@ fn bootstrap() { .collect::>(); let swarm_ids: Vec<_> = swarms.iter().map(Swarm::local_peer_id).cloned().collect(); - swarms[0].bootstrap(); + let qid = swarms[0].bootstrap().unwrap(); // Expected known peers let expected_known = swarm_ids.iter().skip(1).cloned().collect::>(); + let mut first = true; // Run test block_on( @@ -163,14 +164,23 @@ fn bootstrap() { for (i, swarm) in swarms.iter_mut().enumerate() { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::BootstrapResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::Bootstrap(Ok(ok)), .. + })) => { + assert_eq!(id, qid); assert_eq!(i, 0); - assert_eq!(ok.peer, swarm_ids[0]); - let known = swarm.kbuckets.iter() - .map(|e| e.node.key.preimage().clone()) - .collect::>(); - assert_eq!(expected_known, known); - return Poll::Ready(()) + if first { + // Bootstrapping must start with a self-lookup. + assert_eq!(ok.peer, swarm_ids[0]); + } + first = false; + if ok.num_remaining == 0 { + let known = swarm.kbuckets.iter() + .map(|e| e.node.key.preimage().clone()) + .collect::>(); + assert_eq!(expected_known, known); + return Poll::Ready(()) + } } // Ignore any other event. Poll::Ready(Some(_)) => (), @@ -210,7 +220,17 @@ fn query_iter() { // propagate forwards through the list of peers. let search_target = PeerId::random(); let search_target_key = kbucket::Key::new(search_target.clone()); - swarms[0].get_closest_peers(search_target.clone()); + let qid = swarms[0].get_closest_peers(search_target.clone()); + + match swarms[0].query(&qid) { + Some(q) => match q.info() { + QueryInfo::GetClosestPeers { key } => { + assert_eq!(&key[..], search_target.borrow() as &[u8]) + }, + i => panic!("Unexpected query info: {:?}", i) + } + None => panic!("Query not found: {:?}", qid) + } // Set up expectations. let expected_swarm_id = swarm_ids[0].clone(); @@ -224,7 +244,10 @@ fn query_iter() { for (i, swarm) in swarms.iter_mut().enumerate() { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetClosestPeersResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::GetClosestPeers(Ok(ok)), .. + })) => { + assert_eq!(id, qid); assert_eq!(&ok.key[..], search_target.as_bytes()); assert_eq!(swarm_ids[i], expected_swarm_id); assert_eq!(swarm.queries.size(), 0); @@ -275,7 +298,9 @@ fn unresponsive_not_returned_direct() { for (_, swarm) in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetClosestPeersResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + result: QueryResult::GetClosestPeers(Ok(ok)), .. + })) => { assert_eq!(&ok.key[..], search_target.as_bytes()); assert_eq!(ok.peers.len(), 0); return Poll::Ready(()); @@ -325,7 +350,9 @@ fn unresponsive_not_returned_indirect() { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetClosestPeersResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + result: QueryResult::GetClosestPeers(Ok(ok)), .. + })) => { assert_eq!(&ok.key[..], search_target.as_bytes()); assert_eq!(ok.peers.len(), 1); assert_eq!(ok.peers[0], first_peer_id); @@ -363,14 +390,17 @@ fn get_record_not_found() { let mut swarms = swarms.into_iter().map(|(_, _addr, swarm)| swarm).collect::>(); let target_key = record::Key::from(random_multihash()); - swarms[0].get_record(&target_key, Quorum::One); + let qid = swarms[0].get_record(&target_key, Quorum::One); block_on( poll_fn(move |ctx| { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetRecordResult(Err(e)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::GetRecord(Err(e)), .. + })) => { + assert_eq!(id, qid); if let GetRecordError::NotFound { key, closest_peers, } = e { assert_eq!(key, target_key); assert_eq!(closest_peers.len(), 2); @@ -436,8 +466,23 @@ fn put_record() { }) .collect::>(); + // Initiate put_record queries. + let mut qids = HashSet::new(); for r in records.values() { - swarms[0].put_record(r.clone(), Quorum::All); + let qid = swarms[0].put_record(r.clone(), Quorum::All).unwrap(); + match swarms[0].query(&qid) { + Some(q) => match q.info() { + QueryInfo::PutRecord { phase, record, .. } => { + assert_eq!(phase, &PutRecordPhase::GetClosestPeers); + assert_eq!(record.key, r.key); + assert_eq!(record.value, r.value); + assert!(record.expires.is_some()); + qids.insert(qid); + }, + i => panic!("Unexpected query info: {:?}", i) + } + None => panic!("Query not found: {:?}", qid) + } } // Each test run republishes all records once. @@ -451,8 +496,17 @@ fn put_record() { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::PutRecordResult(res))) | - Poll::Ready(Some(KademliaEvent::RepublishRecordResult(res))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::PutRecord(res), stats + })) | + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::RepublishRecord(res), stats + })) => { + assert!(qids.is_empty() || qids.remove(&id)); + assert!(stats.duration().is_some()); + assert!(stats.num_successes() >= replication_factor.get() as u32); + assert!(stats.num_requests() >= stats.num_successes()); + assert_eq!(stats.num_failures(), 0); match res { Err(e) => panic!("{:?}", e), Ok(ok) => { @@ -551,7 +605,7 @@ fn put_record() { } #[test] -fn get_value() { +fn get_record() { let mut swarms = build_nodes(3); // Let first peer know of second peer and second peer know of third peer. @@ -566,14 +620,17 @@ fn get_value() { let record = Record::new(random_multihash(), vec![4,5,6]); swarms[1].1.store.put(record.clone()).unwrap(); - swarms[0].1.get_record(&record.key, Quorum::One); + let qid = swarms[0].1.get_record(&record.key, Quorum::One); block_on( poll_fn(move |ctx| { for (_, swarm) in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetRecordResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::GetRecord(Ok(ok)), .. + })) => { + assert_eq!(id, qid); assert_eq!(ok.records.len(), 1); assert_eq!(ok.records.first(), Some(&record)); return Poll::Ready(()); @@ -592,7 +649,7 @@ fn get_value() { } #[test] -fn get_value_many() { +fn get_record_many() { // TODO: Randomise let num_nodes = 12; let mut swarms = build_connected_nodes(num_nodes, 3).into_iter() @@ -607,14 +664,17 @@ fn get_value_many() { } let quorum = Quorum::N(NonZeroUsize::new(num_results).unwrap()); - swarms[0].get_record(&record.key, quorum); + let qid = swarms[0].get_record(&record.key, quorum); block_on( poll_fn(move |ctx| { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetRecordResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::GetRecord(Ok(ok)), .. + })) => { + assert_eq!(id, qid); assert_eq!(ok.records.len(), num_results); assert_eq!(ok.records.first(), Some(&record)); return Poll::Ready(()); @@ -672,8 +732,10 @@ fn add_provider() { let mut results = Vec::new(); // Initiate the first round of publishing. + let mut qids = HashSet::new(); for k in &keys { - swarms[0].start_providing(k.clone()); + let qid = swarms[0].start_providing(k.clone()).unwrap(); + qids.insert(qid); } block_on( @@ -682,8 +744,13 @@ fn add_provider() { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::StartProvidingResult(res))) | - Poll::Ready(Some(KademliaEvent::RepublishProviderResult(res))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::StartProviding(res), .. + })) | + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::RepublishProvider(res), .. + })) => { + assert!(qids.is_empty() || qids.remove(&id)); match res { Err(e) => panic!(e), Ok(ok) => { @@ -784,7 +851,7 @@ fn exceed_jobs_max_queries() { let (_, _addr, mut swarm) = build_node(); let num = JOBS_MAX_QUERIES + 1; for _ in 0 .. num { - swarm.bootstrap(); + swarm.get_closest_peers(PeerId::random()); } assert_eq!(swarm.queries.size(), num); @@ -794,8 +861,10 @@ fn exceed_jobs_max_queries() { for _ in 0 .. num { // There are no other nodes, so the queries finish instantly. if let Poll::Ready(Some(e)) = swarm.poll_next_unpin(ctx) { - if let KademliaEvent::BootstrapResult(r) = e { - assert!(r.is_ok(), "Unexpected error") + if let KademliaEvent::QueryResult { + result: QueryResult::GetClosestPeers(Ok(r)), .. + } = e { + assert!(r.peers.is_empty()) } else { panic!("Unexpected event: {:?}", e) } diff --git a/protocols/kad/src/lib.rs b/protocols/kad/src/lib.rs index 603c345f..09139223 100644 --- a/protocols/kad/src/lib.rs +++ b/protocols/kad/src/lib.rs @@ -41,10 +41,33 @@ mod dht_proto { pub use addresses::Addresses; pub use behaviour::{ - AddProviderError, AddProviderOk, AddProviderResult, BootstrapError, BootstrapOk, - BootstrapResult, GetClosestPeersError, GetClosestPeersOk, GetClosestPeersResult, - GetProvidersError, GetProvidersOk, GetProvidersResult, GetRecordError, GetRecordOk, - GetRecordResult, PutRecordError, PutRecordOk, PutRecordResult, + QueryResult, + QueryInfo, + QueryStats, + + BootstrapResult, + BootstrapOk, + BootstrapError, + + GetRecordResult, + GetRecordOk, + GetRecordError, + + PutRecordResult, + PutRecordOk, + PutRecordError, + + GetClosestPeersResult, + GetClosestPeersOk, + GetClosestPeersError, + + AddProviderResult, + AddProviderOk, + AddProviderError, + + GetProvidersResult, + GetProvidersOk, + GetProvidersError, }; pub use behaviour::{Kademlia, KademliaConfig, KademliaEvent, Quorum}; pub use protocol::KadConnectionType; diff --git a/protocols/kad/src/query.rs b/protocols/kad/src/query.rs index f00879a1..706ca622 100644 --- a/protocols/kad/src/query.rs +++ b/protocols/kad/src/query.rs @@ -89,16 +89,40 @@ impl QueryPool { /// Adds a query to the pool that contacts a fixed set of peers. pub fn add_fixed(&mut self, peers: I, inner: TInner) -> QueryId where - I: IntoIterator> + I: IntoIterator { - let peers = peers.into_iter().map(|k| k.into_preimage()).collect::>(); + let id = self.next_query_id(); + self.continue_fixed(id, peers, inner); + id + } + + /// Continues an earlier query with a fixed set of peers, reusing + /// the given query ID, which must be from a query that finished + /// earlier. + pub fn continue_fixed(&mut self, id: QueryId, peers: I, inner: TInner) + where + I: IntoIterator + { + assert!(!self.queries.contains_key(&id)); let parallelism = self.config.replication_factor.get(); let peer_iter = QueryPeerIter::Fixed(FixedPeersIter::new(peers, parallelism)); - self.add(peer_iter, inner) + let query = Query::new(id, peer_iter, inner); + self.queries.insert(id, query); } /// Adds a query to the pool that iterates towards the closest peers to the target. pub fn add_iter_closest(&mut self, target: T, peers: I, inner: TInner) -> QueryId + where + T: Into, + I: IntoIterator> + { + let id = self.next_query_id(); + self.continue_iter_closest(id, target, peers, inner); + id + } + + /// Adds a query to the pool that iterates towards the closest peers to the target. + pub fn continue_iter_closest(&mut self, id: QueryId, target: T, peers: I, inner: TInner) where T: Into, I: IntoIterator> @@ -108,14 +132,13 @@ impl QueryPool { .. ClosestPeersIterConfig::default() }; let peer_iter = QueryPeerIter::Closest(ClosestPeersIter::with_config(cfg, target, peers)); - self.add(peer_iter, inner) - } - - fn add(&mut self, peer_iter: QueryPeerIter, inner: TInner) -> QueryId { - let id = QueryId(self.next_id); - self.next_id = self.next_id.wrapping_add(1); let query = Query::new(id, peer_iter, inner); self.queries.insert(id, query); + } + + fn next_query_id(&mut self) -> QueryId { + let id = QueryId(self.next_id); + self.next_id = self.next_id.wrapping_add(1); id } @@ -136,7 +159,7 @@ impl QueryPool { let mut waiting = None; for (&query_id, query) in self.queries.iter_mut() { - query.started = query.started.or(Some(now)); + query.stats.start = query.stats.start.or(Some(now)); match query.next(now) { PeersIterState::Finished => { finished = Some(query_id); @@ -148,7 +171,7 @@ impl QueryPool { break } PeersIterState::Waiting(None) | PeersIterState::WaitingAtCapacity => { - let elapsed = now - query.started.unwrap_or(now); + let elapsed = now - query.stats.start.unwrap_or(now); if elapsed >= self.config.timeout { timeout = Some(query_id); break @@ -163,12 +186,14 @@ impl QueryPool { } if let Some(query_id) = finished { - let query = self.queries.remove(&query_id).expect("s.a."); + let mut query = self.queries.remove(&query_id).expect("s.a."); + query.stats.end = Some(now); return QueryPoolState::Finished(query) } if let Some(query_id) = timeout { - let query = self.queries.remove(&query_id).expect("s.a."); + let mut query = self.queries.remove(&query_id).expect("s.a."); + query.stats.end = Some(now); return QueryPoolState::Timeout(query) } @@ -206,9 +231,8 @@ pub struct Query { id: QueryId, /// The peer iterator that drives the query state. peer_iter: QueryPeerIter, - /// The instant when the query started (i.e. began waiting for the first - /// result from a peer). - started: Option, + /// Execution statistics of the query. + stats: QueryStats, /// The opaque inner query state. pub inner: TInner, } @@ -222,7 +246,7 @@ enum QueryPeerIter { impl Query { /// Creates a new query without starting it. fn new(id: QueryId, peer_iter: QueryPeerIter, inner: TInner) -> Self { - Query { id, inner, peer_iter, started: None } + Query { id, inner, peer_iter, stats: QueryStats::empty() } } /// Gets the unique ID of the query. @@ -230,11 +254,19 @@ impl Query { self.id } + /// Gets the current execution statistics of the query. + pub fn stats(&self) -> &QueryStats { + &self.stats + } + /// Informs the query that the attempt to contact `peer` failed. pub fn on_failure(&mut self, peer: &PeerId) { - match &mut self.peer_iter { + let updated = match &mut self.peer_iter { QueryPeerIter::Closest(iter) => iter.on_failure(peer), QueryPeerIter::Fixed(iter) => iter.on_failure(peer) + }; + if updated { + self.stats.failure += 1; } } @@ -245,9 +277,12 @@ impl Query { where I: IntoIterator { - match &mut self.peer_iter { + let updated = match &mut self.peer_iter { QueryPeerIter::Closest(iter) => iter.on_success(peer, new_peers), QueryPeerIter::Fixed(iter) => iter.on_success(peer) + }; + if updated { + self.stats.success += 1; } } @@ -261,10 +296,16 @@ impl Query { /// Advances the state of the underlying peer iterator. fn next(&mut self, now: Instant) -> PeersIterState { - match &mut self.peer_iter { + let state = match &mut self.peer_iter { QueryPeerIter::Closest(iter) => iter.next(now), QueryPeerIter::Fixed(iter) => iter.next() + }; + + if let PeersIterState::Waiting(Some(_)) = state { + self.stats.requests += 1; } + + state } /// Finishes the query prematurely. @@ -278,13 +319,24 @@ impl Query { } } + /// Checks whether the query has finished. + /// + /// A finished query is eventually reported by `QueryPool::next()` and + /// removed from the pool. + pub fn is_finished(&self) -> bool { + match &self.peer_iter { + QueryPeerIter::Closest(iter) => iter.is_finished(), + QueryPeerIter::Fixed(iter) => iter.is_finished() + } + } + /// Consumes the query, producing the final `QueryResult`. pub fn into_result(self) -> QueryResult> { let peers = match self.peer_iter { QueryPeerIter::Closest(iter) => Either::Left(iter.into_result()), QueryPeerIter::Fixed(iter) => Either::Right(iter.into_result()) }; - QueryResult { inner: self.inner, peers } + QueryResult { peers, inner: self.inner, stats: self.stats } } } @@ -293,6 +345,90 @@ pub struct QueryResult { /// The opaque inner query state. pub inner: TInner, /// The successfully contacted peers. - pub peers: TPeers + pub peers: TPeers, + /// The collected query statistics. + pub stats: QueryStats } +/// Execution statistics of a query. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct QueryStats { + requests: u32, + success: u32, + failure: u32, + start: Option, + end: Option +} + +impl QueryStats { + pub fn empty() -> Self { + QueryStats { + requests: 0, + success: 0, + failure: 0, + start: None, + end: None, + } + } + + /// Gets the total number of requests initiated by the query. + pub fn num_requests(&self) -> u32 { + self.requests + } + + /// Gets the number of successful requests. + pub fn num_successes(&self) -> u32 { + self.success + } + + /// Gets the number of failed requests. + pub fn num_failures(&self) -> u32 { + self.failure + } + + /// Gets the number of pending requests. + /// + /// > **Note**: A query can finish while still having pending + /// > requests, if the termination conditions are already met. + pub fn num_pending(&self) -> u32 { + self.requests - (self.success + self.failure) + } + + /// Gets the duration of the query. + /// + /// If the query has not yet finished, the duration is measured from the + /// start of the query to the current instant. + /// + /// If the query did not yet start (i.e. yield the first peer to contact), + /// `None` is returned. + pub fn duration(&self) -> Option { + if let Some(s) = self.start { + if let Some(e) = self.end { + Some(e - s) + } else { + Some(Instant::now() - s) + } + } else { + None + } + } + + /// Merges these stats with the given stats of another query, + /// e.g. to accumulate statistics from a multi-phase query. + /// + /// Counters are merged cumulatively while the instants for + /// start and end of the queries are taken as the minimum and + /// maximum, respectively. + pub fn merge(self, other: QueryStats) -> Self { + QueryStats { + requests: self.requests + other.requests, + success: self.success + other.success, + failure: self.failure + other.failure, + start: match (self.start, other.start) { + (Some(a), Some(b)) => Some(std::cmp::min(a, b)), + (a, b) => a.or(b) + }, + end: std::cmp::max(self.end, other.end) + } + } +} diff --git a/protocols/kad/src/query/peers/closest.rs b/protocols/kad/src/query/peers/closest.rs index 7092f7af..e0cb1912 100644 --- a/protocols/kad/src/query/peers/closest.rs +++ b/protocols/kad/src/query/peers/closest.rs @@ -132,8 +132,7 @@ impl ClosestPeersIter { } } - /// Callback for delivering the result of a successful request to a peer - /// that the iterator is waiting on. + /// Callback for delivering the result of a successful request to a peer. /// /// Delivering results of requests back to the iterator allows the iterator to make /// progress. The iterator is said to make progress either when the given @@ -141,18 +140,20 @@ impl ClosestPeersIter { /// or when the iterator did not yet accumulate `num_results` closest peers and /// `closer_peers` contains a new peer, regardless of its distance to the target. /// - /// After calling this function, `next` should eventually be called again - /// to advance the state of the iterator. + /// If the iterator is currently waiting for a result from `peer`, + /// the iterator state is updated and `true` is returned. In that + /// case, after calling this function, `next` should eventually be + /// called again to obtain the new state of the iterator. /// /// If the iterator is finished, it is not currently waiting for a /// result from `peer`, or a result for `peer` has already been reported, - /// calling this function has no effect. - pub fn on_success(&mut self, peer: &PeerId, closer_peers: I) + /// calling this function has no effect and `false` is returned. + pub fn on_success(&mut self, peer: &PeerId, closer_peers: I) -> bool where I: IntoIterator { if let State::Finished = self.state { - return + return false } let key = Key::from(peer.clone()); @@ -160,7 +161,7 @@ impl ClosestPeersIter { // Mark the peer as succeeded. match self.closest_peers.entry(distance) { - Entry::Vacant(..) => return, + Entry::Vacant(..) => return false, Entry::Occupied(mut e) => match e.get().state { PeerState::Waiting(..) => { debug_assert!(self.num_waiting > 0); @@ -172,7 +173,7 @@ impl ClosestPeersIter { } PeerState::NotContacted | PeerState::Failed - | PeerState::Succeeded => return + | PeerState::Succeeded => return false } } @@ -209,28 +210,31 @@ impl ClosestPeersIter { State::Stalled } State::Finished => State::Finished - } + }; + + true } - /// Callback for informing the iterator about a failed request to a peer - /// that the iterator is waiting on. + /// Callback for informing the iterator about a failed request to a peer. /// - /// After calling this function, `next` should eventually be called again - /// to advance the state of the iterator. + /// If the iterator is currently waiting for a result from `peer`, + /// the iterator state is updated and `true` is returned. In that + /// case, after calling this function, `next` should eventually be + /// called again to obtain the new state of the iterator. /// /// If the iterator is finished, it is not currently waiting for a /// result from `peer`, or a result for `peer` has already been reported, - /// calling this function has no effect. - pub fn on_failure(&mut self, peer: &PeerId) { + /// calling this function has no effect and `false` is returned. + pub fn on_failure(&mut self, peer: &PeerId) -> bool { if let State::Finished = self.state { - return + return false } let key = Key::from(peer.clone()); let distance = key.distance(&self.target); match self.closest_peers.entry(distance) { - Entry::Vacant(_) => return, + Entry::Vacant(_) => return false, Entry::Occupied(mut e) => match e.get().state { PeerState::Waiting(_) => { debug_assert!(self.num_waiting > 0); @@ -240,9 +244,13 @@ impl ClosestPeersIter { PeerState::Unresponsive => { e.get_mut().set_state(PeerState::Failed); } - _ => {} + PeerState::NotContacted + | PeerState::Failed + | PeerState::Succeeded => return false } } + + true } /// Returns the list of peers for which the iterator is currently waiting @@ -398,7 +406,7 @@ impl ClosestPeersIter { } /// Checks whether the iterator has finished. - pub fn finished(&self) -> bool { + pub fn is_finished(&self) -> bool { self.state == State::Finished } @@ -523,25 +531,16 @@ mod tests { use multihash::Multihash; use quickcheck::*; - use rand::{Rng, thread_rng}; + use rand::{Rng, rngs::StdRng, SeedableRng}; use libp2p_core::PeerId; use super::*; - fn random_peers(n: usize) -> impl Iterator + Clone { - (0 .. n).map(|_| PeerId::random()) - } - - fn random_iter(g: &mut G) -> ClosestPeersIter { - let known_closest_peers = random_peers(g.gen_range(1, 60)).map(Key::from); - let target = Key::from(Into::::into(PeerId::random())); - let config = ClosestPeersIterConfig { - parallelism: g.gen_range(1, 10), - num_results: g.gen_range(1, 25), - peer_timeout: Duration::from_secs(g.gen_range(10, 30)), - }; - ClosestPeersIter::with_config(config, target, known_closest_peers) + fn random_peers(n: usize, g: &mut R) -> Vec { + (0 .. n).map(|_| PeerId::from_multihash( + multihash::wrap(multihash::Code::Sha2_256, &g.gen::<[u8; 32]>()) + ).unwrap()).collect() } fn sorted>(target: &T, peers: &Vec>) -> bool { @@ -550,42 +549,63 @@ mod tests { impl Arbitrary for ClosestPeersIter { fn arbitrary(g: &mut G) -> ClosestPeersIter { - random_iter(g) + let known_closest_peers = random_peers(g.gen_range(1, 60), g) + .into_iter() + .map(Key::from); + let target = Key::from(Into::::into(PeerId::random())); + let config = ClosestPeersIterConfig { + parallelism: g.gen_range(1, 10), + num_results: g.gen_range(1, 25), + peer_timeout: Duration::from_secs(g.gen_range(10, 30)), + }; + ClosestPeersIter::with_config(config, target, known_closest_peers) + } + } + + #[derive(Clone, Debug)] + struct Seed([u8; 32]); + + impl Arbitrary for Seed { + fn arbitrary(g: &mut G) -> Seed { + Seed(g.gen()) } } #[test] fn new_iter() { - let iter = random_iter(&mut thread_rng()); - let target = iter.target.clone(); + fn prop(iter: ClosestPeersIter) { + let target = iter.target.clone(); - let (keys, states): (Vec<_>, Vec<_>) = iter.closest_peers - .values() - .map(|e| (e.key.clone(), &e.state)) - .unzip(); + let (keys, states): (Vec<_>, Vec<_>) = iter.closest_peers + .values() + .map(|e| (e.key.clone(), &e.state)) + .unzip(); - let none_contacted = states - .iter() - .all(|s| match s { - PeerState::NotContacted => true, - _ => false - }); + let none_contacted = states + .iter() + .all(|s| match s { + PeerState::NotContacted => true, + _ => false + }); - assert!(none_contacted, - "Unexpected peer state in new iterator."); - assert!(sorted(&target, &keys), - "Closest peers in new iterator not sorted by distance to target."); - assert_eq!(iter.num_waiting(), 0, - "Unexpected peers in progress in new iterator."); - assert_eq!(iter.into_result().count(), 0, - "Unexpected closest peers in new iterator"); + assert!(none_contacted, + "Unexpected peer state in new iterator."); + assert!(sorted(&target, &keys), + "Closest peers in new iterator not sorted by distance to target."); + assert_eq!(iter.num_waiting(), 0, + "Unexpected peers in progress in new iterator."); + assert_eq!(iter.into_result().count(), 0, + "Unexpected closest peers in new iterator"); + } + + QuickCheck::new().tests(10).quickcheck(prop as fn(_) -> _) } #[test] fn termination_and_parallelism() { - fn prop(mut iter: ClosestPeersIter) { + fn prop(mut iter: ClosestPeersIter, seed: Seed) { let now = Instant::now(); - let mut rng = thread_rng(); + let mut rng = StdRng::from_seed(seed.0); let mut expected = iter.closest_peers .values() @@ -632,7 +652,7 @@ mod tests { for (i, k) in expected.iter().enumerate() { if rng.gen_bool(0.75) { let num_closer = rng.gen_range(0, iter.config.num_results + 1); - let closer_peers = random_peers(num_closer).collect::>(); + let closer_peers = random_peers(num_closer, &mut rng); remaining.extend(closer_peers.iter().cloned().map(Key::from)); iter.on_success(k.preimage(), closer_peers); } else { @@ -680,14 +700,16 @@ mod tests { } } - QuickCheck::new().tests(10).quickcheck(prop as fn(_) -> _) + QuickCheck::new().tests(10).quickcheck(prop as fn(_, _) -> _) } #[test] fn no_duplicates() { - fn prop(mut iter: ClosestPeersIter) -> bool { + fn prop(mut iter: ClosestPeersIter, seed: Seed) -> bool { let now = Instant::now(); - let closer = random_peers(1).collect::>(); + let mut rng = StdRng::from_seed(seed.0); + + let closer = random_peers(1, &mut rng); // A first peer reports a "closer" peer. let peer1 = match iter.next(now) { @@ -702,7 +724,7 @@ mod tests { match iter.next(now) { PeersIterState::Waiting(Some(p)) => { let peer2 = p.into_owned(); - iter.on_success(&peer2, closer.clone()) + assert!(iter.on_success(&peer2, closer.clone())) } PeersIterState::Finished => {} _ => panic!("Unexpectedly iter state."), @@ -715,7 +737,7 @@ mod tests { true } - QuickCheck::new().tests(10).quickcheck(prop as fn(_) -> _) + QuickCheck::new().tests(10).quickcheck(prop as fn(_, _) -> _) } #[test] @@ -742,7 +764,7 @@ mod tests { Peer { state, .. } => panic!("Unexpected peer state: {:?}", state) } - let finished = iter.finished(); + let finished = iter.is_finished(); iter.on_success(&peer, iter::empty()); let closest = iter.into_result().collect::>(); diff --git a/protocols/kad/src/query/peers/fixed.rs b/protocols/kad/src/query/peers/fixed.rs index 407d5548..edb86ef4 100644 --- a/protocols/kad/src/query/peers/fixed.rs +++ b/protocols/kad/src/query/peers/fixed.rs @@ -39,6 +39,7 @@ pub struct FixedPeersIter { state: State, } +#[derive(Debug, PartialEq, Eq)] enum State { Waiting { num_waiting: usize }, Finished @@ -57,7 +58,12 @@ enum PeerState { } impl FixedPeersIter { - pub fn new(peers: Vec, parallelism: usize) -> Self { + pub fn new(peers: I, parallelism: usize) -> Self + where + I: IntoIterator + { + let peers = peers.into_iter().collect::>(); + Self { parallelism, peers: FnvHashMap::default(), @@ -66,21 +72,46 @@ impl FixedPeersIter { } } - pub fn on_success(&mut self, peer: &PeerId) { + /// Callback for delivering the result of a successful request to a peer. + /// + /// If the iterator is currently waiting for a result from `peer`, + /// the iterator state is updated and `true` is returned. In that + /// case, after calling this function, `next` should eventually be + /// called again to obtain the new state of the iterator. + /// + /// If the iterator is finished, it is not currently waiting for a + /// result from `peer`, or a result for `peer` has already been reported, + /// calling this function has no effect and `false` is returned. + pub fn on_success(&mut self, peer: &PeerId) -> bool { if let State::Waiting { num_waiting } = &mut self.state { if let Some(state @ PeerState::Waiting) = self.peers.get_mut(peer) { *state = PeerState::Succeeded; *num_waiting -= 1; + return true } } + false } - pub fn on_failure(&mut self, peer: &PeerId) { - if let State::Waiting { .. } = &self.state { + /// Callback for informing the iterator about a failed request to a peer. + /// + /// If the iterator is currently waiting for a result from `peer`, + /// the iterator state is updated and `true` is returned. In that + /// case, after calling this function, `next` should eventually be + /// called again to obtain the new state of the iterator. + /// + /// If the iterator is finished, it is not currently waiting for a + /// result from `peer`, or a result for `peer` has already been reported, + /// calling this function has no effect and `false` is returned. + pub fn on_failure(&mut self, peer: &PeerId) -> bool { + if let State::Waiting { num_waiting } = &mut self.state { if let Some(state @ PeerState::Waiting) = self.peers.get_mut(peer) { *state = PeerState::Failed; + *num_waiting -= 1; + return true } } + false } pub fn is_waiting(&self, peer: &PeerId) -> bool { @@ -93,6 +124,11 @@ impl FixedPeersIter { } } + /// Checks whether the iterator has finished. + pub fn is_finished(&self) -> bool { + self.state == State::Finished + } + pub fn next(&mut self) -> PeersIterState { match &mut self.state { State::Finished => return PeersIterState::Finished, @@ -133,3 +169,30 @@ impl FixedPeersIter { } } +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn decrease_num_waiting_on_failure() { + let mut iter = FixedPeersIter::new(vec![PeerId::random(), PeerId::random()], 1); + + match iter.next() { + PeersIterState::Waiting(Some(peer)) => { + let peer = peer.into_owned(); + iter.on_failure(&peer); + }, + _ => panic!("Expected iterator to yield peer."), + } + + match iter.next() { + PeersIterState::Waiting(Some(_)) => {}, + PeersIterState::WaitingAtCapacity => panic!( + "Expected iterator to return another peer given that the \ + previous `on_failure` call should have allowed another peer \ + to be queried.", + ), + _ => panic!("Expected iterator to yield peer."), + } + } +} diff --git a/protocols/mdns/Cargo.toml b/protocols/mdns/Cargo.toml index a0684422..f61572d1 100644 --- a/protocols/mdns/Cargo.toml +++ b/protocols/mdns/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "libp2p-mdns" edition = "2018" -version = "0.18.0" +version = "0.19.0" description = "Implementation of the libp2p mDNS discovery method" authors = ["Parity Technologies "] license = "MIT" @@ -16,8 +16,8 @@ dns-parser = "0.8" either = "1.5.3" futures = "0.3.1" lazy_static = "1.2" -libp2p-core = { version = "0.18.0", path = "../../core" } -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } log = "0.4" net2 = "0.2" rand = "0.7" diff --git a/protocols/noise/Cargo.toml b/protocols/noise/Cargo.toml index 50392188..1f812f1e 100644 --- a/protocols/noise/Cargo.toml +++ b/protocols/noise/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "libp2p-noise" description = "Cryptographic handshake protocol using the noise framework." -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,7 +11,7 @@ edition = "2018" curve25519-dalek = "2.0.0" futures = "0.3.1" lazy_static = "1.2" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4" prost = "0.6.1" rand = "0.7.2" @@ -28,7 +28,7 @@ snow = { version = "0.6.1", features = ["default-resolver"], default-features = [dev-dependencies] env_logger = "0.7.1" -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } quickcheck = "0.9.0" sodiumoxide = "^0.2.5" diff --git a/protocols/noise/src/io/handshake.rs b/protocols/noise/src/io/handshake.rs index a8982354..047951ed 100644 --- a/protocols/noise/src/io/handshake.rs +++ b/protocols/noise/src/io/handshake.rs @@ -365,10 +365,10 @@ where let mut payload_buf = vec![0; len]; state.io.read_exact(&mut payload_buf).await?; - let pb = payload_proto::Identity::decode(&payload_buf[..])?; + let pb = payload_proto::NoiseHandshakePayload::decode(&payload_buf[..])?; - if !pb.pubkey.is_empty() { - let pk = identity::PublicKey::from_protobuf_encoding(&pb.pubkey) + if !pb.identity_key.is_empty() { + let pk = identity::PublicKey::from_protobuf_encoding(&pb.identity_key) .map_err(|_| NoiseError::InvalidKey)?; if let Some(ref k) = state.id_remote_pubkey { if k != &pk { @@ -377,8 +377,8 @@ where } state.id_remote_pubkey = Some(pk); } - if !pb.signature.is_empty() { - state.dh_remote_pubkey_sig = Some(pb.signature); + if !pb.identity_sig.is_empty() { + state.dh_remote_pubkey_sig = Some(pb.identity_sig); } Ok(()) @@ -389,12 +389,12 @@ async fn send_identity(state: &mut State) -> Result<(), NoiseError> where T: AsyncWrite + Unpin, { - let mut pb = payload_proto::Identity::default(); + let mut pb = payload_proto::NoiseHandshakePayload::default(); if state.send_identity { - pb.pubkey = state.identity.public.clone().into_protobuf_encoding() + pb.identity_key = state.identity.public.clone().into_protobuf_encoding() } if let Some(ref sig) = state.identity.signature { - pb.signature = sig.clone() + pb.identity_sig = sig.clone() } let mut buf = Vec::with_capacity(pb.encoded_len()); pb.encode(&mut buf).expect("Vec provides capacity as needed"); diff --git a/protocols/noise/src/io/handshake/payload.proto b/protocols/noise/src/io/handshake/payload.proto index 51b79645..1893dc55 100644 --- a/protocols/noise/src/io/handshake/payload.proto +++ b/protocols/noise/src/io/handshake/payload.proto @@ -4,8 +4,8 @@ package payload.proto; // Payloads for Noise handshake messages. -message Identity { - bytes pubkey = 1; - bytes signature = 2; +message NoiseHandshakePayload { + bytes identity_key = 1; + bytes identity_sig = 2; + bytes data = 3; } - diff --git a/protocols/noise/src/lib.rs b/protocols/noise/src/lib.rs index da9794fc..f73a9e9a 100644 --- a/protocols/noise/src/lib.rs +++ b/protocols/noise/src/lib.rs @@ -27,6 +27,9 @@ //! implementations for various noise handshake patterns (currently `IK`, `IX`, and `XX`) //! over a particular choice of Diffie–Hellman key agreement (currently only X25519). //! +//! > **Note**: Only the `XX` handshake pattern is currently guaranteed to provide +//! > interoperability with other libp2p implementations. +//! //! All upgrades produce as output a pair, consisting of the remote's static public key //! and a `NoiseOutput` which represents the established cryptographic session with the //! remote, implementing `futures::io::AsyncRead` and `futures::io::AsyncWrite`. @@ -38,11 +41,11 @@ //! ``` //! use libp2p_core::{identity, Transport, upgrade}; //! use libp2p_tcp::TcpConfig; -//! use libp2p_noise::{Keypair, X25519, NoiseConfig}; +//! use libp2p_noise::{Keypair, X25519Spec, NoiseConfig}; //! //! # fn main() { //! let id_keys = identity::Keypair::generate_ed25519(); -//! let dh_keys = Keypair::::new().into_authentic(&id_keys).unwrap(); +//! let dh_keys = Keypair::::new().into_authentic(&id_keys).unwrap(); //! let noise = NoiseConfig::xx(dh_keys).into_authenticated(); //! let builder = TcpConfig::new().upgrade(upgrade::Version::V1).authenticate(noise); //! // let transport = builder.multiplex(...); @@ -60,7 +63,8 @@ pub use io::NoiseOutput; pub use io::handshake; pub use io::handshake::{Handshake, RemoteIdentity, IdentityExchange}; pub use protocol::{Keypair, AuthenticKeypair, KeypairIdentity, PublicKey, SecretKey}; -pub use protocol::{Protocol, ProtocolParams, x25519::X25519, IX, IK, XX}; +pub use protocol::{Protocol, ProtocolParams, IX, IK, XX}; +pub use protocol::{x25519::X25519, x25519_spec::X25519Spec}; use futures::prelude::*; use libp2p_core::{identity, PeerId, UpgradeInfo, InboundUpgrade, OutboundUpgrade}; diff --git a/protocols/noise/src/protocol.rs b/protocols/noise/src/protocol.rs index 1fd9fa03..5844dbc8 100644 --- a/protocols/noise/src/protocol.rs +++ b/protocols/noise/src/protocol.rs @@ -21,6 +21,7 @@ //! Components of a Noise protocol. pub mod x25519; +pub mod x25519_spec; use crate::NoiseError; use libp2p_core::identity; @@ -71,6 +72,7 @@ pub trait Protocol { /// /// The trivial case is when the keys are byte for byte identical. #[allow(unused_variables)] + #[deprecated] fn linked(id_pk: &identity::PublicKey, dh_pk: &PublicKey) -> bool { false } @@ -87,6 +89,7 @@ pub trait Protocol { /// without a signature, otherwise a signature over the static DH public key /// must be given and is verified with the public identity key, establishing /// the authenticity of the static DH public key w.r.t. the public identity key. + #[allow(deprecated)] fn verify(id_pk: &identity::PublicKey, dh_pk: &PublicKey, sig: &Option>) -> bool where C: AsRef<[u8]> @@ -95,6 +98,13 @@ pub trait Protocol { || sig.as_ref().map_or(false, |s| id_pk.verify(dh_pk.as_ref(), s)) } + + fn sign(id_keys: &identity::Keypair, dh_pk: &PublicKey) -> Result, NoiseError> + where + C: AsRef<[u8]> + { + Ok(id_keys.sign(dh_pk.as_ref())?) + } } /// DH keypair. @@ -151,9 +161,10 @@ impl Keypair { /// is authentic w.r.t. the given identity keypair, by signing the DH public key. pub fn into_authentic(self, id_keys: &identity::Keypair) -> Result, NoiseError> where - T: AsRef<[u8]> + T: AsRef<[u8]>, + T: Protocol { - let sig = id_keys.sign(self.public.as_ref())?; + let sig = T::sign(id_keys, &self.public)?; let identity = KeypairIdentity { public: id_keys.public(), diff --git a/protocols/noise/src/protocol/x25519.rs b/protocols/noise/src/protocol/x25519.rs index 8a19971d..80bba174 100644 --- a/protocols/noise/src/protocol/x25519.rs +++ b/protocols/noise/src/protocol/x25519.rs @@ -18,7 +18,10 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -//! Noise protocols based on X25519. +//! Legacy Noise protocols based on X25519. +//! +//! **Note**: This set of protocols is not interoperable with other +//! libp2p implementations. use crate::{NoiseConfig, NoiseError, Protocol, ProtocolParams}; use curve25519_dalek::edwards::CompressedEdwardsY; @@ -92,7 +95,11 @@ impl UpgradeInfo for NoiseConfig { } } -/// Noise protocols for X25519. +/// Legacy Noise protocol for X25519. +/// +/// **Note**: This `Protocol` provides no configuration that +/// is interoperable with other libp2p implementations. +/// See [`crate::X25519Spec`] instead. impl Protocol for X25519 { fn params_ik() -> ProtocolParams { PARAMS_IK.clone() diff --git a/protocols/noise/src/protocol/x25519_spec.rs b/protocols/noise/src/protocol/x25519_spec.rs new file mode 100644 index 00000000..446ff7cc --- /dev/null +++ b/protocols/noise/src/protocol/x25519_spec.rs @@ -0,0 +1,150 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//! [libp2p-noise-spec] compliant Noise protocols based on X25519. +//! +//! [libp2p-noise-spec]: https://github.com/libp2p/specs/tree/master/noise + +use crate::{NoiseConfig, NoiseError, Protocol, ProtocolParams}; +use libp2p_core::UpgradeInfo; +use libp2p_core::identity; +use rand::Rng; +use x25519_dalek::{X25519_BASEPOINT_BYTES, x25519}; +use zeroize::Zeroize; + +use super::{*, x25519::X25519}; + +/// Prefix of static key signatures for domain separation. +const STATIC_KEY_DOMAIN: &str = "noise-libp2p-static-key:"; + +/// A X25519 key. +#[derive(Clone)] +pub struct X25519Spec([u8; 32]); + +impl AsRef<[u8]> for X25519Spec { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + +impl Zeroize for X25519Spec { + fn zeroize(&mut self) { + self.0.zeroize() + } +} + +impl Keypair { + /// Create a new X25519 keypair. + pub fn new() -> Keypair { + let mut sk_bytes = [0u8; 32]; + rand::thread_rng().fill(&mut sk_bytes); + let sk = SecretKey(X25519Spec(sk_bytes)); // Copy + sk_bytes.zeroize(); + Self::from(sk) + } +} + +/// Promote a X25519 secret key into a keypair. +impl From> for Keypair { + fn from(secret: SecretKey) -> Keypair { + let public = PublicKey(X25519Spec(x25519((secret.0).0, X25519_BASEPOINT_BYTES))); + Keypair { secret, public } + } +} + +impl UpgradeInfo for NoiseConfig { + type Info = &'static [u8]; + type InfoIter = std::iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + std::iter::once(b"/noise") + } +} + +/// Noise protocols for X25519 with libp2p-spec compliant signatures. +/// +/// **Note**: Only the XX handshake pattern is currently guaranteed to be +/// interoperable with other libp2p implementations. +impl Protocol for X25519Spec { + fn params_ik() -> ProtocolParams { + X25519::params_ik() + } + + fn params_ix() -> ProtocolParams { + X25519::params_ix() + } + + fn params_xx() -> ProtocolParams { + X25519::params_xx() + } + + fn public_from_bytes(bytes: &[u8]) -> Result, NoiseError> { + if bytes.len() != 32 { + return Err(NoiseError::InvalidKey) + } + let mut pk = [0u8; 32]; + pk.copy_from_slice(bytes); + Ok(PublicKey(X25519Spec(pk))) + } + + fn verify(id_pk: &identity::PublicKey, dh_pk: &PublicKey, sig: &Option>) -> bool + { + sig.as_ref().map_or(false, |s| { + id_pk.verify(&[STATIC_KEY_DOMAIN.as_bytes(), dh_pk.as_ref()].concat(), s) + }) + } + + fn sign(id_keys: &identity::Keypair, dh_pk: &PublicKey) -> Result, NoiseError> { + Ok(id_keys.sign(&[STATIC_KEY_DOMAIN.as_bytes(), dh_pk.as_ref()].concat())?) + } +} + +#[doc(hidden)] +impl snow::types::Dh for Keypair { + fn name(&self) -> &'static str { "25519" } + fn pub_len(&self) -> usize { 32 } + fn priv_len(&self) -> usize { 32 } + fn pubkey(&self) -> &[u8] { self.public.as_ref() } + fn privkey(&self) -> &[u8] { self.secret.as_ref() } + + fn set(&mut self, sk: &[u8]) { + let mut secret = [0u8; 32]; + secret.copy_from_slice(&sk[..]); + self.secret = SecretKey(X25519Spec(secret)); // Copy + self.public = PublicKey(X25519Spec(x25519(secret, X25519_BASEPOINT_BYTES))); + secret.zeroize(); + } + + fn generate(&mut self, rng: &mut dyn snow::types::Random) { + let mut secret = [0u8; 32]; + rng.fill_bytes(&mut secret); + self.secret = SecretKey(X25519Spec(secret)); // Copy + self.public = PublicKey(X25519Spec(x25519(secret, X25519_BASEPOINT_BYTES))); + secret.zeroize(); + } + + fn dh(&self, pk: &[u8], shared_secret: &mut [u8]) -> Result<(), ()> { + let mut p = [0; 32]; + p.copy_from_slice(&pk[.. 32]); + let ss = x25519((self.secret.0).0, p); + shared_secret[.. 32].copy_from_slice(&ss[..]); + Ok(()) + } +} diff --git a/protocols/noise/tests/smoke.rs b/protocols/noise/tests/smoke.rs index 1ac04491..744d447a 100644 --- a/protocols/noise/tests/smoke.rs +++ b/protocols/noise/tests/smoke.rs @@ -22,7 +22,7 @@ use futures::{future::{self, Either}, prelude::*}; use libp2p_core::identity; use libp2p_core::upgrade::{self, Negotiated, apply_inbound, apply_outbound}; use libp2p_core::transport::{Transport, ListenerEvent}; -use libp2p_noise::{Keypair, X25519, NoiseConfig, RemoteIdentity, NoiseError, NoiseOutput}; +use libp2p_noise::{Keypair, X25519, X25519Spec, NoiseConfig, RemoteIdentity, NoiseError, NoiseOutput}; use libp2p_tcp::{TcpConfig, TcpTransStream}; use log::info; use quickcheck::QuickCheck; @@ -38,6 +38,37 @@ fn core_upgrade_compat() { let _ = TcpConfig::new().upgrade(upgrade::Version::V1).authenticate(noise); } +#[test] +fn xx_spec() { + let _ = env_logger::try_init(); + fn prop(mut messages: Vec) -> bool { + messages.truncate(5); + let server_id = identity::Keypair::generate_ed25519(); + let client_id = identity::Keypair::generate_ed25519(); + + let server_id_public = server_id.public(); + let client_id_public = client_id.public(); + + let server_dh = Keypair::::new().into_authentic(&server_id).unwrap(); + let server_transport = TcpConfig::new() + .and_then(move |output, endpoint| { + upgrade::apply(output, NoiseConfig::xx(server_dh), endpoint, upgrade::Version::V1) + }) + .and_then(move |out, _| expect_identity(out, &client_id_public)); + + let client_dh = Keypair::::new().into_authentic(&client_id).unwrap(); + let client_transport = TcpConfig::new() + .and_then(move |output, endpoint| { + upgrade::apply(output, NoiseConfig::xx(client_dh), endpoint, upgrade::Version::V1) + }) + .and_then(move |out, _| expect_identity(out, &server_id_public)); + + run(server_transport, client_transport, messages); + true + } + QuickCheck::new().max_tests(30).quickcheck(prop as fn(Vec) -> bool) +} + #[test] fn xx() { let _ = env_logger::try_init(); @@ -144,15 +175,15 @@ fn ik_xx() { QuickCheck::new().max_tests(30).quickcheck(prop as fn(Vec) -> bool) } -type Output = (RemoteIdentity, NoiseOutput>); +type Output = (RemoteIdentity, NoiseOutput>); -fn run(server_transport: T, client_transport: U, messages: I) +fn run(server_transport: T, client_transport: U, messages: I) where - T: Transport, + T: Transport>, T::Dial: Send + 'static, T::Listener: Send + Unpin + 'static, T::ListenerUpgrade: Send + 'static, - U: Transport, + U: Transport>, U::Dial: Send + 'static, U::Listener: Send + 'static, U::ListenerUpgrade: Send + 'static, @@ -218,8 +249,8 @@ where }) } -fn expect_identity(output: Output, pk: &identity::PublicKey) - -> impl Future> +fn expect_identity(output: Output, pk: &identity::PublicKey) + -> impl Future, NoiseError>> { match output.0 { RemoteIdentity::IdentityKey(ref k) if k == pk => future::ok(output), diff --git a/protocols/ping/Cargo.toml b/protocols/ping/Cargo.toml index b9323e81..0b6a9960 100644 --- a/protocols/ping/Cargo.toml +++ b/protocols/ping/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-ping" edition = "2018" description = "Ping protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,8 +11,8 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } log = "0.4.1" rand = "0.7.2" void = "1.0" @@ -20,7 +20,7 @@ wasm-timer = "0.2" [dev-dependencies] async-std = "1.0" -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } -libp2p-secio = { version = "0.18.0", path = "../../protocols/secio" } -libp2p-yamux = { version = "0.18.0", path = "../../muxers/yamux" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } +libp2p-secio = { version = "0.19.0", path = "../../protocols/secio" } +libp2p-yamux = { version = "0.19.0", path = "../../muxers/yamux" } quickcheck = "0.9.0" diff --git a/protocols/ping/src/protocol.rs b/protocols/ping/src/protocol.rs index 9bd5682f..fab16b36 100644 --- a/protocols/ping/src/protocol.rs +++ b/protocols/ping/src/protocol.rs @@ -83,7 +83,7 @@ where type Future = BoxFuture<'static, Result>; fn upgrade_outbound(self, mut socket: TSocket, _: Self::Info) -> Self::Future { - let payload: [u8; 32] = thread_rng().sample(distributions::Standard); + let payload: [u8; PING_SIZE] = thread_rng().sample(distributions::Standard); debug!("Preparing ping payload {:?}", payload); async move { socket.write_all(&payload).await?; diff --git a/protocols/plaintext/Cargo.toml b/protocols/plaintext/Cargo.toml index 07321552..e00b5ac4 100644 --- a/protocols/plaintext/Cargo.toml +++ b/protocols/plaintext/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-plaintext" edition = "2018" description = "Plaintext encryption dummy protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -13,7 +13,7 @@ categories = ["network-programming", "asynchronous"] bytes = "0.5" futures = "0.3.1" futures_codec = "0.3.4" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.8" prost = "0.6.1" rw-stream-sink = "0.2.0" diff --git a/protocols/pnet/Cargo.toml b/protocols/pnet/Cargo.toml index 6897cfb4..de5ea5cf 100644 --- a/protocols/pnet/Cargo.toml +++ b/protocols/pnet/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-pnet" edition = "2018" description = "Private swarm support for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" diff --git a/protocols/pnet/src/lib.rs b/protocols/pnet/src/lib.rs index ab9b0bc3..496cd7d6 100644 --- a/protocols/pnet/src/lib.rs +++ b/protocols/pnet/src/lib.rs @@ -55,6 +55,11 @@ const FINGERPRINT_SIZE: usize = 16; pub struct PreSharedKey([u8; KEY_SIZE]); impl PreSharedKey { + /// Create a new pre shared key from raw bytes + pub fn new(data: [u8; KEY_SIZE]) -> Self { + Self(data) + } + /// Compute PreSharedKey fingerprint identical to the go-libp2p fingerprint. /// The computation of the fingerprint is not specified in the spec. /// diff --git a/protocols/secio/Cargo.toml b/protocols/secio/Cargo.toml index a5be9059..a24f4990 100644 --- a/protocols/secio/Cargo.toml +++ b/protocols/secio/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-secio" edition = "2018" description = "Secio encryption protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -16,7 +16,7 @@ ctr = "0.3" futures = "0.3.1" hmac = "0.7.0" lazy_static = "1.2.0" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.6" prost = "0.6.1" pin-project = "0.4.6" @@ -48,8 +48,8 @@ aes-all = ["aesni"] [dev-dependencies] async-std = "1.0" criterion = "0.3" -libp2p-mplex = { version = "0.18.0", path = "../../muxers/mplex" } -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } +libp2p-mplex = { version = "0.19.0", path = "../../muxers/mplex" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } [[bench]] name = "bench" diff --git a/src/lib.rs b/src/lib.rs index c907a2a7..7f24d409 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,7 +85,7 @@ //! Example ([`secio`] + [`yamux`] Protocol Upgrade): //! //! ```rust -//! # #[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "secio", feature = "yamux"))] { +//! # #[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "secio", feature = "yamux"))] { //! use libp2p::{Transport, core::upgrade, tcp::TcpConfig, secio::SecioConfig, identity::Keypair, yamux}; //! let tcp = TcpConfig::new(); //! let secio = SecioConfig::new(Keypair::generate_ed25519()); @@ -217,8 +217,8 @@ pub use libp2p_plaintext as plaintext; pub use libp2p_secio as secio; #[doc(inline)] pub use libp2p_swarm as swarm; -#[cfg(feature = "tcp")] -#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] +#[cfg(any(feature = "tcp-async-std", feature = "tcp-tokio-std"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp-async-std", feature = "tcp-tokio-std"))))] #[cfg(not(any(target_os = "emscripten", target_os = "unknown")))] #[doc(inline)] pub use libp2p_tcp as tcp; @@ -262,16 +262,14 @@ pub use self::simple::SimpleProtocol; pub use self::swarm::Swarm; pub use self::transport_ext::TransportExt; -use std::{error, io, time::Duration}; - /// Builds a `Transport` that supports the most commonly-used protocols that libp2p supports. /// /// > **Note**: This `Transport` is not suitable for production usage, as its implementation /// > reserves the right to support additional protocols or remove deprecated protocols. -#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))] -#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))))] +#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))] +#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))))] pub fn build_development_transport(keypair: identity::Keypair) - -> io::Result> + Send + Sync), Error = impl error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> + -> std::io::Result> + Send + Sync), Error = impl std::error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> { build_tcp_ws_secio_mplex_yamux(keypair) } @@ -282,10 +280,10 @@ pub fn build_development_transport(keypair: identity::Keypair) /// and mplex or yamux as the multiplexing layer. /// /// > **Note**: If you ever need to express the type of this `Transport`. -#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))] -#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))))] +#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))] +#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))))] pub fn build_tcp_ws_secio_mplex_yamux(keypair: identity::Keypair) - -> io::Result> + Send + Sync), Error = impl error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> + -> std::io::Result> + Send + Sync), Error = impl std::error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> { let transport = { let tcp = tcp::TcpConfig::new().nodelay(true); @@ -299,7 +297,7 @@ pub fn build_tcp_ws_secio_mplex_yamux(keypair: identity::Keypair) .authenticate(secio::SecioConfig::new(keypair)) .multiplex(core::upgrade::SelectUpgrade::new(yamux::Config::default(), mplex::MplexConfig::new())) .map(|(peer, muxer), _| (peer, core::muxing::StreamMuxerBox::new(muxer))) - .timeout(Duration::from_secs(20))) + .timeout(std::time::Duration::from_secs(20))) } /// Builds an implementation of `Transport` that is suitable for usage with the `Swarm`. @@ -308,10 +306,10 @@ pub fn build_tcp_ws_secio_mplex_yamux(keypair: identity::Keypair) /// and mplex or yamux as the multiplexing layer. /// /// > **Note**: If you ever need to express the type of this `Transport`. -#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux", feature = "pnet"))] -#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux", feature = "pnet"))))] +#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux", feature = "pnet"))] +#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux", feature = "pnet"))))] pub fn build_tcp_ws_pnet_secio_mplex_yamux(keypair: identity::Keypair, psk: PreSharedKey) - -> io::Result> + Send + Sync), Error = impl error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> + -> std::io::Result> + Send + Sync), Error = impl std::error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> { let transport = { let tcp = tcp::TcpConfig::new().nodelay(true); @@ -326,5 +324,5 @@ pub fn build_tcp_ws_pnet_secio_mplex_yamux(keypair: identity::Keypair, psk: PreS .authenticate(secio::SecioConfig::new(keypair)) .multiplex(core::upgrade::SelectUpgrade::new(yamux::Config::default(), mplex::MplexConfig::new())) .map(|(peer, muxer), _| (peer, core::muxing::StreamMuxerBox::new(muxer))) - .timeout(Duration::from_secs(20))) + .timeout(std::time::Duration::from_secs(20))) } diff --git a/swarm/Cargo.toml b/swarm/Cargo.toml index 7feaa30a..46785665 100644 --- a/swarm/Cargo.toml +++ b/swarm/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-swarm" edition = "2018" description = "The libp2p swarm" -version = "0.18.1" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,7 +11,7 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../core" } +libp2p-core = { version = "0.19.0", path = "../core" } log = "0.4" rand = "0.7" smallvec = "1.0" @@ -19,6 +19,6 @@ wasm-timer = "0.2" void = "1" [dev-dependencies] -libp2p-mplex = { version = "0.18.0", path = "../muxers/mplex" } +libp2p-mplex = { version = "0.19.0", path = "../muxers/mplex" } quickcheck = "0.9.0" rand = "0.7.2" diff --git a/swarm/src/behaviour.rs b/swarm/src/behaviour.rs index fdddc6a0..16e804f9 100644 --- a/swarm/src/behaviour.rs +++ b/swarm/src/behaviour.rs @@ -291,12 +291,10 @@ pub enum DialPeerCondition { /// If there is an ongoing dialing attempt, the addresses reported by /// [`NetworkBehaviour::addresses_of_peer`] are added to the ongoing /// dialing attempt, ignoring duplicates. - /// - /// This condition implies [`DialPeerCondition::Disconnected`]. NotDialing, - // TODO: Once multiple dialing attempts per peer are permitted. - // See https://github.com/libp2p/rust-libp2p/pull/1506. - // Always, + /// A new dialing attempt is always initiated, only subject to the + /// configured connection limits. + Always, } impl Default for DialPeerCondition { diff --git a/swarm/src/lib.rs b/swarm/src/lib.rs index 1c62d8d9..f848f64f 100644 --- a/swarm/src/lib.rs +++ b/swarm/src/lib.rs @@ -115,7 +115,6 @@ use libp2p_core::{ NetworkInfo, NetworkEvent, NetworkConfig, - Peer, peer::ConnectedPeer, }, upgrade::ProtocolName, @@ -124,7 +123,7 @@ use registry::{Addresses, AddressIntoIter}; use smallvec::SmallVec; use std::{error, fmt, hash::Hash, io, ops::{Deref, DerefMut}, pin::Pin, task::{Context, Poll}}; use std::collections::HashSet; -use std::num::NonZeroU32; +use std::num::{NonZeroU32, NonZeroUsize}; use upgrade::UpgradeInfoSend as _; /// Contains the state of the network, plus the way it should behave. @@ -379,70 +378,31 @@ where TBehaviour: NetworkBehaviour, /// /// If a new dialing attempt has been initiated, `Ok(true)` is returned. /// - /// If there is an ongoing dialing attempt, the current addresses of the - /// peer, as reported by [`NetworkBehaviour::addresses_of_peer`] are added - /// to the ongoing dialing attempt, ignoring duplicates. In this case no - /// new dialing attempt is initiated. - /// /// If no new dialing attempt has been initiated, meaning there is an ongoing /// dialing attempt or `addresses_of_peer` reports no addresses, `Ok(false)` /// is returned. - pub fn dial(me: &mut Self, peer_id: &PeerId) -> Result { + pub fn dial(me: &mut Self, peer_id: &PeerId) -> Result<(), DialError> { let mut addrs = me.behaviour.addresses_of_peer(peer_id).into_iter(); - match me.network.peer(peer_id.clone()) { - Peer::Disconnected(peer) => { - if let Some(first) = addrs.next() { - let handler = me.behaviour.new_handler().into_node_handler_builder(); - match peer.connect(first, addrs, handler) { - Ok(_) => return Ok(true), - Err(error) => { - log::debug!( - "New dialing attempt to disconnected peer {:?} failed: {:?}.", - peer_id, error); - me.behaviour.inject_dial_failure(&peer_id); - return Err(error) - } - } - } else { - log::debug!( - "New dialing attempt to disconnected peer {:?} failed: no address.", - peer_id - ); - me.behaviour.inject_dial_failure(&peer_id); - } - Ok(false) - }, - Peer::Connected(peer) => { - if let Some(first) = addrs.next() { - let handler = me.behaviour.new_handler().into_node_handler_builder(); - match peer.connect(first, addrs, handler) { - Ok(_) => return Ok(true), - Err(error) => { - log::debug!( - "New dialing attempt to connected peer {:?} failed: {:?}.", - peer_id, error); - me.behaviour.inject_dial_failure(&peer_id); - return Err(error) - } - } - } else { - log::debug!( - "New dialing attempt to disconnected peer {:?} failed: no address.", - peer_id - ); - me.behaviour.inject_dial_failure(&peer_id); - } - Ok(false) - } - Peer::Dialing(mut peer) => { - peer.connection().add_addresses(addrs); - Ok(false) - }, - Peer::Local => { - me.behaviour.inject_dial_failure(&peer_id); - Err(ConnectionLimit { current: 0, limit: 0 }) - } + let peer = me.network.peer(peer_id.clone()); + + let result = + if let Some(first) = addrs.next() { + let handler = me.behaviour.new_handler().into_node_handler_builder(); + peer.dial(first, addrs, handler) + .map(|_| ()) + .map_err(DialError::ConnectionLimit) + } else { + Err(DialError::NoAddresses) + }; + + if let Err(error) = &result { + log::debug!( + "New dialing attempt to peer {:?} failed: {:?}.", + peer_id, error); + me.behaviour.inject_dial_failure(&peer_id); } + + result } /// Returns an iterator that produces the list of addresses we're listening on. @@ -721,18 +681,22 @@ where TBehaviour: NetworkBehaviour, if !this.network.is_dialing(&peer_id) => true, _ => false }; - if condition_matched { - if let Ok(true) = ExpandedSwarm::dial(this, &peer_id) { - return Poll::Ready(SwarmEvent::Dialing(peer_id)); + if ExpandedSwarm::dial(this, &peer_id).is_ok() { + return Poll::Ready(SwarmEvent::Dialing(peer_id)) } - } else { + // Even if the condition for a _new_ dialing attempt is not met, + // we always add any potentially new addresses of the peer to an + // ongoing dialing attempt, if there is one. log::trace!("Condition for new dialing attempt to {:?} not met: {:?}", peer_id, condition); if let Some(mut peer) = this.network.peer(peer_id.clone()).into_dialing() { let addrs = this.behaviour.addresses_of_peer(peer.id()); - peer.connection().add_addresses(addrs); + let mut attempt = peer.some_attempt(); + for addr in addrs { + attempt.add_address(addr); + } } } } @@ -1038,6 +1002,48 @@ where TBehaviour: NetworkBehaviour, self } + /// Configures the number of events from the [`NetworkBehaviour`] in + /// destination to the [`ProtocolsHandler`] that can be buffered before + /// the [`Swarm`] has to wait. An individual buffer with this number of + /// events exists for each individual connection. + /// + /// The ideal value depends on the executor used, the CPU speed, and the + /// volume of events. If this value is too low, then the [`Swarm`] will + /// be sleeping more often than necessary. Increasing this value increases + /// the overall memory usage. + pub fn notify_handler_buffer_size(mut self, n: NonZeroUsize) -> Self { + self.network_config.set_notify_handler_buffer_size(n); + self + } + + /// Configures the number of extra events from the [`ProtocolsHandler`] in + /// destination to the [`NetworkBehaviour`] that can be buffered before + /// the [`ProtocolsHandler`] has to go to sleep. + /// + /// There exists a buffer of events received from [`ProtocolsHandler`]s + /// that the [`NetworkBehaviour`] has yet to process. This buffer is + /// shared between all instances of [`ProtocolsHandler`]. Each instance of + /// [`ProtocolsHandler`] is guaranteed one slot in this buffer, meaning + /// that delivering an event for the first time is guaranteed to be + /// instantaneous. Any extra event delivery, however, must wait for that + /// first event to be delivered or for an "extra slot" to be available. + /// + /// This option configures the number of such "extra slots" in this + /// shared buffer. These extra slots are assigned in a first-come, + /// first-served basis. + /// + /// The ideal value depends on the executor used, the CPU speed, the + /// average number of connections, and the volume of events. If this value + /// is too low, then the [`ProtocolsHandler`]s will be sleeping more often + /// than necessary. Increasing this value increases the overall memory + /// usage, and more importantly the latency between the moment when an + /// event is emitted and the moment when it is received by the + /// [`NetworkBehaviour`]. + pub fn connection_event_buffer_size(mut self, n: usize) -> Self { + self.network_config.set_connection_event_buffer_size(n); + self + } + /// Configures a limit for the number of simultaneous incoming /// connection attempts. pub fn incoming_connection_limit(mut self, n: usize) -> Self { @@ -1104,6 +1110,35 @@ where TBehaviour: NetworkBehaviour, } } +/// The possible failures of [`ExpandedSwarm::dial`]. +#[derive(Debug)] +pub enum DialError { + /// The configured limit for simultaneous outgoing connections + /// has been reached. + ConnectionLimit(ConnectionLimit), + /// [`NetworkBehaviour::addresses_of_peer`] returned no addresses + /// for the peer to dial. + NoAddresses +} + +impl fmt::Display for DialError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DialError::ConnectionLimit(err) => write!(f, "Dial error: {}", err), + DialError::NoAddresses => write!(f, "Dial error: no addresses for peer.") + } + } +} + +impl error::Error for DialError { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + match self { + DialError::ConnectionLimit(err) => Some(err), + DialError::NoAddresses => None + } + } +} + /// Dummy implementation of [`NetworkBehaviour`] that doesn't do anything. #[derive(Clone, Default)] pub struct DummyBehaviour { diff --git a/transports/dns/Cargo.toml b/transports/dns/Cargo.toml index 11d1958f..acd122f1 100644 --- a/transports/dns/Cargo.toml +++ b/transports/dns/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-dns" edition = "2018" description = "DNS transport implementation for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -10,6 +10,6 @@ keywords = ["peer-to-peer", "libp2p", "networking"] categories = ["network-programming", "asynchronous"] [dependencies] -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.1" futures = "0.3.1" diff --git a/transports/dns/src/lib.rs b/transports/dns/src/lib.rs index 7c0a372c..82e54151 100644 --- a/transports/dns/src/lib.rs +++ b/transports/dns/src/lib.rs @@ -124,6 +124,7 @@ where // As an optimization, we immediately pass through if no component of the address contain // a DNS protocol. let contains_dns = addr.iter().any(|cmp| match cmp { + Protocol::Dns(_) => true, Protocol::Dns4(_) => true, Protocol::Dns6(_) => true, _ => false, @@ -139,7 +140,7 @@ where trace!("Dialing address with DNS: {}", addr); let resolve_futs = addr.iter() .map(|cmp| match cmp { - Protocol::Dns4(ref name) | Protocol::Dns6(ref name) => { + Protocol::Dns(ref name) | Protocol::Dns4(ref name) | Protocol::Dns6(ref name) => { let name = name.to_string(); let to_resolve = format!("{}:0", name); let (tx, rx) = oneshot::channel(); @@ -151,7 +152,12 @@ where }); }); - let is_dns4 = if let Protocol::Dns4(_) = cmp { true } else { false }; + let (dns4, dns6) = match cmp { + Protocol::Dns(_) => (true, true), + Protocol::Dns4(_) => (true, false), + Protocol::Dns6(_) => (false, true), + _ => unreachable!(), + }; async move { let list = rx.await @@ -166,7 +172,7 @@ where list.into_iter() .filter_map(|addr| { - if (is_dns4 && addr.is_ipv4()) || (!is_dns4 && addr.is_ipv6()) { + if (dns4 && addr.is_ipv4()) || (dns6 && addr.is_ipv6()) { Some(Protocol::from(addr)) } else { None diff --git a/transports/tcp/Cargo.toml b/transports/tcp/Cargo.toml index b1d5d996..8b6a5af4 100644 --- a/transports/tcp/Cargo.toml +++ b/transports/tcp/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-tcp" edition = "2018" description = "TCP/IP transport protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -15,9 +15,7 @@ futures = "0.3.1" futures-timer = "3.0" get_if_addrs = "0.5.3" ipnet = "2.0.0" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.1" +socket2 = "0.3.12" tokio = { version = "0.2", default-features = false, features = ["tcp"], optional = true } - -[features] -default = ["async-std"] diff --git a/transports/tcp/src/lib.rs b/transports/tcp/src/lib.rs index aaa70f6e..1e870f78 100644 --- a/transports/tcp/src/lib.rs +++ b/transports/tcp/src/lib.rs @@ -39,8 +39,10 @@ use libp2p_core::{ transport::{ListenerEvent, TransportError} }; use log::{debug, trace}; +use socket2::{Socket, Domain, Type}; use std::{ collections::VecDeque, + convert::TryFrom, io, iter::{self, FromIterator}, net::{IpAddr, SocketAddr}, @@ -108,7 +110,22 @@ impl Transport for $tcp_config { async fn do_listen(cfg: $tcp_config, socket_addr: SocketAddr) -> Result>, io::Error>, io::Error>>, io::Error> { - let listener = <$tcp_listener>::bind(&socket_addr).await?; + let socket = if socket_addr.is_ipv4() { + Socket::new(Domain::ipv4(), Type::stream(), Some(socket2::Protocol::tcp()))? + } else { + let s = Socket::new(Domain::ipv6(), Type::stream(), Some(socket2::Protocol::tcp()))?; + s.set_only_v6(true)?; + s + }; + if cfg!(target_family = "unix") { + socket.set_reuse_address(true)?; + } + socket.bind(&socket_addr.into())?; + socket.listen(1024)?; // we may want to make this configurable + + let listener = <$tcp_listener>::try_from(socket.into_tcp_listener()) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + let local_addr = listener.local_addr()?; let port = local_addr.port(); @@ -485,42 +502,45 @@ mod tests { #[test] #[cfg(feature = "async-std")] fn wildcard_expansion() { - let mut listener = TcpConfig::new() - .listen_on("/ip4/0.0.0.0/tcp/0".parse().unwrap()) - .expect("listener"); + fn test(addr: Multiaddr) { + let mut listener = TcpConfig::new().listen_on(addr).expect("listener"); - // Get the first address. - let addr = futures::executor::block_on_stream(listener.by_ref()) - .next() - .expect("some event") - .expect("no error") - .into_new_address() - .expect("listen address"); + // Get the first address. + let addr = futures::executor::block_on_stream(listener.by_ref()) + .next() + .expect("some event") + .expect("no error") + .into_new_address() + .expect("listen address"); - // Process all initial `NewAddress` events and make sure they - // do not contain wildcard address or port. - let server = listener - .take_while(|event| match event.as_ref().unwrap() { - ListenerEvent::NewAddress(a) => { - let mut iter = a.iter(); - match iter.next().expect("ip address") { - Protocol::Ip4(ip) => assert!(!ip.is_unspecified()), - Protocol::Ip6(ip) => assert!(!ip.is_unspecified()), - other => panic!("Unexpected protocol: {}", other) + // Process all initial `NewAddress` events and make sure they + // do not contain wildcard address or port. + let server = listener + .take_while(|event| match event.as_ref().unwrap() { + ListenerEvent::NewAddress(a) => { + let mut iter = a.iter(); + match iter.next().expect("ip address") { + Protocol::Ip4(ip) => assert!(!ip.is_unspecified()), + Protocol::Ip6(ip) => assert!(!ip.is_unspecified()), + other => panic!("Unexpected protocol: {}", other) + } + if let Protocol::Tcp(port) = iter.next().expect("port") { + assert_ne!(0, port) + } else { + panic!("No TCP port in address: {}", a) + } + futures::future::ready(true) } - if let Protocol::Tcp(port) = iter.next().expect("port") { - assert_ne!(0, port) - } else { - panic!("No TCP port in address: {}", a) - } - futures::future::ready(true) - } - _ => futures::future::ready(false) - }) - .for_each(|_| futures::future::ready(())); + _ => futures::future::ready(false) + }) + .for_each(|_| futures::future::ready(())); - let client = TcpConfig::new().dial(addr).expect("dialer"); - async_std::task::block_on(futures::future::join(server, client)).1.unwrap(); + let client = TcpConfig::new().dial(addr).expect("dialer"); + async_std::task::block_on(futures::future::join(server, client)).1.unwrap(); + } + + test("/ip4/0.0.0.0/tcp/0".parse().unwrap()); + test("/ip6/::1/tcp/0".parse().unwrap()); } #[test] @@ -575,43 +595,47 @@ mod tests { #[test] #[cfg(feature = "async-std")] fn communicating_between_dialer_and_listener() { - let (ready_tx, ready_rx) = futures::channel::oneshot::channel(); - let mut ready_tx = Some(ready_tx); + fn test(addr: Multiaddr) { + let (ready_tx, ready_rx) = futures::channel::oneshot::channel(); + let mut ready_tx = Some(ready_tx); - async_std::task::spawn(async move { - let addr = "/ip4/127.0.0.1/tcp/0".parse::().unwrap(); - let tcp = TcpConfig::new(); - let mut listener = tcp.listen_on(addr).unwrap(); + async_std::task::spawn(async move { + let tcp = TcpConfig::new(); + let mut listener = tcp.listen_on(addr).unwrap(); - loop { - match listener.next().await.unwrap().unwrap() { - ListenerEvent::NewAddress(listen_addr) => { - ready_tx.take().unwrap().send(listen_addr).unwrap(); - }, - ListenerEvent::Upgrade { upgrade, .. } => { - let mut upgrade = upgrade.await.unwrap(); - let mut buf = [0u8; 3]; - upgrade.read_exact(&mut buf).await.unwrap(); - assert_eq!(buf, [1, 2, 3]); - upgrade.write_all(&[4, 5, 6]).await.unwrap(); - }, - _ => unreachable!() + loop { + match listener.next().await.unwrap().unwrap() { + ListenerEvent::NewAddress(listen_addr) => { + ready_tx.take().unwrap().send(listen_addr).unwrap(); + }, + ListenerEvent::Upgrade { upgrade, .. } => { + let mut upgrade = upgrade.await.unwrap(); + let mut buf = [0u8; 3]; + upgrade.read_exact(&mut buf).await.unwrap(); + assert_eq!(buf, [1, 2, 3]); + upgrade.write_all(&[4, 5, 6]).await.unwrap(); + }, + _ => unreachable!() + } } - } - }); + }); - async_std::task::block_on(async move { - let addr = ready_rx.await.unwrap(); - let tcp = TcpConfig::new(); + async_std::task::block_on(async move { + let addr = ready_rx.await.unwrap(); + let tcp = TcpConfig::new(); - // Obtain a future socket through dialing - let mut socket = tcp.dial(addr.clone()).unwrap().await.unwrap(); - socket.write_all(&[0x1, 0x2, 0x3]).await.unwrap(); + // Obtain a future socket through dialing + let mut socket = tcp.dial(addr.clone()).unwrap().await.unwrap(); + socket.write_all(&[0x1, 0x2, 0x3]).await.unwrap(); - let mut buf = [0u8; 3]; - socket.read_exact(&mut buf).await.unwrap(); - assert_eq!(buf, [4, 5, 6]); - }); + let mut buf = [0u8; 3]; + socket.read_exact(&mut buf).await.unwrap(); + assert_eq!(buf, [4, 5, 6]); + }); + } + + test("/ip4/127.0.0.1/tcp/0".parse().unwrap()); + test("/ip6/::1/tcp/0".parse().unwrap()); } #[test] diff --git a/transports/uds/Cargo.toml b/transports/uds/Cargo.toml index ee83f852..0fcfa6ff 100644 --- a/transports/uds/Cargo.toml +++ b/transports/uds/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-uds" edition = "2018" description = "Unix domain sockets transport for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,7 +11,7 @@ categories = ["network-programming", "asynchronous"] [target.'cfg(all(unix, not(any(target_os = "emscripten", target_os = "unknown"))))'.dependencies] async-std = { version = "1.0", optional = true } -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.1" futures = "0.3.1" tokio = { version = "0.2", default-features = false, features = ["uds"], optional = true } diff --git a/transports/wasm-ext/Cargo.toml b/transports/wasm-ext/Cargo.toml index 4661e8b8..c04d00b7 100644 --- a/transports/wasm-ext/Cargo.toml +++ b/transports/wasm-ext/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libp2p-wasm-ext" -version = "0.18.0" +version = "0.19.0" authors = ["Pierre Krieger "] edition = "2018" description = "Allows passing in an external transport in a WASM environment" @@ -12,7 +12,7 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" js-sys = "0.3.19" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } parity-send-wrapper = "0.1.0" wasm-bindgen = "0.2.42" wasm-bindgen-futures = "0.4.4" diff --git a/transports/websocket/Cargo.toml b/transports/websocket/Cargo.toml index 33d0419b..7dfe6c75 100644 --- a/transports/websocket/Cargo.toml +++ b/transports/websocket/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-websocket" edition = "2018" description = "WebSocket transport for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -14,7 +14,7 @@ async-tls = "0.7.0" bytes = "0.5" either = "1.5.3" futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.8" quicksink = "0.1" rustls = "0.17.0" @@ -25,4 +25,4 @@ webpki = "0.21" webpki-roots = "0.18" [dev-dependencies] -libp2p-tcp = { version = "0.18.0", path = "../tcp" } +libp2p-tcp = { version = "0.19.0", path = "../tcp" } diff --git a/transports/websocket/src/framed.rs b/transports/websocket/src/framed.rs index 191636a0..182b81ab 100644 --- a/transports/websocket/src/framed.rs +++ b/transports/websocket/src/framed.rs @@ -353,6 +353,8 @@ fn host_and_dnsname(addr: &Multiaddr) -> Result<(String, Option Ok((format!("{}:{}", ip, port), None)), + (Some(Protocol::Dns(h)), Some(Protocol::Tcp(port))) => + Ok((format!("{}:{}", &h, port), Some(tls::dns_name_ref(&h)?.to_owned()))), (Some(Protocol::Dns4(h)), Some(Protocol::Tcp(port))) => Ok((format!("{}:{}", &h, port), Some(tls::dns_name_ref(&h)?.to_owned()))), (Some(Protocol::Dns6(h)), Some(Protocol::Tcp(port))) => @@ -371,7 +373,7 @@ fn location_to_multiaddr(location: &str) -> Result> { let mut a = Multiaddr::empty(); match url.host() { Some(url::Host::Domain(h)) => { - a.push(Protocol::Dns4(h.into())) + a.push(Protocol::Dns(h.into())) } Some(url::Host::Ipv4(ip)) => { a.push(Protocol::Ip4(ip))