From bdbab3239224e47500dbd4ed9ef25cce71e211ad Mon Sep 17 00:00:00 2001 From: Toralf Wittner Date: Fri, 24 Apr 2020 19:07:27 +0200 Subject: [PATCH 01/25] tcp: Set IPV6_V6ONLY for IPv6 listeners. (#1555) * tcp: Set IPV6_V6ONLY for IPv6 listeners. The current behaviour of listening on an IPv6 address varies depending on the operating system's IP address stack implementation. Some support IPv4-mapped IPv6 addresses (e.g. Linux and newer versions of Windows) so a single IPv6 address would support IPv4-mapped addresses too. Others do not (e.g. OpenBSD). If they do, then some support them by default (e.g. Linux) and some do not (e.g. Windows). This PR attempts to implement the same behaviour accross operating systems. The strategy is as follows: Disable IPv4-mapped IPv6 addresses, hence the socket option IPV6_V6ONLY is always set to true. This allows binding two sockets to the same port and also avoids the problem of comparing mixed addresses which leads issues such as #1552. * Update CHANGELOG and address review concerns. * Update CHANGELOG.md Co-Authored-By: Pierre Krieger Co-authored-by: Pierre Krieger --- CHANGELOG.md | 5 ++ transports/tcp/Cargo.toml | 1 + transports/tcp/src/lib.rs | 152 ++++++++++++++++++++++---------------- 3 files changed, 94 insertions(+), 64 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c49535ac..678b4c33 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ initialization. Unless `KademliaConfig::set_replication_factor` is used change has no effect. [PR 1536](https://github.com/libp2p/rust-libp2p/pull/1536) +- `libp2p-tcp`: On listeners started with an IPv6 multi-address the socket + option `IPV6_V6ONLY` is set to true. Instead of relying on IPv4-mapped IPv6 + address support, two listeners can be started if IPv4 and IPv6 should both + be supported. IPv4 listener addresses are not affected by this change. + [PR 1555](https://github.com/libp2p/rust-libp2p/pull/1555) # Version 0.18.1 (2020-04-17) diff --git a/transports/tcp/Cargo.toml b/transports/tcp/Cargo.toml index b1d5d996..1133941c 100644 --- a/transports/tcp/Cargo.toml +++ b/transports/tcp/Cargo.toml @@ -17,6 +17,7 @@ get_if_addrs = "0.5.3" ipnet = "2.0.0" libp2p-core = { version = "0.18.0", path = "../../core" } log = "0.4.1" +socket2 = "0.3.12" tokio = { version = "0.2", default-features = false, features = ["tcp"], optional = true } [features] diff --git a/transports/tcp/src/lib.rs b/transports/tcp/src/lib.rs index aaa70f6e..1e870f78 100644 --- a/transports/tcp/src/lib.rs +++ b/transports/tcp/src/lib.rs @@ -39,8 +39,10 @@ use libp2p_core::{ transport::{ListenerEvent, TransportError} }; use log::{debug, trace}; +use socket2::{Socket, Domain, Type}; use std::{ collections::VecDeque, + convert::TryFrom, io, iter::{self, FromIterator}, net::{IpAddr, SocketAddr}, @@ -108,7 +110,22 @@ impl Transport for $tcp_config { async fn do_listen(cfg: $tcp_config, socket_addr: SocketAddr) -> Result>, io::Error>, io::Error>>, io::Error> { - let listener = <$tcp_listener>::bind(&socket_addr).await?; + let socket = if socket_addr.is_ipv4() { + Socket::new(Domain::ipv4(), Type::stream(), Some(socket2::Protocol::tcp()))? + } else { + let s = Socket::new(Domain::ipv6(), Type::stream(), Some(socket2::Protocol::tcp()))?; + s.set_only_v6(true)?; + s + }; + if cfg!(target_family = "unix") { + socket.set_reuse_address(true)?; + } + socket.bind(&socket_addr.into())?; + socket.listen(1024)?; // we may want to make this configurable + + let listener = <$tcp_listener>::try_from(socket.into_tcp_listener()) + .map_err(|e| io::Error::new(io::ErrorKind::Other, e))?; + let local_addr = listener.local_addr()?; let port = local_addr.port(); @@ -485,42 +502,45 @@ mod tests { #[test] #[cfg(feature = "async-std")] fn wildcard_expansion() { - let mut listener = TcpConfig::new() - .listen_on("/ip4/0.0.0.0/tcp/0".parse().unwrap()) - .expect("listener"); + fn test(addr: Multiaddr) { + let mut listener = TcpConfig::new().listen_on(addr).expect("listener"); - // Get the first address. - let addr = futures::executor::block_on_stream(listener.by_ref()) - .next() - .expect("some event") - .expect("no error") - .into_new_address() - .expect("listen address"); + // Get the first address. + let addr = futures::executor::block_on_stream(listener.by_ref()) + .next() + .expect("some event") + .expect("no error") + .into_new_address() + .expect("listen address"); - // Process all initial `NewAddress` events and make sure they - // do not contain wildcard address or port. - let server = listener - .take_while(|event| match event.as_ref().unwrap() { - ListenerEvent::NewAddress(a) => { - let mut iter = a.iter(); - match iter.next().expect("ip address") { - Protocol::Ip4(ip) => assert!(!ip.is_unspecified()), - Protocol::Ip6(ip) => assert!(!ip.is_unspecified()), - other => panic!("Unexpected protocol: {}", other) + // Process all initial `NewAddress` events and make sure they + // do not contain wildcard address or port. + let server = listener + .take_while(|event| match event.as_ref().unwrap() { + ListenerEvent::NewAddress(a) => { + let mut iter = a.iter(); + match iter.next().expect("ip address") { + Protocol::Ip4(ip) => assert!(!ip.is_unspecified()), + Protocol::Ip6(ip) => assert!(!ip.is_unspecified()), + other => panic!("Unexpected protocol: {}", other) + } + if let Protocol::Tcp(port) = iter.next().expect("port") { + assert_ne!(0, port) + } else { + panic!("No TCP port in address: {}", a) + } + futures::future::ready(true) } - if let Protocol::Tcp(port) = iter.next().expect("port") { - assert_ne!(0, port) - } else { - panic!("No TCP port in address: {}", a) - } - futures::future::ready(true) - } - _ => futures::future::ready(false) - }) - .for_each(|_| futures::future::ready(())); + _ => futures::future::ready(false) + }) + .for_each(|_| futures::future::ready(())); - let client = TcpConfig::new().dial(addr).expect("dialer"); - async_std::task::block_on(futures::future::join(server, client)).1.unwrap(); + let client = TcpConfig::new().dial(addr).expect("dialer"); + async_std::task::block_on(futures::future::join(server, client)).1.unwrap(); + } + + test("/ip4/0.0.0.0/tcp/0".parse().unwrap()); + test("/ip6/::1/tcp/0".parse().unwrap()); } #[test] @@ -575,43 +595,47 @@ mod tests { #[test] #[cfg(feature = "async-std")] fn communicating_between_dialer_and_listener() { - let (ready_tx, ready_rx) = futures::channel::oneshot::channel(); - let mut ready_tx = Some(ready_tx); + fn test(addr: Multiaddr) { + let (ready_tx, ready_rx) = futures::channel::oneshot::channel(); + let mut ready_tx = Some(ready_tx); - async_std::task::spawn(async move { - let addr = "/ip4/127.0.0.1/tcp/0".parse::().unwrap(); - let tcp = TcpConfig::new(); - let mut listener = tcp.listen_on(addr).unwrap(); + async_std::task::spawn(async move { + let tcp = TcpConfig::new(); + let mut listener = tcp.listen_on(addr).unwrap(); - loop { - match listener.next().await.unwrap().unwrap() { - ListenerEvent::NewAddress(listen_addr) => { - ready_tx.take().unwrap().send(listen_addr).unwrap(); - }, - ListenerEvent::Upgrade { upgrade, .. } => { - let mut upgrade = upgrade.await.unwrap(); - let mut buf = [0u8; 3]; - upgrade.read_exact(&mut buf).await.unwrap(); - assert_eq!(buf, [1, 2, 3]); - upgrade.write_all(&[4, 5, 6]).await.unwrap(); - }, - _ => unreachable!() + loop { + match listener.next().await.unwrap().unwrap() { + ListenerEvent::NewAddress(listen_addr) => { + ready_tx.take().unwrap().send(listen_addr).unwrap(); + }, + ListenerEvent::Upgrade { upgrade, .. } => { + let mut upgrade = upgrade.await.unwrap(); + let mut buf = [0u8; 3]; + upgrade.read_exact(&mut buf).await.unwrap(); + assert_eq!(buf, [1, 2, 3]); + upgrade.write_all(&[4, 5, 6]).await.unwrap(); + }, + _ => unreachable!() + } } - } - }); + }); - async_std::task::block_on(async move { - let addr = ready_rx.await.unwrap(); - let tcp = TcpConfig::new(); + async_std::task::block_on(async move { + let addr = ready_rx.await.unwrap(); + let tcp = TcpConfig::new(); - // Obtain a future socket through dialing - let mut socket = tcp.dial(addr.clone()).unwrap().await.unwrap(); - socket.write_all(&[0x1, 0x2, 0x3]).await.unwrap(); + // Obtain a future socket through dialing + let mut socket = tcp.dial(addr.clone()).unwrap().await.unwrap(); + socket.write_all(&[0x1, 0x2, 0x3]).await.unwrap(); - let mut buf = [0u8; 3]; - socket.read_exact(&mut buf).await.unwrap(); - assert_eq!(buf, [4, 5, 6]); - }); + let mut buf = [0u8; 3]; + socket.read_exact(&mut buf).await.unwrap(); + assert_eq!(buf, [4, 5, 6]); + }); + } + + test("/ip4/127.0.0.1/tcp/0".parse().unwrap()); + test("/ip6/::1/tcp/0".parse().unwrap()); } #[test] From 3d4ae5da4b004050de325cbdc52219de5fdbca23 Mon Sep 17 00:00:00 2001 From: Arjan Topolovec Date: Tue, 28 Apr 2020 11:38:25 +0200 Subject: [PATCH 02/25] fix: NetworkBehaviour crash if ignoring a non-last struct field (#1554) Macro was using field_n (field index including ignored fields) for calculations instad of enum_n (field index excluding ignored fields). When calculating te nesting the calculation was made between the number of non-ignored fields and a field_n that lead to and overflow. Co-authored-by: Pierre Krieger --- misc/core-derive/src/lib.rs | 4 ++-- misc/core-derive/tests/test.rs | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/misc/core-derive/src/lib.rs b/misc/core-derive/src/lib.rs index d7d28f15..ba9b0411 100644 --- a/misc/core-derive/src/lib.rs +++ b/misc/core-derive/src/lib.rs @@ -288,7 +288,7 @@ fn build_struct(ast: &DeriveInput, data_struct: &DataStruct) -> TokenStream { quote!{ ev } }; - for _ in 0 .. data_struct.fields.iter().filter(|f| !is_ignored(f)).count() - 1 - field_n { + for _ in 0 .. data_struct.fields.iter().filter(|f| !is_ignored(f)).count() - 1 - enum_n { elem = quote!{ #either_ident::First(#elem) }; } @@ -378,7 +378,7 @@ fn build_struct(ast: &DeriveInput, data_struct: &DataStruct) -> TokenStream { } else { quote!{ event } }; - for _ in 0 .. data_struct.fields.iter().filter(|f| !is_ignored(f)).count() - 1 - field_n { + for _ in 0 .. data_struct.fields.iter().filter(|f| !is_ignored(f)).count() - 1 - enum_n { wrapped_event = quote!{ #either_ident::First(#wrapped_event) }; } diff --git a/misc/core-derive/tests/test.rs b/misc/core-derive/tests/test.rs index b65fb16e..6181de8e 100644 --- a/misc/core-derive/tests/test.rs +++ b/misc/core-derive/tests/test.rs @@ -109,6 +109,33 @@ fn three_fields() { } } +#[test] +fn three_fields_non_last_ignored() { + #[allow(dead_code)] + #[derive(NetworkBehaviour)] + struct Foo { + ping: libp2p::ping::Ping, + #[behaviour(ignore)] + identify: String, + kad: libp2p::kad::Kademlia, + } + + impl libp2p::swarm::NetworkBehaviourEventProcess for Foo { + fn inject_event(&mut self, _: libp2p::ping::PingEvent) { + } + } + + impl libp2p::swarm::NetworkBehaviourEventProcess for Foo { + fn inject_event(&mut self, _: libp2p::kad::KademliaEvent) { + } + } + + #[allow(dead_code)] + fn foo() { + require_net_behaviour::(); + } +} + #[test] fn custom_polling() { #[allow(dead_code)] From 1be34fd7a363b04a985b596abf0dba4941ae9a01 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 28 Apr 2020 16:18:35 +0200 Subject: [PATCH 03/25] protocols/gossipsub: Correct max transmit size error handling (#1558) If a user sends a message that is over the maximum transmission size gossipsub will disconnect from the peer being sent the message. This PR updates the logic to simply emit an error, not send the over-sized message but maintain the long-lived streams for future messages. Co-authored-by: Age Manning --- protocols/gossipsub/src/handler.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/protocols/gossipsub/src/handler.rs b/protocols/gossipsub/src/handler.rs index 495bf305..b9cf9559 100644 --- a/protocols/gossipsub/src/handler.rs +++ b/protocols/gossipsub/src/handler.rs @@ -27,7 +27,7 @@ use libp2p_swarm::protocols_handler::{ KeepAlive, ProtocolsHandler, ProtocolsHandlerEvent, ProtocolsHandlerUpgrErr, SubstreamProtocol, }; use libp2p_swarm::NegotiatedSubstream; -use log::{debug, trace, warn}; +use log::{debug, error, trace, warn}; use smallvec::SmallVec; use std::{ borrow::Cow, @@ -274,7 +274,13 @@ impl ProtocolsHandler for GossipsubHandler { Some(OutboundSubstreamState::PendingFlush(substream)) } Err(e) => { - return Poll::Ready(ProtocolsHandlerEvent::Close(e)); + if let io::ErrorKind::PermissionDenied = e.kind() { + error!("Message over the maximum transmission limit was not sent."); + self.outbound_substream = + Some(OutboundSubstreamState::WaitingOutput(substream)); + } else { + return Poll::Ready(ProtocolsHandlerEvent::Close(e)); + } } } } From 69fd8ac195cac1656e39f1e54c14ceb28d7e707a Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Wed, 6 May 2020 13:50:18 +0200 Subject: [PATCH 04/25] Update CI to clang-10 (#1565) --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e708f594..44715bfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: container: image: rust env: - CC: clang-9 + CC: clang-10 steps: - uses: actions/checkout@v1 - name: Install Rust @@ -50,9 +50,9 @@ jobs: - name: Install a recent version of clang run: | wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - - echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-9 main" >> /etc/apt/sources.list + echo "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic-10 main" >> /etc/apt/sources.list apt-get update - apt-get install -y clang-9 + apt-get install -y clang-10 - name: Install CMake run: apt-get install -y cmake - name: Cache cargo registry From bd5e81fedbc5aa9bf941a8ba5da02dee26f291e3 Mon Sep 17 00:00:00 2001 From: Roman Borschel Date: Wed, 6 May 2020 14:30:23 +0200 Subject: [PATCH 05/25] Spec-compliant Noise Protocol (#1545) * Add spec-compliant x25519 protocol. * Cosmetic protobuf spec compliance. * Keep support for IK/IX. Update changelog. * Update docs. * CI --- CHANGELOG.md | 13 ++ protocols/noise/src/io/handshake.rs | 16 +- .../noise/src/io/handshake/payload.proto | 8 +- protocols/noise/src/lib.rs | 10 +- protocols/noise/src/protocol.rs | 15 +- protocols/noise/src/protocol/x25519.rs | 11 +- protocols/noise/src/protocol/x25519_spec.rs | 150 ++++++++++++++++++ protocols/noise/tests/smoke.rs | 45 +++++- 8 files changed, 242 insertions(+), 26 deletions(-) create mode 100644 protocols/noise/src/protocol/x25519_spec.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 678b4c33..b585911a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,22 @@ # Version ??? +- `libp2p-noise`: Added the `X25519Spec` protocol suite which uses + libp2p-noise-spec compliant signatures on static keys as well as the + `/noise` protocol upgrade, hence providing a libp2p-noise-spec compliant + `XX` handshake. `IK` and `IX` are still supported with `X25519Spec` + though not guaranteed to be interoperable with other libp2p + implementations as these handshake patterns are not currently + included in the libp2p-noise-spec. The `X25519Spec` implementation + will eventually replace the current `X25519` implementation, with + the former being removed. To upgrade without interruptions, you may + temporarily include `NoiseConfig`s for both implementations as + alternatives in your transport upgrade pipeline. + - `libp2p-kad`: Consider fixed (K_VALUE) amount of peers at closest query initialization. Unless `KademliaConfig::set_replication_factor` is used change has no effect. [PR 1536](https://github.com/libp2p/rust-libp2p/pull/1536) + - `libp2p-tcp`: On listeners started with an IPv6 multi-address the socket option `IPV6_V6ONLY` is set to true. Instead of relying on IPv4-mapped IPv6 address support, two listeners can be started if IPv4 and IPv6 should both diff --git a/protocols/noise/src/io/handshake.rs b/protocols/noise/src/io/handshake.rs index a8982354..047951ed 100644 --- a/protocols/noise/src/io/handshake.rs +++ b/protocols/noise/src/io/handshake.rs @@ -365,10 +365,10 @@ where let mut payload_buf = vec![0; len]; state.io.read_exact(&mut payload_buf).await?; - let pb = payload_proto::Identity::decode(&payload_buf[..])?; + let pb = payload_proto::NoiseHandshakePayload::decode(&payload_buf[..])?; - if !pb.pubkey.is_empty() { - let pk = identity::PublicKey::from_protobuf_encoding(&pb.pubkey) + if !pb.identity_key.is_empty() { + let pk = identity::PublicKey::from_protobuf_encoding(&pb.identity_key) .map_err(|_| NoiseError::InvalidKey)?; if let Some(ref k) = state.id_remote_pubkey { if k != &pk { @@ -377,8 +377,8 @@ where } state.id_remote_pubkey = Some(pk); } - if !pb.signature.is_empty() { - state.dh_remote_pubkey_sig = Some(pb.signature); + if !pb.identity_sig.is_empty() { + state.dh_remote_pubkey_sig = Some(pb.identity_sig); } Ok(()) @@ -389,12 +389,12 @@ async fn send_identity(state: &mut State) -> Result<(), NoiseError> where T: AsyncWrite + Unpin, { - let mut pb = payload_proto::Identity::default(); + let mut pb = payload_proto::NoiseHandshakePayload::default(); if state.send_identity { - pb.pubkey = state.identity.public.clone().into_protobuf_encoding() + pb.identity_key = state.identity.public.clone().into_protobuf_encoding() } if let Some(ref sig) = state.identity.signature { - pb.signature = sig.clone() + pb.identity_sig = sig.clone() } let mut buf = Vec::with_capacity(pb.encoded_len()); pb.encode(&mut buf).expect("Vec provides capacity as needed"); diff --git a/protocols/noise/src/io/handshake/payload.proto b/protocols/noise/src/io/handshake/payload.proto index 51b79645..1893dc55 100644 --- a/protocols/noise/src/io/handshake/payload.proto +++ b/protocols/noise/src/io/handshake/payload.proto @@ -4,8 +4,8 @@ package payload.proto; // Payloads for Noise handshake messages. -message Identity { - bytes pubkey = 1; - bytes signature = 2; +message NoiseHandshakePayload { + bytes identity_key = 1; + bytes identity_sig = 2; + bytes data = 3; } - diff --git a/protocols/noise/src/lib.rs b/protocols/noise/src/lib.rs index da9794fc..f73a9e9a 100644 --- a/protocols/noise/src/lib.rs +++ b/protocols/noise/src/lib.rs @@ -27,6 +27,9 @@ //! implementations for various noise handshake patterns (currently `IK`, `IX`, and `XX`) //! over a particular choice of Diffie–Hellman key agreement (currently only X25519). //! +//! > **Note**: Only the `XX` handshake pattern is currently guaranteed to provide +//! > interoperability with other libp2p implementations. +//! //! All upgrades produce as output a pair, consisting of the remote's static public key //! and a `NoiseOutput` which represents the established cryptographic session with the //! remote, implementing `futures::io::AsyncRead` and `futures::io::AsyncWrite`. @@ -38,11 +41,11 @@ //! ``` //! use libp2p_core::{identity, Transport, upgrade}; //! use libp2p_tcp::TcpConfig; -//! use libp2p_noise::{Keypair, X25519, NoiseConfig}; +//! use libp2p_noise::{Keypair, X25519Spec, NoiseConfig}; //! //! # fn main() { //! let id_keys = identity::Keypair::generate_ed25519(); -//! let dh_keys = Keypair::::new().into_authentic(&id_keys).unwrap(); +//! let dh_keys = Keypair::::new().into_authentic(&id_keys).unwrap(); //! let noise = NoiseConfig::xx(dh_keys).into_authenticated(); //! let builder = TcpConfig::new().upgrade(upgrade::Version::V1).authenticate(noise); //! // let transport = builder.multiplex(...); @@ -60,7 +63,8 @@ pub use io::NoiseOutput; pub use io::handshake; pub use io::handshake::{Handshake, RemoteIdentity, IdentityExchange}; pub use protocol::{Keypair, AuthenticKeypair, KeypairIdentity, PublicKey, SecretKey}; -pub use protocol::{Protocol, ProtocolParams, x25519::X25519, IX, IK, XX}; +pub use protocol::{Protocol, ProtocolParams, IX, IK, XX}; +pub use protocol::{x25519::X25519, x25519_spec::X25519Spec}; use futures::prelude::*; use libp2p_core::{identity, PeerId, UpgradeInfo, InboundUpgrade, OutboundUpgrade}; diff --git a/protocols/noise/src/protocol.rs b/protocols/noise/src/protocol.rs index 1fd9fa03..5844dbc8 100644 --- a/protocols/noise/src/protocol.rs +++ b/protocols/noise/src/protocol.rs @@ -21,6 +21,7 @@ //! Components of a Noise protocol. pub mod x25519; +pub mod x25519_spec; use crate::NoiseError; use libp2p_core::identity; @@ -71,6 +72,7 @@ pub trait Protocol { /// /// The trivial case is when the keys are byte for byte identical. #[allow(unused_variables)] + #[deprecated] fn linked(id_pk: &identity::PublicKey, dh_pk: &PublicKey) -> bool { false } @@ -87,6 +89,7 @@ pub trait Protocol { /// without a signature, otherwise a signature over the static DH public key /// must be given and is verified with the public identity key, establishing /// the authenticity of the static DH public key w.r.t. the public identity key. + #[allow(deprecated)] fn verify(id_pk: &identity::PublicKey, dh_pk: &PublicKey, sig: &Option>) -> bool where C: AsRef<[u8]> @@ -95,6 +98,13 @@ pub trait Protocol { || sig.as_ref().map_or(false, |s| id_pk.verify(dh_pk.as_ref(), s)) } + + fn sign(id_keys: &identity::Keypair, dh_pk: &PublicKey) -> Result, NoiseError> + where + C: AsRef<[u8]> + { + Ok(id_keys.sign(dh_pk.as_ref())?) + } } /// DH keypair. @@ -151,9 +161,10 @@ impl Keypair { /// is authentic w.r.t. the given identity keypair, by signing the DH public key. pub fn into_authentic(self, id_keys: &identity::Keypair) -> Result, NoiseError> where - T: AsRef<[u8]> + T: AsRef<[u8]>, + T: Protocol { - let sig = id_keys.sign(self.public.as_ref())?; + let sig = T::sign(id_keys, &self.public)?; let identity = KeypairIdentity { public: id_keys.public(), diff --git a/protocols/noise/src/protocol/x25519.rs b/protocols/noise/src/protocol/x25519.rs index 8a19971d..80bba174 100644 --- a/protocols/noise/src/protocol/x25519.rs +++ b/protocols/noise/src/protocol/x25519.rs @@ -18,7 +18,10 @@ // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER // DEALINGS IN THE SOFTWARE. -//! Noise protocols based on X25519. +//! Legacy Noise protocols based on X25519. +//! +//! **Note**: This set of protocols is not interoperable with other +//! libp2p implementations. use crate::{NoiseConfig, NoiseError, Protocol, ProtocolParams}; use curve25519_dalek::edwards::CompressedEdwardsY; @@ -92,7 +95,11 @@ impl UpgradeInfo for NoiseConfig { } } -/// Noise protocols for X25519. +/// Legacy Noise protocol for X25519. +/// +/// **Note**: This `Protocol` provides no configuration that +/// is interoperable with other libp2p implementations. +/// See [`crate::X25519Spec`] instead. impl Protocol for X25519 { fn params_ik() -> ProtocolParams { PARAMS_IK.clone() diff --git a/protocols/noise/src/protocol/x25519_spec.rs b/protocols/noise/src/protocol/x25519_spec.rs new file mode 100644 index 00000000..446ff7cc --- /dev/null +++ b/protocols/noise/src/protocol/x25519_spec.rs @@ -0,0 +1,150 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//! [libp2p-noise-spec] compliant Noise protocols based on X25519. +//! +//! [libp2p-noise-spec]: https://github.com/libp2p/specs/tree/master/noise + +use crate::{NoiseConfig, NoiseError, Protocol, ProtocolParams}; +use libp2p_core::UpgradeInfo; +use libp2p_core::identity; +use rand::Rng; +use x25519_dalek::{X25519_BASEPOINT_BYTES, x25519}; +use zeroize::Zeroize; + +use super::{*, x25519::X25519}; + +/// Prefix of static key signatures for domain separation. +const STATIC_KEY_DOMAIN: &str = "noise-libp2p-static-key:"; + +/// A X25519 key. +#[derive(Clone)] +pub struct X25519Spec([u8; 32]); + +impl AsRef<[u8]> for X25519Spec { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + +impl Zeroize for X25519Spec { + fn zeroize(&mut self) { + self.0.zeroize() + } +} + +impl Keypair { + /// Create a new X25519 keypair. + pub fn new() -> Keypair { + let mut sk_bytes = [0u8; 32]; + rand::thread_rng().fill(&mut sk_bytes); + let sk = SecretKey(X25519Spec(sk_bytes)); // Copy + sk_bytes.zeroize(); + Self::from(sk) + } +} + +/// Promote a X25519 secret key into a keypair. +impl From> for Keypair { + fn from(secret: SecretKey) -> Keypair { + let public = PublicKey(X25519Spec(x25519((secret.0).0, X25519_BASEPOINT_BYTES))); + Keypair { secret, public } + } +} + +impl UpgradeInfo for NoiseConfig { + type Info = &'static [u8]; + type InfoIter = std::iter::Once; + + fn protocol_info(&self) -> Self::InfoIter { + std::iter::once(b"/noise") + } +} + +/// Noise protocols for X25519 with libp2p-spec compliant signatures. +/// +/// **Note**: Only the XX handshake pattern is currently guaranteed to be +/// interoperable with other libp2p implementations. +impl Protocol for X25519Spec { + fn params_ik() -> ProtocolParams { + X25519::params_ik() + } + + fn params_ix() -> ProtocolParams { + X25519::params_ix() + } + + fn params_xx() -> ProtocolParams { + X25519::params_xx() + } + + fn public_from_bytes(bytes: &[u8]) -> Result, NoiseError> { + if bytes.len() != 32 { + return Err(NoiseError::InvalidKey) + } + let mut pk = [0u8; 32]; + pk.copy_from_slice(bytes); + Ok(PublicKey(X25519Spec(pk))) + } + + fn verify(id_pk: &identity::PublicKey, dh_pk: &PublicKey, sig: &Option>) -> bool + { + sig.as_ref().map_or(false, |s| { + id_pk.verify(&[STATIC_KEY_DOMAIN.as_bytes(), dh_pk.as_ref()].concat(), s) + }) + } + + fn sign(id_keys: &identity::Keypair, dh_pk: &PublicKey) -> Result, NoiseError> { + Ok(id_keys.sign(&[STATIC_KEY_DOMAIN.as_bytes(), dh_pk.as_ref()].concat())?) + } +} + +#[doc(hidden)] +impl snow::types::Dh for Keypair { + fn name(&self) -> &'static str { "25519" } + fn pub_len(&self) -> usize { 32 } + fn priv_len(&self) -> usize { 32 } + fn pubkey(&self) -> &[u8] { self.public.as_ref() } + fn privkey(&self) -> &[u8] { self.secret.as_ref() } + + fn set(&mut self, sk: &[u8]) { + let mut secret = [0u8; 32]; + secret.copy_from_slice(&sk[..]); + self.secret = SecretKey(X25519Spec(secret)); // Copy + self.public = PublicKey(X25519Spec(x25519(secret, X25519_BASEPOINT_BYTES))); + secret.zeroize(); + } + + fn generate(&mut self, rng: &mut dyn snow::types::Random) { + let mut secret = [0u8; 32]; + rng.fill_bytes(&mut secret); + self.secret = SecretKey(X25519Spec(secret)); // Copy + self.public = PublicKey(X25519Spec(x25519(secret, X25519_BASEPOINT_BYTES))); + secret.zeroize(); + } + + fn dh(&self, pk: &[u8], shared_secret: &mut [u8]) -> Result<(), ()> { + let mut p = [0; 32]; + p.copy_from_slice(&pk[.. 32]); + let ss = x25519((self.secret.0).0, p); + shared_secret[.. 32].copy_from_slice(&ss[..]); + Ok(()) + } +} diff --git a/protocols/noise/tests/smoke.rs b/protocols/noise/tests/smoke.rs index 1ac04491..744d447a 100644 --- a/protocols/noise/tests/smoke.rs +++ b/protocols/noise/tests/smoke.rs @@ -22,7 +22,7 @@ use futures::{future::{self, Either}, prelude::*}; use libp2p_core::identity; use libp2p_core::upgrade::{self, Negotiated, apply_inbound, apply_outbound}; use libp2p_core::transport::{Transport, ListenerEvent}; -use libp2p_noise::{Keypair, X25519, NoiseConfig, RemoteIdentity, NoiseError, NoiseOutput}; +use libp2p_noise::{Keypair, X25519, X25519Spec, NoiseConfig, RemoteIdentity, NoiseError, NoiseOutput}; use libp2p_tcp::{TcpConfig, TcpTransStream}; use log::info; use quickcheck::QuickCheck; @@ -38,6 +38,37 @@ fn core_upgrade_compat() { let _ = TcpConfig::new().upgrade(upgrade::Version::V1).authenticate(noise); } +#[test] +fn xx_spec() { + let _ = env_logger::try_init(); + fn prop(mut messages: Vec) -> bool { + messages.truncate(5); + let server_id = identity::Keypair::generate_ed25519(); + let client_id = identity::Keypair::generate_ed25519(); + + let server_id_public = server_id.public(); + let client_id_public = client_id.public(); + + let server_dh = Keypair::::new().into_authentic(&server_id).unwrap(); + let server_transport = TcpConfig::new() + .and_then(move |output, endpoint| { + upgrade::apply(output, NoiseConfig::xx(server_dh), endpoint, upgrade::Version::V1) + }) + .and_then(move |out, _| expect_identity(out, &client_id_public)); + + let client_dh = Keypair::::new().into_authentic(&client_id).unwrap(); + let client_transport = TcpConfig::new() + .and_then(move |output, endpoint| { + upgrade::apply(output, NoiseConfig::xx(client_dh), endpoint, upgrade::Version::V1) + }) + .and_then(move |out, _| expect_identity(out, &server_id_public)); + + run(server_transport, client_transport, messages); + true + } + QuickCheck::new().max_tests(30).quickcheck(prop as fn(Vec) -> bool) +} + #[test] fn xx() { let _ = env_logger::try_init(); @@ -144,15 +175,15 @@ fn ik_xx() { QuickCheck::new().max_tests(30).quickcheck(prop as fn(Vec) -> bool) } -type Output = (RemoteIdentity, NoiseOutput>); +type Output = (RemoteIdentity, NoiseOutput>); -fn run(server_transport: T, client_transport: U, messages: I) +fn run(server_transport: T, client_transport: U, messages: I) where - T: Transport, + T: Transport>, T::Dial: Send + 'static, T::Listener: Send + Unpin + 'static, T::ListenerUpgrade: Send + 'static, - U: Transport, + U: Transport>, U::Dial: Send + 'static, U::Listener: Send + 'static, U::ListenerUpgrade: Send + 'static, @@ -218,8 +249,8 @@ where }) } -fn expect_identity(output: Output, pk: &identity::PublicKey) - -> impl Future> +fn expect_identity(output: Output, pk: &identity::PublicKey) + -> impl Future, NoiseError>> { match output.0 { RemoteIdentity::IdentityKey(ref k) if k == pk => future::ok(output), From 44c0c76981ff69154c05fc0d8e0ef839ed05774f Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 12 May 2020 12:23:40 +0200 Subject: [PATCH 06/25] protocols/kad/query/fixed: Decrease num_waiting on failure (#1572) Make sure to decrease `num_waiting` when being notified of a peer failure to allow an additional peer to be queried. Given that `FixedPeersIter` is initialized with `replication_factor` by `QueryPool` this bug will not surface today. --- protocols/kad/src/query/peers/fixed.rs | 30 +++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/protocols/kad/src/query/peers/fixed.rs b/protocols/kad/src/query/peers/fixed.rs index 407d5548..8a48aea7 100644 --- a/protocols/kad/src/query/peers/fixed.rs +++ b/protocols/kad/src/query/peers/fixed.rs @@ -76,9 +76,10 @@ impl FixedPeersIter { } pub fn on_failure(&mut self, peer: &PeerId) { - if let State::Waiting { .. } = &self.state { + if let State::Waiting { num_waiting } = &mut self.state { if let Some(state @ PeerState::Waiting) = self.peers.get_mut(peer) { *state = PeerState::Failed; + *num_waiting -= 1; } } } @@ -133,3 +134,30 @@ impl FixedPeersIter { } } +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn decrease_num_waiting_on_failure() { + let mut iter = FixedPeersIter::new(vec![PeerId::random(), PeerId::random()], 1); + + match iter.next() { + PeersIterState::Waiting(Some(peer)) => { + let peer = peer.into_owned(); + iter.on_failure(&peer); + }, + _ => panic!("Expected iterator to yield peer."), + } + + match iter.next() { + PeersIterState::Waiting(Some(_)) => {}, + PeersIterState::WaitingAtCapacity => panic!( + "Expected iterator to return another peer given that the \ + previous `on_failure` call should have allowed another peer \ + to be queried.", + ), + _ => panic!("Expected iterator to yield peer."), + } + } +} From 5ba7c4831bd3558e04d874525c363e98a0c69b64 Mon Sep 17 00:00:00 2001 From: Roman Borschel Date: Tue, 12 May 2020 13:10:18 +0200 Subject: [PATCH 07/25] Permit concurrent dialing attempts per peer. (#1506) * Permit concurrent dialing attempts per peer. This is a follow-up to https://github.com/libp2p/rust-libp2p/pull/1440 and relates to https://github.com/libp2p/rust-libp2p/issues/925. This change permits multiple dialing attempts per peer. Note though that `libp2p-swarm` does not yet make use of this ability, retaining the current behaviour. The essence of the changes are that the `Peer` API now provides `Peer::dial()`, i.e. regardless of the state in which the peer is. A dialing attempt is always made up of one or more addresses tried sequentially, as before, but now there can be multiple dialing attempts per peer. A configurable per-peer limit for outgoing connections and thus concurrent dialing attempts is also included. * Introduce `DialError` in `libp2p-swarm`. For a cleaner API and to treat the case of no addresses for a peer as an error, such that a `NetworkBehaviourAction::DialPeer` request is always matched up with either `inject_connection_established` or `inject_dial_error`. * Fix rustdoc link. * Add `DialPeerCondition::Always`. * Adapt to master. * Update changelog. --- CHANGELOG.md | 4 + core/src/connection/pool.rs | 34 ++- core/src/network.rs | 74 +++--- core/src/network/peer.rs | 414 +++++++++++++++++++------------ core/tests/network_dial_error.rs | 153 +++++++----- swarm/src/behaviour.rs | 8 +- swarm/src/lib.rs | 123 +++++---- 7 files changed, 473 insertions(+), 337 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b585911a..fcd438ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Version ??? +- `libp2p-core`, `libp2p-swarm`: Added support for multiple dialing + attempts per peer, with a configurable limit. + [PR 1506](https://github.com/libp2p/rust-libp2p/pull/1506) + - `libp2p-noise`: Added the `X25519Spec` protocol suite which uses libp2p-noise-spec compliant signatures on static keys as well as the `/noise` protocol upgrade, hence providing a libp2p-noise-spec compliant diff --git a/core/src/connection/pool.rs b/core/src/connection/pool.rs index f8c9e1a7..7da7efa8 100644 --- a/core/src/connection/pool.rs +++ b/core/src/connection/pool.rs @@ -225,12 +225,7 @@ where TPeerId: Clone + Send + 'static, { let endpoint = info.to_connected_point(); - if let Some(limit) = self.limits.max_incoming { - let current = self.iter_pending_incoming().count(); - if current >= limit { - return Err(ConnectionLimit { limit, current }) - } - } + self.limits.check_incoming(|| self.iter_pending_incoming().count())?; Ok(self.add_pending(future, handler, endpoint, None)) } @@ -267,6 +262,11 @@ where TPeerId: Clone + Send + 'static, { self.limits.check_outgoing(|| self.iter_pending_outgoing().count())?; + + if let Some(peer) = &info.peer_id { + self.limits.check_outgoing_per_peer(|| self.num_peer_outgoing(peer))?; + } + let endpoint = info.to_connected_point(); Ok(self.add_pending(future, handler, endpoint, info.peer_id.cloned())) } @@ -465,6 +465,13 @@ where self.established.get(peer).map_or(0, |conns| conns.len()) } + /// Counts the number of pending outgoing connections to the given peer. + pub fn num_peer_outgoing(&self, peer: &TPeerId) -> usize { + self.iter_pending_outgoing() + .filter(|info| info.peer_id == Some(peer)) + .count() + } + /// Returns an iterator over all established connections of `peer`. pub fn iter_peer_established<'a>(&'a mut self, peer: &TPeerId) -> EstablishedConnectionIter<'a, @@ -837,6 +844,7 @@ pub struct PoolLimits { pub max_outgoing: Option, pub max_incoming: Option, pub max_established_per_peer: Option, + pub max_outgoing_per_peer: Option, } impl PoolLimits { @@ -854,6 +862,20 @@ impl PoolLimits { Self::check(current, self.max_outgoing) } + fn check_incoming(&self, current: F) -> Result<(), ConnectionLimit> + where + F: FnOnce() -> usize + { + Self::check(current, self.max_incoming) + } + + fn check_outgoing_per_peer(&self, current: F) -> Result<(), ConnectionLimit> + where + F: FnOnce() -> usize + { + Self::check(current, self.max_outgoing_per_peer) + } + fn check(current: F, limit: Option) -> Result<(), ConnectionLimit> where F: FnOnce() -> usize diff --git a/core/src/network.rs b/core/src/network.rs index ccb21e84..73240abd 100644 --- a/core/src/network.rs +++ b/core/src/network.rs @@ -50,6 +50,7 @@ use crate::{ }; use fnv::{FnvHashMap}; use futures::{prelude::*, future}; +use smallvec::SmallVec; use std::{ collections::hash_map, convert::TryFrom as _, @@ -78,21 +79,17 @@ where /// The ongoing dialing attempts. /// - /// The `Network` enforces a single ongoing dialing attempt per peer, - /// even if multiple (established) connections per peer are allowed. - /// However, a single dialing attempt operates on a list of addresses - /// to connect to, which can be extended with new addresses while - /// the connection attempt is still in progress. Thereby each - /// dialing attempt is associated with a new connection and hence a new - /// connection ID. + /// There may be multiple ongoing dialing attempts to the same peer. + /// Each dialing attempt is associated with a new connection and hence + /// a new connection ID. /// /// > **Note**: `dialing` must be consistent with the pending outgoing /// > connections in `pool`. That is, for every entry in `dialing` /// > there must exist a pending outgoing connection in `pool` with /// > the same connection ID. This is ensured by the implementation of /// > `Network` (see `dial_peer_impl` and `on_connection_failed`) - /// > together with the implementation of `DialingConnection::abort`. - dialing: FnvHashMap, + /// > together with the implementation of `DialingAttempt::abort`. + dialing: FnvHashMap>, } impl fmt::Debug for @@ -381,8 +378,11 @@ where Poll::Pending => return Poll::Pending, Poll::Ready(PoolEvent::ConnectionEstablished { connection, num_established }) => { match self.dialing.entry(connection.peer_id().clone()) { - hash_map::Entry::Occupied(e) if e.get().id == connection.id() => { - e.remove(); + hash_map::Entry::Occupied(mut e) => { + e.get_mut().retain(|s| s.current.0 != connection.id()); + if e.get().is_empty() { + e.remove(); + } }, _ => {} } @@ -453,7 +453,7 @@ fn dial_peer_impl::Error, TConnInfo, TPeerId>, - dialing: &mut FnvHashMap, + dialing: &mut FnvHashMap>, opts: DialingOpts ) -> Result where @@ -489,14 +489,12 @@ where }; if let Ok(id) = &result { - let former = dialing.insert(opts.peer, - peer::DialingAttempt { - id: *id, - current: opts.address, - next: opts.remaining, + dialing.entry(opts.peer).or_default().push( + peer::DialingState { + current: (*id, opts.address), + remaining: opts.remaining, }, ); - debug_assert!(former.is_none()); } result @@ -508,7 +506,7 @@ where /// If the failed connection attempt was a dialing attempt and there /// are more addresses to try, new `DialingOpts` are returned. fn on_connection_failed<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId>( - dialing: &mut FnvHashMap, + dialing: &mut FnvHashMap>, id: ConnectionId, endpoint: ConnectedPoint, error: PendingConnectionError, @@ -521,27 +519,34 @@ where TPeerId: Eq + Hash + Clone, { // Check if the failed connection is associated with a dialing attempt. - // TODO: could be more optimal than iterating over everything - let dialing_peer = dialing.iter() // (1) - .find(|(_, a)| a.id == id) - .map(|(p, _)| p.clone()); + let dialing_failed = dialing.iter_mut() + .find_map(|(peer, attempts)| { + if let Some(pos) = attempts.iter().position(|s| s.current.0 == id) { + let attempt = attempts.remove(pos); + let last = attempts.is_empty(); + Some((peer.clone(), attempt, last)) + } else { + None + } + }); - if let Some(peer_id) = dialing_peer { - // A pending outgoing connection to a known peer failed. - let mut attempt = dialing.remove(&peer_id).expect("by (1)"); + if let Some((peer_id, mut attempt, last)) = dialing_failed { + if last { + dialing.remove(&peer_id); + } - let num_remain = u32::try_from(attempt.next.len()).unwrap(); - let failed_addr = attempt.current.clone(); + let num_remain = u32::try_from(attempt.remaining.len()).unwrap(); + let failed_addr = attempt.current.1.clone(); let (opts, attempts_remaining) = if num_remain > 0 { if let Some(handler) = handler { - let next_attempt = attempt.next.remove(0); + let next_attempt = attempt.remaining.remove(0); let opts = DialingOpts { peer: peer_id.clone(), handler, address: next_attempt, - remaining: attempt.next + remaining: attempt.remaining }; (Some(opts), num_remain) } else { @@ -581,9 +586,13 @@ where /// Information about the network obtained by [`Network::info()`]. #[derive(Clone, Debug)] pub struct NetworkInfo { + /// The total number of connected peers. pub num_peers: usize, + /// The total number of connections, both established and pending. pub num_connections: usize, + /// The total number of pending connections, both incoming and outgoing. pub num_connections_pending: usize, + /// The total number of established connections. pub num_connections_established: usize, } @@ -633,4 +642,9 @@ impl NetworkConfig { self.pool_limits.max_established_per_peer = Some(n); self } + + pub fn set_outgoing_per_peer_limit(&mut self, n: usize) -> &mut Self { + self.pool_limits.max_outgoing_per_peer = Some(n); + self + } } diff --git a/core/src/network/peer.rs b/core/src/network/peer.rs index b06be772..8f9dd099 100644 --- a/core/src/network/peer.rs +++ b/core/src/network/peer.rs @@ -35,8 +35,11 @@ use crate::{ IntoConnectionHandler, PendingConnection, Substream, + pool::Pool, }, }; +use fnv::FnvHashMap; +use smallvec::SmallVec; use std::{ collections::hash_map, error, @@ -47,6 +50,10 @@ use super::{Network, DialingOpts}; /// The possible representations of a peer in a [`Network`], as /// seen by the local node. +/// +/// > **Note**: In any state there may always be a pending incoming +/// > connection attempt from the peer, however, the remote identity +/// > of a peer is only known once a connection is fully established. pub enum Peer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> where TTrans: Transport, @@ -63,10 +70,6 @@ where /// There exists no established connection to the peer and there is /// currently no ongoing dialing (i.e. outgoing connection) attempt /// in progress. - /// - /// > **Note**: In this state there may always be a pending incoming - /// > connection attempt from the peer, however, the remote identity - /// > of a peer is only known once a connection is fully established. Disconnected(DisconnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId>), /// The peer represents the local node. @@ -82,20 +85,20 @@ where TPeerId: fmt::Debug + Eq + Hash, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { - match *self { - Peer::Connected(ConnectedPeer { ref peer_id, .. }) => { + match self { + Peer::Connected(p) => { f.debug_struct("Connected") - .field("peer_id", peer_id) + .field("peer", &p) .finish() } - Peer::Dialing(DialingPeer { ref peer_id, .. } ) => { - f.debug_struct("DialingPeer") - .field("peer_id", peer_id) + Peer::Dialing(p) => { + f.debug_struct("Dialing") + .field("peer", &p) .finish() } - Peer::Disconnected(DisconnectedPeer { ref peer_id, .. }) => { + Peer::Disconnected(p) => { f.debug_struct("Disconnected") - .field("peer_id", peer_id) + .field("peer", &p) .finish() } Peer::Local => { @@ -164,12 +167,11 @@ where TTrans::Dial: Send + 'static, TMuxer: StreamMuxer + Send + Sync + 'static, TMuxer::OutboundSubstream: Send, - TMuxer::Substream: Send, TInEvent: Send + 'static, TOutEvent: Send + 'static, THandler: IntoConnectionHandler + Send + 'static, - THandler::Handler: ConnectionHandler, InEvent = TInEvent, OutEvent = TOutEvent> + Send + 'static, - ::OutboundOpenInfo: Send + 'static, // TODO: shouldn't be necessary + THandler::Handler: ConnectionHandler, InEvent = TInEvent, OutEvent = TOutEvent> + Send, + ::OutboundOpenInfo: Send, ::Error: error::Error + Send + 'static, TConnInfo: fmt::Debug + ConnectionInfo + Send + 'static, TPeerId: Eq + Hash + Clone + Send + 'static, @@ -208,7 +210,41 @@ where } } - /// Converts the peer into a `ConnectedPeer`, if there an established connection exists. + /// Initiates a new dialing attempt to this peer using the given addresses. + /// + /// The connection ID of the first connection attempt, i.e. to `address`, + /// is returned, together with a [`DialingPeer`] for further use. The + /// `remaining` addresses are tried in order in subsequent connection + /// attempts in the context of the same dialing attempt, if the connection + /// attempt to the first address fails. + pub fn dial(self, address: Multiaddr, remaining: I, handler: THandler) + -> Result< + (ConnectionId, DialingPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId>), + ConnectionLimit + > + where + I: IntoIterator, + { + let (peer_id, network) = match self { + Peer::Connected(p) => (p.peer_id, p.network), + Peer::Dialing(p) => (p.peer_id, p.network), + Peer::Disconnected(p) => (p.peer_id, p.network), + Peer::Local => return Err(ConnectionLimit { current: 0, limit: 0 }) + }; + + let id = network.dial_peer(DialingOpts { + peer: peer_id.clone(), + handler, + address, + remaining: remaining.into_iter().collect(), + })?; + + Ok((id, DialingPeer { network, peer_id })) + } + + /// Converts the peer into a `ConnectedPeer`, if an established connection exists. + /// + /// Succeeds if the there is at least one established connection to the peer. pub fn into_connected(self) -> Option< ConnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> > { @@ -221,6 +257,8 @@ where } /// Converts the peer into a `DialingPeer`, if a dialing attempt exists. + /// + /// Succeeds if the there is at least one pending outgoing connection to the peer. pub fn into_dialing(self) -> Option< DialingPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> > { @@ -245,7 +283,8 @@ where } /// The representation of a peer in a [`Network`] to whom at least -/// one established connection exists. +/// one established connection exists. There may also be additional ongoing +/// dialing attempts to the peer. pub struct ConnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> where TTrans: Transport, @@ -267,57 +306,12 @@ where &self.peer_id } - /// Attempts to establish a new connection to this peer using the given addresses, - /// if there is currently no ongoing dialing attempt. - /// - /// Existing established connections are not affected. - /// - /// > **Note**: If there is an ongoing dialing attempt, a `DialingPeer` - /// > is returned with the given addresses and handler being ignored. - /// > You may want to check [`ConnectedPeer::is_dialing`] first. - pub fn connect(self, address: Multiaddr, remaining: I, handler: THandler) - -> Result, - ConnectionLimit> - where - I: IntoIterator, - THandler: Send + 'static, - THandler::Handler: Send, - ::Error: error::Error + Send, - ::OutboundOpenInfo: Send, - THandler::Handler: ConnectionHandler, InEvent = TInEvent, OutEvent = TOutEvent> + Send, - TTrans: Transport + Clone, - TTrans::Error: Send + 'static, - TTrans::Dial: Send + 'static, - TMuxer: StreamMuxer + Send + Sync + 'static, - TMuxer::OutboundSubstream: Send, - TMuxer::Substream: Send, - TConnInfo: fmt::Debug + Send + 'static, - TPeerId: Eq + Hash + Clone + Send + 'static, - TInEvent: Send + 'static, - TOutEvent: Send + 'static, - - { - if self.network.dialing.contains_key(&self.peer_id) { - let peer = DialingPeer { - network: self.network, - peer_id: self.peer_id - }; - Ok(peer) - } else { - self.network.dial_peer(DialingOpts { - peer: self.peer_id.clone(), - handler, - address, - remaining: remaining.into_iter().collect(), - })?; - Ok(DialingPeer { - network: self.network, - peer_id: self.peer_id, - }) - } + /// Returns the `ConnectedPeer` into a `Peer`. + pub fn into_peer(self) -> Peer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> { + Peer::Connected(self) } - /// Obtains an existing connection to the peer. + /// Obtains an established connection to the peer by ID. pub fn connection<'b>(&'b mut self, id: ConnectionId) -> Option> { @@ -348,7 +342,7 @@ where } } - /// Gets an iterator over all established connections of the peer. + /// Gets an iterator over all established connections to the peer. pub fn connections<'b>(&'b mut self) -> EstablishedConnectionIter<'b, impl Iterator, @@ -386,11 +380,13 @@ impl<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> fmt::Debug f where TTrans: Transport, THandler: IntoConnectionHandler, - TPeerId: fmt::Debug, + TPeerId: Eq + Hash + fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { f.debug_struct("ConnectedPeer") .field("peer_id", &self.peer_id) + .field("established", &self.network.pool.iter_peer_established_info(&self.peer_id)) + .field("attempts", &self.network.dialing.get(&self.peer_id)) .finish() } } @@ -419,8 +415,16 @@ where &self.peer_id } - /// Disconnects from this peer, closing all pending connections. - pub fn disconnect(self) -> DisconnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> { + /// Returns the `DialingPeer` into a `Peer`. + pub fn into_peer(self) -> Peer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> { + Peer::Dialing(self) + } + + /// Disconnects from this peer, closing all established connections and + /// aborting all dialing attempts. + pub fn disconnect(self) + -> DisconnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> + { self.network.disconnect(&self.peer_id); DisconnectedPeer { network: self.network, peer_id: self.peer_id } } @@ -443,20 +447,50 @@ where } } - /// Obtains the connection that is currently being established. - pub fn connection<'b>(&'b mut self) -> DialingConnection<'b, TInEvent, TConnInfo, TPeerId> { - let attempt = match self.network.dialing.entry(self.peer_id.clone()) { - hash_map::Entry::Occupied(e) => e, - _ => unreachable!("By `Peer::new` and the definition of `DialingPeer`.") - }; - - let inner = self.network.pool - .get_outgoing(attempt.get().id) - .expect("By consistency of `network.pool` with `network.dialing`."); - - DialingConnection { - inner, dialing: attempt, peer_id: &self.peer_id + /// Obtains a dialing attempt to the peer by connection ID of + /// the current connection attempt. + pub fn attempt<'b>(&'b mut self, id: ConnectionId) + -> Option> + { + if let hash_map::Entry::Occupied(attempts) = self.network.dialing.entry(self.peer_id.clone()) { + if let Some(pos) = attempts.get().iter().position(|s| s.current.0 == id) { + if let Some(inner) = self.network.pool.get_outgoing(id) { + return Some(DialingAttempt { pos, inner, attempts }) + } + } } + None + } + + /// The number of ongoing dialing attempts, i.e. pending outgoing connections + /// to this peer. + pub fn num_attempts(&self) -> usize { + self.network.pool.num_peer_outgoing(&self.peer_id) + } + + /// Gets an iterator over all dialing (i.e. pending outgoing) connections to the peer. + pub fn attempts<'b>(&'b mut self) + -> DialingAttemptIter<'b, + TInEvent, + TOutEvent, + THandler, + TTrans::Error, + ::Error, + TConnInfo, + TPeerId> + { + DialingAttemptIter::new(&self.peer_id, &mut self.network.pool, &mut self.network.dialing) + } + + /// Obtains some dialing connection to the peer. + /// + /// At least one dialing connection is guaranteed to exist on a `DialingPeer`. + pub fn some_attempt<'b>(&'b mut self) + -> DialingAttempt<'b, TInEvent, TConnInfo, TPeerId> + { + self.attempts() + .into_first() + .expect("By `Peer::new` and the definition of `DialingPeer`.") } } @@ -465,11 +499,13 @@ impl<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> fmt::Debug f where TTrans: Transport, THandler: IntoConnectionHandler, - TPeerId: fmt::Debug, + TPeerId: Eq + Hash + fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { f.debug_struct("DialingPeer") .field("peer_id", &self.peer_id) + .field("established", &self.network.pool.iter_peer_established_info(&self.peer_id)) + .field("attempts", &self.network.dialing.get(&self.peer_id)) .finish() } } @@ -500,46 +536,19 @@ where } } -impl<'a, TTrans, TInEvent, TOutEvent, TMuxer, THandler, TConnInfo, TPeerId> +impl<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> DisconnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> where - TTrans: Transport + Clone, - TTrans::Error: Send + 'static, - TTrans::Dial: Send + 'static, - TMuxer: StreamMuxer + Send + Sync + 'static, - TMuxer::OutboundSubstream: Send, - TMuxer::Substream: Send, - THandler: IntoConnectionHandler + Send + 'static, - THandler::Handler: ConnectionHandler, InEvent = TInEvent, OutEvent = TOutEvent> + Send, - ::OutboundOpenInfo: Send, - ::Error: error::Error + Send, - TInEvent: Send + 'static, - TOutEvent: Send + 'static, + TTrans: Transport, + THandler: IntoConnectionHandler, { pub fn id(&self) -> &TPeerId { &self.peer_id } - /// Attempts to connect to this peer using the given addresses. - pub fn connect(self, first: Multiaddr, rest: TIter, handler: THandler) - -> Result, - ConnectionLimit> - where - TIter: IntoIterator, - TConnInfo: fmt::Debug + ConnectionInfo + Send + 'static, - TPeerId: Eq + Hash + Clone + Send + 'static, - { - self.network.dial_peer(DialingOpts { - peer: self.peer_id.clone(), - handler, - address: first, - remaining: rest.into_iter().collect(), - })?; - Ok(DialingPeer { - network: self.network, - peer_id: self.peer_id, - }) - + /// Returns the `DisconnectedPeer` into a `Peer`. + pub fn into_peer(self) -> Peer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId> { + Peer::Disconnected(self) } /// Moves the peer into a connected state by supplying an existing @@ -550,8 +559,7 @@ where /// # Panics /// /// Panics if `connected.peer_id()` does not identify the current peer. - /// - pub fn set_connected( + pub fn set_connected( self, connected: Connected, connection: Connection, @@ -559,8 +567,17 @@ where ConnectedPeer<'a, TTrans, TInEvent, TOutEvent, THandler, TConnInfo, TPeerId>, ConnectionLimit > where + TInEvent: Send + 'static, + TOutEvent: Send + 'static, + THandler: Send + 'static, + TTrans::Error: Send + 'static, + THandler::Handler: ConnectionHandler, InEvent = TInEvent, OutEvent = TOutEvent> + Send, + ::OutboundOpenInfo: Send, + ::Error: error::Error + Send + 'static, TConnInfo: fmt::Debug + ConnectionInfo + Clone + Send + 'static, - TPeerId: Eq + Hash + Clone + fmt::Debug, + TPeerId: Eq + Hash + Clone + Send + fmt::Debug + 'static, + TMuxer: StreamMuxer + Send + Sync + 'static, + TMuxer::OutboundSubstream: Send, { if connected.peer_id() != &self.peer_id { panic!("Invalid peer ID given: {:?}. Expected: {:?}", connected.peer_id(), self.peer_id) @@ -574,71 +591,142 @@ where } } -/// Attempt to reach a peer. +/// The (internal) state of a `DialingAttempt`, tracking the +/// current connection attempt as well as remaining addresses. #[derive(Debug, Clone)] -pub(super) struct DialingAttempt { - /// Identifier for the reach attempt. - pub(super) id: ConnectionId, - /// Multiaddr currently being attempted. - pub(super) current: Multiaddr, +pub(super) struct DialingState { + /// The ID and (remote) address of the current connection attempt. + pub(super) current: (ConnectionId, Multiaddr), /// Multiaddresses to attempt if the current one fails. - pub(super) next: Vec, + pub(super) remaining: Vec, } -/// A `DialingConnection` is a [`PendingConnection`] where the local peer -/// has the role of the dialer (i.e. initiator) and the (expected) remote -/// peer ID is known. -pub struct DialingConnection<'a, TInEvent, TConnInfo, TPeerId> { - peer_id: &'a TPeerId, +/// A `DialingAttempt` is an ongoing outgoing connection attempt to +/// a known / expected remote peer ID and a list of alternative addresses +/// to connect to, if the current connection attempt fails. +pub struct DialingAttempt<'a, TInEvent, TConnInfo, TPeerId> { + /// The underlying pending connection in the `Pool`. inner: PendingConnection<'a, TInEvent, TConnInfo, TPeerId>, - dialing: hash_map::OccupiedEntry<'a, TPeerId, DialingAttempt>, + /// All current dialing attempts of the peer. + attempts: hash_map::OccupiedEntry<'a, TPeerId, SmallVec<[DialingState; 10]>>, + /// The position of the current `DialingState` of this connection in the `attempts`. + pos: usize, } impl<'a, TInEvent, TConnInfo, TPeerId> - DialingConnection<'a, TInEvent, TConnInfo, TPeerId> + DialingAttempt<'a, TInEvent, TConnInfo, TPeerId> { - /// Returns the local connection ID. + /// Returns the ID of the current connection attempt. pub fn id(&self) -> ConnectionId { self.inner.id() } - /// Returns the (expected) peer ID of the ongoing connection attempt. + /// Returns the (expected) peer ID of the dialing attempt. pub fn peer_id(&self) -> &TPeerId { - self.peer_id + self.attempts.key() } - /// Returns information about this endpoint of the connection attempt. - pub fn endpoint(&self) -> &ConnectedPoint { - self.inner.endpoint() - } - - /// Aborts the connection attempt. - pub fn abort(self) - where - TPeerId: Eq + Hash + Clone, - { - self.dialing.remove(); - self.inner.abort(); - } - - /// Adds new candidate addresses to the end of the addresses used - /// in the ongoing dialing process. - /// - /// Duplicates are ignored. - pub fn add_addresses(&mut self, addrs: impl IntoIterator) { - for addr in addrs { - self.add_address(addr); + /// Returns the remote address of the current connection attempt. + pub fn address(&self) -> &Multiaddr { + match self.inner.endpoint() { + ConnectedPoint::Dialer { address } => address, + ConnectedPoint::Listener { .. } => unreachable!("by definition of a `DialingAttempt`.") } } - /// Adds an address to the end of the addresses used in the ongoing - /// dialing process. + /// Aborts the dialing attempt. /// - /// Duplicates are ignored. + /// Aborting a dialing attempt involves aborting the current connection + /// attempt and dropping any remaining addresses given to [`Peer::dial()`] + /// that have not yet been tried. + pub fn abort(mut self) { + self.attempts.get_mut().remove(self.pos); + if self.attempts.get().is_empty() { + self.attempts.remove(); + } + self.inner.abort(); + } + + /// Adds an address to the end of the remaining addresses + /// for this dialing attempt. Duplicates are ignored. pub fn add_address(&mut self, addr: Multiaddr) { - if self.dialing.get().next.iter().all(|a| a != &addr) { - self.dialing.get_mut().next.push(addr); + let remaining = &mut self.attempts.get_mut()[self.pos].remaining; + if remaining.iter().all(|a| a != &addr) { + remaining.push(addr); } } } +/// An iterator over the ongoing dialing attempts to a peer. +pub struct DialingAttemptIter<'a, TInEvent, TOutEvent, THandler, TTransErr, THandlerErr, TConnInfo, TPeerId> { + /// The peer whose dialing attempts are being iterated. + peer_id: &'a TPeerId, + /// The underlying connection `Pool` of the `Network`. + pool: &'a mut Pool, + /// The state of all current dialing attempts known to the `Network`. + /// + /// Ownership of the `OccupiedEntry` for `peer_id` containing all attempts must be + /// borrowed to each `DialingAttempt` in order for it to remove the entry if the + /// last dialing attempt is aborted. + dialing: &'a mut FnvHashMap>, + /// The current position of the iterator in `dialing[peer_id]`. + pos: usize, + /// The total number of elements in `dialing[peer_id]` to iterate over. + end: usize, +} + +// Note: Ideally this would be an implementation of `Iterator`, but that +// requires GATs (cf. https://github.com/rust-lang/rust/issues/44265) and +// a different definition of `Iterator`. +impl<'a, TInEvent, TOutEvent, THandler, TTransErr, THandlerErr, TConnInfo, TPeerId> + DialingAttemptIter<'a, TInEvent, TOutEvent, THandler, TTransErr, THandlerErr, TConnInfo, TPeerId> +where + TConnInfo: ConnectionInfo, + TPeerId: Eq + Hash + Clone, +{ + fn new( + peer_id: &'a TPeerId, + pool: &'a mut Pool, + dialing: &'a mut FnvHashMap>, + ) -> Self { + let end = dialing.get(peer_id).map_or(0, |conns| conns.len()); + Self { pos: 0, end, pool, dialing, peer_id } + } + + /// Obtains the next dialing connection, if any. + pub fn next<'b>(&'b mut self) -> Option> { + if self.pos == self.end { + return None + } + + if let hash_map::Entry::Occupied(attempts) = self.dialing.entry(self.peer_id.clone()) { + let id = attempts.get()[self.pos].current.0; + if let Some(inner) = self.pool.get_outgoing(id) { + let conn = DialingAttempt { pos: self.pos, inner, attempts }; + self.pos += 1; + return Some(conn) + } + } + + None + } + + /// Returns the first connection, if any, consuming the iterator. + pub fn into_first<'b>(self) + -> Option> + where 'a: 'b + { + if self.pos == self.end { + return None + } + + if let hash_map::Entry::Occupied(attempts) = self.dialing.entry(self.peer_id.clone()) { + let id = attempts.get()[self.pos].current.0; + if let Some(inner) = self.pool.get_outgoing(id) { + return Some(DialingAttempt { pos: self.pos, inner, attempts }) + } + } + + None + } +} diff --git a/core/tests/network_dial_error.rs b/core/tests/network_dial_error.rs index c01cebdd..630eccc0 100644 --- a/core/tests/network_dial_error.rs +++ b/core/tests/network_dial_error.rs @@ -22,47 +22,60 @@ mod util; use futures::prelude::*; use libp2p_core::identity; -use libp2p_core::multiaddr::multiaddr; +use libp2p_core::multiaddr::{multiaddr, Multiaddr}; use libp2p_core::{ Network, PeerId, Transport, connection::PendingConnectionError, muxing::StreamMuxerBox, - network::NetworkEvent, + network::{NetworkEvent, NetworkConfig}, + transport, upgrade, }; +use rand::Rng; use rand::seq::SliceRandom; -use std::{io, task::Poll}; +use std::{io, error::Error, fmt, task::Poll}; use util::TestHandler; -type TestNetwork = Network; +type TestNetwork = Network; +type TestTransport = transport::boxed::Boxed<(PeerId, StreamMuxerBox), BoxError>; + +#[derive(Debug)] +struct BoxError(Box); + +impl Error for BoxError {} + +impl fmt::Display for BoxError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Transport error: {}", self.0) + } +} + +fn new_network(cfg: NetworkConfig) -> TestNetwork { + let local_key = identity::Keypair::generate_ed25519(); + let local_public_key = local_key.public(); + let transport: TestTransport = libp2p_tcp::TcpConfig::new() + .upgrade(upgrade::Version::V1) + .authenticate(libp2p_secio::SecioConfig::new(local_key)) + .multiplex(libp2p_mplex::MplexConfig::new()) + .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))) + .and_then(|(peer, mplex), _| { + // Gracefully close the connection to allow protocol + // negotiation to complete. + util::CloseMuxer::new(mplex).map_ok(move |mplex| (peer, mplex)) + }) + .map_err(|e| BoxError(Box::new(e))) + .boxed(); + TestNetwork::new(transport, local_public_key.into(), cfg) +} #[test] fn deny_incoming_connec() { // Checks whether refusing an incoming connection on a swarm triggers the correct events. - let mut swarm1 = { - let local_key = identity::Keypair::generate_ed25519(); - let local_public_key = local_key.public(); - let transport = libp2p_tcp::TcpConfig::new() - .upgrade(upgrade::Version::V1) - .authenticate(libp2p_secio::SecioConfig::new(local_key)) - .multiplex(libp2p_mplex::MplexConfig::new()) - .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))); - TestNetwork::new(transport, local_public_key.into(), Default::default()) - }; - - let mut swarm2 = { - let local_key = identity::Keypair::generate_ed25519(); - let local_public_key = local_key.public(); - let transport = libp2p_tcp::TcpConfig::new() - .upgrade(upgrade::Version::V1) - .authenticate(libp2p_secio::SecioConfig::new(local_key)) - .multiplex(libp2p_mplex::MplexConfig::new()) - .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))); - TestNetwork::new(transport, local_public_key.into(), Default::default()) - }; + let mut swarm1 = new_network(NetworkConfig::default()); + let mut swarm2 = new_network(NetworkConfig::default()); swarm1.listen_on("/ip4/127.0.0.1/tcp/0".parse().unwrap()).unwrap(); @@ -76,8 +89,7 @@ fn deny_incoming_connec() { swarm2 .peer(swarm1.local_peer_id().clone()) - .into_disconnected().unwrap() - .connect(address.clone(), Vec::new(), TestHandler()) + .dial(address.clone(), Vec::new(), TestHandler()) .unwrap(); async_std::task::block_on(future::poll_fn(|cx| -> Poll> { @@ -119,22 +131,7 @@ fn dial_self() { // // The last two can happen in any order. - let mut swarm = { - let local_key = identity::Keypair::generate_ed25519(); - let local_public_key = local_key.public(); - let transport = libp2p_tcp::TcpConfig::new() - .upgrade(upgrade::Version::V1) - .authenticate(libp2p_secio::SecioConfig::new(local_key)) - .multiplex(libp2p_mplex::MplexConfig::new()) - .and_then(|(peer, mplex), _| { - // Gracefully close the connection to allow protocol - // negotiation to complete. - util::CloseMuxer::new(mplex).map_ok(move |mplex| (peer, mplex)) - }) - .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))); - TestNetwork::new(transport, local_public_key.into(), Default::default()) - }; - + let mut swarm = new_network(NetworkConfig::default()); swarm.listen_on("/ip4/127.0.0.1/tcp/0".parse().unwrap()).unwrap(); let (local_address, mut swarm) = async_std::task::block_on( @@ -193,36 +190,16 @@ fn dial_self() { fn dial_self_by_id() { // Trying to dial self by passing the same `PeerId` shouldn't even be possible in the first // place. - - let mut swarm = { - let local_key = identity::Keypair::generate_ed25519(); - let local_public_key = local_key.public(); - let transport = libp2p_tcp::TcpConfig::new() - .upgrade(upgrade::Version::V1) - .authenticate(libp2p_secio::SecioConfig::new(local_key)) - .multiplex(libp2p_mplex::MplexConfig::new()) - .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))); - TestNetwork::new(transport, local_public_key.into(), Default::default()) - }; - + let mut swarm = new_network(NetworkConfig::default()); let peer_id = swarm.local_peer_id().clone(); assert!(swarm.peer(peer_id).into_disconnected().is_none()); } #[test] fn multiple_addresses_err() { - // Tries dialing multiple addresses, and makes sure there's one dialing error per addresses. + // Tries dialing multiple addresses, and makes sure there's one dialing error per address. - let mut swarm = { - let local_key = identity::Keypair::generate_ed25519(); - let local_public_key = local_key.public(); - let transport = libp2p_tcp::TcpConfig::new() - .upgrade(upgrade::Version::V1) - .authenticate(libp2p_secio::SecioConfig::new(local_key)) - .multiplex(libp2p_mplex::MplexConfig::new()) - .map(|(conn_info, muxer), _| (conn_info, StreamMuxerBox::new(muxer))); - TestNetwork::new(transport, local_public_key.into(), Default::default()) - }; + let mut swarm = new_network(NetworkConfig::default()); let mut addresses = Vec::new(); for _ in 0 .. 3 { @@ -238,8 +215,7 @@ fn multiple_addresses_err() { let target = PeerId::random(); swarm.peer(target.clone()) - .into_disconnected().unwrap() - .connect(first, rest, TestHandler()) + .dial(first, rest, TestHandler()) .unwrap(); async_std::task::block_on(future::poll_fn(|cx| -> Poll> { @@ -267,3 +243,44 @@ fn multiple_addresses_err() { } })).unwrap(); } + +#[test] +fn connection_limit() { + let outgoing_per_peer_limit = rand::thread_rng().gen_range(1, 10); + let outgoing_limit = 2 * outgoing_per_peer_limit; + + let mut cfg = NetworkConfig::default(); + cfg.set_outgoing_per_peer_limit(outgoing_per_peer_limit); + cfg.set_outgoing_limit(outgoing_limit); + let mut network = new_network(cfg); + + let target = PeerId::random(); + for _ in 0 .. outgoing_per_peer_limit { + network.peer(target.clone()) + .dial(Multiaddr::empty(), Vec::new(), TestHandler()) + .ok() + .expect("Unexpected connection limit."); + } + + let err = network.peer(target) + .dial(Multiaddr::empty(), Vec::new(), TestHandler()) + .expect_err("Unexpected dialing success."); + + assert_eq!(err.current, outgoing_per_peer_limit); + assert_eq!(err.limit, outgoing_per_peer_limit); + + let target2 = PeerId::random(); + for _ in outgoing_per_peer_limit .. outgoing_limit { + network.peer(target2.clone()) + .dial(Multiaddr::empty(), Vec::new(), TestHandler()) + .ok() + .expect("Unexpected connection limit."); + } + + let err = network.peer(target2) + .dial(Multiaddr::empty(), Vec::new(), TestHandler()) + .expect_err("Unexpected dialing success."); + + assert_eq!(err.current, outgoing_limit); + assert_eq!(err.limit, outgoing_limit); +} diff --git a/swarm/src/behaviour.rs b/swarm/src/behaviour.rs index fdddc6a0..16e804f9 100644 --- a/swarm/src/behaviour.rs +++ b/swarm/src/behaviour.rs @@ -291,12 +291,10 @@ pub enum DialPeerCondition { /// If there is an ongoing dialing attempt, the addresses reported by /// [`NetworkBehaviour::addresses_of_peer`] are added to the ongoing /// dialing attempt, ignoring duplicates. - /// - /// This condition implies [`DialPeerCondition::Disconnected`]. NotDialing, - // TODO: Once multiple dialing attempts per peer are permitted. - // See https://github.com/libp2p/rust-libp2p/pull/1506. - // Always, + /// A new dialing attempt is always initiated, only subject to the + /// configured connection limits. + Always, } impl Default for DialPeerCondition { diff --git a/swarm/src/lib.rs b/swarm/src/lib.rs index 1c62d8d9..d1d3ea6a 100644 --- a/swarm/src/lib.rs +++ b/swarm/src/lib.rs @@ -115,7 +115,6 @@ use libp2p_core::{ NetworkInfo, NetworkEvent, NetworkConfig, - Peer, peer::ConnectedPeer, }, upgrade::ProtocolName, @@ -379,70 +378,31 @@ where TBehaviour: NetworkBehaviour, /// /// If a new dialing attempt has been initiated, `Ok(true)` is returned. /// - /// If there is an ongoing dialing attempt, the current addresses of the - /// peer, as reported by [`NetworkBehaviour::addresses_of_peer`] are added - /// to the ongoing dialing attempt, ignoring duplicates. In this case no - /// new dialing attempt is initiated. - /// /// If no new dialing attempt has been initiated, meaning there is an ongoing /// dialing attempt or `addresses_of_peer` reports no addresses, `Ok(false)` /// is returned. - pub fn dial(me: &mut Self, peer_id: &PeerId) -> Result { + pub fn dial(me: &mut Self, peer_id: &PeerId) -> Result<(), DialError> { let mut addrs = me.behaviour.addresses_of_peer(peer_id).into_iter(); - match me.network.peer(peer_id.clone()) { - Peer::Disconnected(peer) => { - if let Some(first) = addrs.next() { - let handler = me.behaviour.new_handler().into_node_handler_builder(); - match peer.connect(first, addrs, handler) { - Ok(_) => return Ok(true), - Err(error) => { - log::debug!( - "New dialing attempt to disconnected peer {:?} failed: {:?}.", - peer_id, error); - me.behaviour.inject_dial_failure(&peer_id); - return Err(error) - } - } - } else { - log::debug!( - "New dialing attempt to disconnected peer {:?} failed: no address.", - peer_id - ); - me.behaviour.inject_dial_failure(&peer_id); - } - Ok(false) - }, - Peer::Connected(peer) => { - if let Some(first) = addrs.next() { - let handler = me.behaviour.new_handler().into_node_handler_builder(); - match peer.connect(first, addrs, handler) { - Ok(_) => return Ok(true), - Err(error) => { - log::debug!( - "New dialing attempt to connected peer {:?} failed: {:?}.", - peer_id, error); - me.behaviour.inject_dial_failure(&peer_id); - return Err(error) - } - } - } else { - log::debug!( - "New dialing attempt to disconnected peer {:?} failed: no address.", - peer_id - ); - me.behaviour.inject_dial_failure(&peer_id); - } - Ok(false) - } - Peer::Dialing(mut peer) => { - peer.connection().add_addresses(addrs); - Ok(false) - }, - Peer::Local => { - me.behaviour.inject_dial_failure(&peer_id); - Err(ConnectionLimit { current: 0, limit: 0 }) - } + let peer = me.network.peer(peer_id.clone()); + + let result = + if let Some(first) = addrs.next() { + let handler = me.behaviour.new_handler().into_node_handler_builder(); + peer.dial(first, addrs, handler) + .map(|_| ()) + .map_err(DialError::ConnectionLimit) + } else { + Err(DialError::NoAddresses) + }; + + if let Err(error) = &result { + log::debug!( + "New dialing attempt to peer {:?} failed: {:?}.", + peer_id, error); + me.behaviour.inject_dial_failure(&peer_id); } + + result } /// Returns an iterator that produces the list of addresses we're listening on. @@ -721,18 +681,22 @@ where TBehaviour: NetworkBehaviour, if !this.network.is_dialing(&peer_id) => true, _ => false }; - if condition_matched { - if let Ok(true) = ExpandedSwarm::dial(this, &peer_id) { - return Poll::Ready(SwarmEvent::Dialing(peer_id)); + if ExpandedSwarm::dial(this, &peer_id).is_ok() { + return Poll::Ready(SwarmEvent::Dialing(peer_id)) } - } else { + // Even if the condition for a _new_ dialing attempt is not met, + // we always add any potentially new addresses of the peer to an + // ongoing dialing attempt, if there is one. log::trace!("Condition for new dialing attempt to {:?} not met: {:?}", peer_id, condition); if let Some(mut peer) = this.network.peer(peer_id.clone()).into_dialing() { let addrs = this.behaviour.addresses_of_peer(peer.id()); - peer.connection().add_addresses(addrs); + let mut attempt = peer.some_attempt(); + for addr in addrs { + attempt.add_address(addr); + } } } } @@ -1104,6 +1068,35 @@ where TBehaviour: NetworkBehaviour, } } +/// The possible failures of [`ExpandedSwarm::dial`]. +#[derive(Debug)] +pub enum DialError { + /// The configured limit for simultaneous outgoing connections + /// has been reached. + ConnectionLimit(ConnectionLimit), + /// [`NetworkBehaviour::addresses_of_peer`] returned no addresses + /// for the peer to dial. + NoAddresses +} + +impl fmt::Display for DialError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DialError::ConnectionLimit(err) => write!(f, "Dial error: {}", err), + DialError::NoAddresses => write!(f, "Dial error: no addresses for peer.") + } + } +} + +impl error::Error for DialError { + fn source(&self) -> Option<&(dyn error::Error + 'static)> { + match self { + DialError::ConnectionLimit(err) => Some(err), + DialError::NoAddresses => None + } + } +} + /// Dummy implementation of [`NetworkBehaviour`] that doesn't do anything. #[derive(Clone, Default)] pub struct DummyBehaviour { From 70e797112f1c32fff2106c1dd40921938bf3a99a Mon Sep 17 00:00:00 2001 From: Max Inden Date: Tue, 12 May 2020 21:31:20 +0200 Subject: [PATCH 08/25] protocols/kad/query/closest: Make quickcheck deterministic (#1571) Instead of creating unconstrained random number generators in quickcheck tests to generate test data, have quickcheck provide a `Seed` to seed those random number generators and thus make the test execution deterministic / reproducible. --- protocols/kad/src/query/peers/closest.rs | 98 ++++++++++++++---------- 1 file changed, 56 insertions(+), 42 deletions(-) diff --git a/protocols/kad/src/query/peers/closest.rs b/protocols/kad/src/query/peers/closest.rs index 3e9b6a62..8879c58c 100644 --- a/protocols/kad/src/query/peers/closest.rs +++ b/protocols/kad/src/query/peers/closest.rs @@ -459,22 +459,13 @@ mod tests { use libp2p_core::PeerId; use quickcheck::*; use multihash::Multihash; - use rand::{Rng, thread_rng}; + use rand::{Rng, rngs::StdRng, SeedableRng}; use std::{iter, time::Duration}; - fn random_peers(n: usize) -> impl Iterator + Clone { - (0 .. n).map(|_| PeerId::random()) - } - - fn random_iter(g: &mut G) -> ClosestPeersIter { - let known_closest_peers = random_peers(g.gen_range(1, 60)).map(Key::from); - let target = Key::from(Into::::into(PeerId::random())); - let config = ClosestPeersIterConfig { - parallelism: g.gen_range(1, 10), - num_results: g.gen_range(1, 25), - peer_timeout: Duration::from_secs(g.gen_range(10, 30)), - }; - ClosestPeersIter::with_config(config, target, known_closest_peers) + fn random_peers(n: usize, g: &mut R) -> Vec { + (0 .. n).map(|_| PeerId::from_multihash( + multihash::wrap(multihash::Code::Sha2_256, &g.gen::<[u8; 32]>()) + ).unwrap()).collect() } fn sorted>(target: &T, peers: &Vec>) -> bool { @@ -483,42 +474,63 @@ mod tests { impl Arbitrary for ClosestPeersIter { fn arbitrary(g: &mut G) -> ClosestPeersIter { - random_iter(g) + let known_closest_peers = random_peers(g.gen_range(1, 60), g) + .into_iter() + .map(Key::from); + let target = Key::from(Into::::into(PeerId::random())); + let config = ClosestPeersIterConfig { + parallelism: g.gen_range(1, 10), + num_results: g.gen_range(1, 25), + peer_timeout: Duration::from_secs(g.gen_range(10, 30)), + }; + ClosestPeersIter::with_config(config, target, known_closest_peers) + } + } + + #[derive(Clone, Debug)] + struct Seed([u8; 32]); + + impl Arbitrary for Seed { + fn arbitrary(g: &mut G) -> Seed { + Seed(g.gen()) } } #[test] fn new_iter() { - let iter = random_iter(&mut thread_rng()); - let target = iter.target.clone(); + fn prop(iter: ClosestPeersIter) { + let target = iter.target.clone(); - let (keys, states): (Vec<_>, Vec<_>) = iter.closest_peers - .values() - .map(|e| (e.key.clone(), &e.state)) - .unzip(); + let (keys, states): (Vec<_>, Vec<_>) = iter.closest_peers + .values() + .map(|e| (e.key.clone(), &e.state)) + .unzip(); - let none_contacted = states - .iter() - .all(|s| match s { - PeerState::NotContacted => true, - _ => false - }); + let none_contacted = states + .iter() + .all(|s| match s { + PeerState::NotContacted => true, + _ => false + }); - assert!(none_contacted, - "Unexpected peer state in new iterator."); - assert!(sorted(&target, &keys), - "Closest peers in new iterator not sorted by distance to target."); - assert_eq!(iter.num_waiting(), 0, - "Unexpected peers in progress in new iterator."); - assert_eq!(iter.into_result().count(), 0, - "Unexpected closest peers in new iterator"); + assert!(none_contacted, + "Unexpected peer state in new iterator."); + assert!(sorted(&target, &keys), + "Closest peers in new iterator not sorted by distance to target."); + assert_eq!(iter.num_waiting(), 0, + "Unexpected peers in progress in new iterator."); + assert_eq!(iter.into_result().count(), 0, + "Unexpected closest peers in new iterator"); + } + + QuickCheck::new().tests(10).quickcheck(prop as fn(_) -> _) } #[test] fn termination_and_parallelism() { - fn prop(mut iter: ClosestPeersIter) { + fn prop(mut iter: ClosestPeersIter, seed: Seed) { let now = Instant::now(); - let mut rng = thread_rng(); + let mut rng = StdRng::from_seed(seed.0); let mut expected = iter.closest_peers .values() @@ -565,7 +577,7 @@ mod tests { for (i, k) in expected.iter().enumerate() { if rng.gen_bool(0.75) { let num_closer = rng.gen_range(0, iter.config.num_results + 1); - let closer_peers = random_peers(num_closer).collect::>(); + let closer_peers = random_peers(num_closer, &mut rng); remaining.extend(closer_peers.iter().cloned().map(Key::from)); iter.on_success(k.preimage(), closer_peers); } else { @@ -613,14 +625,16 @@ mod tests { } } - QuickCheck::new().tests(10).quickcheck(prop as fn(_) -> _) + QuickCheck::new().tests(10).quickcheck(prop as fn(_, _) -> _) } #[test] fn no_duplicates() { - fn prop(mut iter: ClosestPeersIter) -> bool { + fn prop(mut iter: ClosestPeersIter, seed: Seed) -> bool { let now = Instant::now(); - let closer = random_peers(1).collect::>(); + let mut rng = StdRng::from_seed(seed.0); + + let closer = random_peers(1, &mut rng); // A first peer reports a "closer" peer. let peer1 = match iter.next(now) { @@ -648,7 +662,7 @@ mod tests { true } - QuickCheck::new().tests(10).quickcheck(prop as fn(_) -> _) + QuickCheck::new().tests(10).quickcheck(prop as fn(_, _) -> _) } #[test] From 06721128f12e00716f56fc9dd5f1ad9dc0ce77f3 Mon Sep 17 00:00:00 2001 From: Max Inden Date: Wed, 13 May 2020 16:47:02 +0200 Subject: [PATCH 09/25] protocols/kad: Remove unnecessary PeerId conversion for fixed iter (#1573) `FixedPeersIter` requires the initial set of peers to be passed as `PeerId`s and not as `Key`s. This commit removes the unnecessary conversion. --- protocols/kad/src/behaviour.rs | 8 +++----- protocols/kad/src/query.rs | 4 +--- protocols/kad/src/query/peers/fixed.rs | 7 ++++++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/protocols/kad/src/behaviour.rs b/protocols/kad/src/behaviour.rs index d243aff3..4401309e 100644 --- a/protocols/kad/src/behaviour.rs +++ b/protocols/kad/src/behaviour.rs @@ -761,7 +761,6 @@ where } QueryInfo::PrepareAddProvider { key, context } => { - let closest_peers = result.peers.map(kbucket::Key::from); let provider_id = params.local_peer_id().clone(); let external_addresses = params.external_addresses().collect(); let inner = QueryInner::new(QueryInfo::AddProvider { @@ -770,7 +769,7 @@ where external_addresses, context, }); - self.queries.add_fixed(closest_peers, inner); + self.queries.add_fixed(result.peers, inner); None } @@ -799,7 +798,7 @@ where let context = PutRecordContext::Cache; let info = QueryInfo::PutRecord { record, quorum, context, num_results: 0 }; let inner = QueryInner::new(info); - self.queries.add_fixed(iter::once(cache_key), inner); + self.queries.add_fixed(iter::once(cache_key.into_preimage()), inner); } Ok(GetRecordOk { records }) } else if records.is_empty() { @@ -814,10 +813,9 @@ where } QueryInfo::PreparePutRecord { record, quorum, context } => { - let closest_peers = result.peers.map(kbucket::Key::from); let info = QueryInfo::PutRecord { record, quorum, context, num_results: 0 }; let inner = QueryInner::new(info); - self.queries.add_fixed(closest_peers, inner); + self.queries.add_fixed(result.peers, inner); None } diff --git a/protocols/kad/src/query.rs b/protocols/kad/src/query.rs index f00879a1..71d95f0d 100644 --- a/protocols/kad/src/query.rs +++ b/protocols/kad/src/query.rs @@ -89,9 +89,8 @@ impl QueryPool { /// Adds a query to the pool that contacts a fixed set of peers. pub fn add_fixed(&mut self, peers: I, inner: TInner) -> QueryId where - I: IntoIterator> + I: IntoIterator { - let peers = peers.into_iter().map(|k| k.into_preimage()).collect::>(); let parallelism = self.config.replication_factor.get(); let peer_iter = QueryPeerIter::Fixed(FixedPeersIter::new(peers, parallelism)); self.add(peer_iter, inner) @@ -295,4 +294,3 @@ pub struct QueryResult { /// The successfully contacted peers. pub peers: TPeers } - diff --git a/protocols/kad/src/query/peers/fixed.rs b/protocols/kad/src/query/peers/fixed.rs index 8a48aea7..402a4c2b 100644 --- a/protocols/kad/src/query/peers/fixed.rs +++ b/protocols/kad/src/query/peers/fixed.rs @@ -57,7 +57,12 @@ enum PeerState { } impl FixedPeersIter { - pub fn new(peers: Vec, parallelism: usize) -> Self { + pub fn new(peers: I, parallelism: usize) -> Self + where + I: IntoIterator + { + let peers = peers.into_iter().collect::>(); + Self { parallelism, peers: FnvHashMap::default(), From a83654ca4c5cd010c1df49f9709e63ed9dc73b3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=BCdiger=20Klaehn?= Date: Wed, 13 May 2020 17:08:12 +0200 Subject: [PATCH 10/25] Add a method to create a pre shared key when you already have the raw bytes (#1570) --- protocols/pnet/src/lib.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/protocols/pnet/src/lib.rs b/protocols/pnet/src/lib.rs index ab9b0bc3..496cd7d6 100644 --- a/protocols/pnet/src/lib.rs +++ b/protocols/pnet/src/lib.rs @@ -55,6 +55,11 @@ const FINGERPRINT_SIZE: usize = 16; pub struct PreSharedKey([u8; KEY_SIZE]); impl PreSharedKey { + /// Create a new pre shared key from raw bytes + pub fn new(data: [u8; KEY_SIZE]) -> Self { + Self(data) + } + /// Compute PreSharedKey fingerprint identical to the go-libp2p fingerprint. /// The computation of the fingerprint is not specified in the spec. /// From 0443fea157ab7d81feb4952cdf8aaae934130377 Mon Sep 17 00:00:00 2001 From: David Craven Date: Wed, 13 May 2020 17:51:11 +0200 Subject: [PATCH 11/25] Update to latest multihash. (#1566) * Update to latest multihash. * Update changelog. Co-authored-by: Pierre Krieger --- CHANGELOG.md | 2 ++ Cargo.toml | 2 +- core/Cargo.toml | 2 +- core/src/peer_id.rs | 11 +++++------ misc/multiaddr/Cargo.toml | 2 +- protocols/kad/Cargo.toml | 2 +- src/lib.rs | 12 +++++------- 7 files changed, 16 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fcd438ea..5c75aef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,8 @@ be supported. IPv4 listener addresses are not affected by this change. [PR 1555](https://github.com/libp2p/rust-libp2p/pull/1555) +- `libp2p-core`: Updated to multihash 0.11.0. + # Version 0.18.1 (2020-04-17) - `libp2p-swarm`: Make sure inject_dial_failure is called in all situations. diff --git a/Cargo.toml b/Cargo.toml index c00945dd..fe5b101c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -55,7 +55,7 @@ secp256k1 = ["libp2p-core/secp256k1", "libp2p-secio/secp256k1"] bytes = "0.5" futures = "0.3.1" multiaddr = { package = "parity-multiaddr", version = "0.8.0", path = "misc/multiaddr" } -multihash = "0.10" +multihash = "0.11.0" lazy_static = "1.2" libp2p-mplex = { version = "0.18.0", path = "muxers/mplex", optional = true } libp2p-identify = { version = "0.18.0", path = "protocols/identify", optional = true } diff --git a/core/Cargo.toml b/core/Cargo.toml index dd7f0461..5c64bc99 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -21,7 +21,7 @@ lazy_static = "1.2" libsecp256k1 = { version = "0.3.1", optional = true } log = "0.4" multiaddr = { package = "parity-multiaddr", version = "0.8.0", path = "../misc/multiaddr" } -multihash = "0.10" +multihash = "0.11.0" multistream-select = { version = "0.8.0", path = "../misc/multistream-select" } parking_lot = "0.10.0" pin-project = "0.4.6" diff --git a/core/src/peer_id.rs b/core/src/peer_id.rs index a4a90096..34b3ff6a 100644 --- a/core/src/peer_id.rs +++ b/core/src/peer_id.rs @@ -77,10 +77,9 @@ impl PeerId { }; let canonical = canonical_algorithm.map(|alg| - alg.hasher().expect("SHA2-256 hasher is always supported").digest(&key_enc)); + alg.digest(&key_enc)); - let multihash = hash_algorithm.hasher() - .expect("Identity and SHA-256 hasher are always supported").digest(&key_enc); + let multihash = hash_algorithm.digest(&key_enc); PeerId { multihash, canonical } } @@ -158,7 +157,7 @@ impl PeerId { pub fn is_public_key(&self, public_key: &PublicKey) -> Option { let alg = self.multihash.algorithm(); let enc = public_key.clone().into_protobuf_encoding(); - Some(alg.hasher()?.digest(&enc) == self.multihash) + Some(alg.digest(&enc) == self.multihash) } } @@ -321,8 +320,8 @@ mod tests { } fn property(data: Vec, algo1: HashAlgo, algo2: HashAlgo) -> bool { - let a = PeerId::try_from(algo1.0.hasher().unwrap().digest(&data)).unwrap(); - let b = PeerId::try_from(algo2.0.hasher().unwrap().digest(&data)).unwrap(); + let a = PeerId::try_from(algo1.0.digest(&data)).unwrap(); + let b = PeerId::try_from(algo2.0.digest(&data)).unwrap(); if algo1 == algo2 || algo1.0 == Code::Identity || algo2.0 == Code::Identity { a == b diff --git a/misc/multiaddr/Cargo.toml b/misc/multiaddr/Cargo.toml index 239407ae..56bb11e7 100644 --- a/misc/multiaddr/Cargo.toml +++ b/misc/multiaddr/Cargo.toml @@ -13,7 +13,7 @@ arrayref = "0.3" bs58 = "0.3.0" byteorder = "1.3.1" data-encoding = "2.1" -multihash = "0.10" +multihash = "0.11.0" percent-encoding = "2.1.0" serde = "1.0.70" static_assertions = "1.1" diff --git a/protocols/kad/Cargo.toml b/protocols/kad/Cargo.toml index a83df53d..b9734d18 100644 --- a/protocols/kad/Cargo.toml +++ b/protocols/kad/Cargo.toml @@ -19,7 +19,7 @@ futures = "0.3.1" log = "0.4" libp2p-core = { version = "0.18.0", path = "../../core" } libp2p-swarm = { version = "0.18.0", path = "../../swarm" } -multihash = "0.10" +multihash = "0.11.0" prost = "0.6.1" rand = "0.7.2" sha2 = "0.8.0" diff --git a/src/lib.rs b/src/lib.rs index c907a2a7..ded22023 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -262,8 +262,6 @@ pub use self::simple::SimpleProtocol; pub use self::swarm::Swarm; pub use self::transport_ext::TransportExt; -use std::{error, io, time::Duration}; - /// Builds a `Transport` that supports the most commonly-used protocols that libp2p supports. /// /// > **Note**: This `Transport` is not suitable for production usage, as its implementation @@ -271,7 +269,7 @@ use std::{error, io, time::Duration}; #[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))] #[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))))] pub fn build_development_transport(keypair: identity::Keypair) - -> io::Result> + Send + Sync), Error = impl error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> + -> std::io::Result> + Send + Sync), Error = impl std::error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> { build_tcp_ws_secio_mplex_yamux(keypair) } @@ -285,7 +283,7 @@ pub fn build_development_transport(keypair: identity::Keypair) #[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))] #[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))))] pub fn build_tcp_ws_secio_mplex_yamux(keypair: identity::Keypair) - -> io::Result> + Send + Sync), Error = impl error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> + -> std::io::Result> + Send + Sync), Error = impl std::error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> { let transport = { let tcp = tcp::TcpConfig::new().nodelay(true); @@ -299,7 +297,7 @@ pub fn build_tcp_ws_secio_mplex_yamux(keypair: identity::Keypair) .authenticate(secio::SecioConfig::new(keypair)) .multiplex(core::upgrade::SelectUpgrade::new(yamux::Config::default(), mplex::MplexConfig::new())) .map(|(peer, muxer), _| (peer, core::muxing::StreamMuxerBox::new(muxer))) - .timeout(Duration::from_secs(20))) + .timeout(std::time::Duration::from_secs(20))) } /// Builds an implementation of `Transport` that is suitable for usage with the `Swarm`. @@ -311,7 +309,7 @@ pub fn build_tcp_ws_secio_mplex_yamux(keypair: identity::Keypair) #[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux", feature = "pnet"))] #[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux", feature = "pnet"))))] pub fn build_tcp_ws_pnet_secio_mplex_yamux(keypair: identity::Keypair, psk: PreSharedKey) - -> io::Result> + Send + Sync), Error = impl error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> + -> std::io::Result> + Send + Sync), Error = impl std::error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> { let transport = { let tcp = tcp::TcpConfig::new().nodelay(true); @@ -326,5 +324,5 @@ pub fn build_tcp_ws_pnet_secio_mplex_yamux(keypair: identity::Keypair, psk: PreS .authenticate(secio::SecioConfig::new(keypair)) .multiplex(core::upgrade::SelectUpgrade::new(yamux::Config::default(), mplex::MplexConfig::new())) .map(|(peer, muxer), _| (peer, core::muxing::StreamMuxerBox::new(muxer))) - .timeout(Duration::from_secs(20))) + .timeout(std::time::Duration::from_secs(20))) } From c271f6f56b7b5c33b1ced135e01291b1d48e8eb8 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Fri, 15 May 2020 14:40:10 +0200 Subject: [PATCH 12/25] Make the number of events buffered to/from tasks configurable (#1574) * Make the number of events buffered to/from tasks configurable * Assign a PR number * Fix comment * Apply suggestions from code review Co-authored-by: Roman Borschel * Rename variables * Apply suggestions from code review Co-authored-by: Roman Borschel Co-authored-by: Roman Borschel --- CHANGELOG.md | 8 +++++-- core/src/connection/manager.rs | 40 +++++++++++++++++++++++++++---- core/src/connection/pool.rs | 7 +++--- core/src/network.rs | 41 ++++++++++++++++++++++++++----- swarm/src/lib.rs | 44 +++++++++++++++++++++++++++++++++- 5 files changed, 122 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c75aef9..b18df2c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ attempts per peer, with a configurable limit. [PR 1506](https://github.com/libp2p/rust-libp2p/pull/1506) +- `libp2p-core`: Updated to multihash 0.11.0. + [PR 1566](https://github.com/libp2p/rust-libp2p/pull/1566) + +- `libp2p-core`: Make the number of events buffered to/from tasks configurable. + [PR 1574](https://github.com/libp2p/rust-libp2p/pull/1574) + - `libp2p-noise`: Added the `X25519Spec` protocol suite which uses libp2p-noise-spec compliant signatures on static keys as well as the `/noise` protocol upgrade, hence providing a libp2p-noise-spec compliant @@ -27,8 +33,6 @@ be supported. IPv4 listener addresses are not affected by this change. [PR 1555](https://github.com/libp2p/rust-libp2p/pull/1555) -- `libp2p-core`: Updated to multihash 0.11.0. - # Version 0.18.1 (2020-04-17) - `libp2p-swarm`: Make sure inject_dial_failure is called in all situations. diff --git a/core/src/connection/manager.rs b/core/src/connection/manager.rs index c366a071..9dbb644a 100644 --- a/core/src/connection/manager.rs +++ b/core/src/connection/manager.rs @@ -99,6 +99,9 @@ pub struct Manager { /// Next available identifier for a new connection / task. next_task_id: TaskId, + /// Size of the task command buffer (per task). + task_command_buffer_size: usize, + /// The executor to use for running the background tasks. If `None`, /// the tasks are kept in `local_spawns` instead and polled on the /// current thread when the manager is polled for new events. @@ -127,6 +130,32 @@ where } } +/// Configuration options when creating a [`Manager`]. +/// +/// The default configuration specifies no dedicated task executor, a +/// task event buffer size of 32, and a task command buffer size of 7. +#[non_exhaustive] +pub struct ManagerConfig { + /// Executor to use to spawn tasks. + pub executor: Option>, + + /// Size of the task command buffer (per task). + pub task_command_buffer_size: usize, + + /// Size of the task event buffer (for all tasks). + pub task_event_buffer_size: usize, +} + +impl Default for ManagerConfig { + fn default() -> Self { + ManagerConfig { + executor: None, + task_event_buffer_size: 32, + task_command_buffer_size: 7, + } + } +} + /// Internal information about a running task. /// /// Contains the sender to deliver event messages to the task, and @@ -196,12 +225,13 @@ pub enum Event<'a, I, O, H, TE, HE, C> { impl Manager { /// Creates a new connection manager. - pub fn new(executor: Option>) -> Self { - let (tx, rx) = mpsc::channel(1); + pub fn new(config: ManagerConfig) -> Self { + let (tx, rx) = mpsc::channel(config.task_event_buffer_size); Self { tasks: FnvHashMap::default(), next_task_id: TaskId(0), - executor, + task_command_buffer_size: config.task_command_buffer_size, + executor: config.executor, local_spawns: FuturesUnordered::new(), events_tx: tx, events_rx: rx @@ -234,7 +264,7 @@ impl Manager { let task_id = self.next_task_id; self.next_task_id.0 += 1; - let (tx, rx) = mpsc::channel(4); + let (tx, rx) = mpsc::channel(self.task_command_buffer_size); self.tasks.insert(task_id, TaskInfo { sender: tx, state: TaskState::Pending }); let task = Box::pin(Task::pending(task_id, self.events_tx.clone(), rx, future, handler)); @@ -269,7 +299,7 @@ impl Manager { let task_id = self.next_task_id; self.next_task_id.0 += 1; - let (tx, rx) = mpsc::channel(4); + let (tx, rx) = mpsc::channel(self.task_command_buffer_size); self.tasks.insert(task_id, TaskInfo { sender: tx, state: TaskState::Established(info) }); diff --git a/core/src/connection/pool.rs b/core/src/connection/pool.rs index 7da7efa8..b319ca76 100644 --- a/core/src/connection/pool.rs +++ b/core/src/connection/pool.rs @@ -19,7 +19,6 @@ // DEALINGS IN THE SOFTWARE. use crate::{ - Executor, ConnectedPoint, PeerId, connection::{ @@ -36,7 +35,7 @@ use crate::{ OutgoingInfo, Substream, PendingConnectionError, - manager::{self, Manager}, + manager::{self, Manager, ManagerConfig}, }, muxing::StreamMuxer, }; @@ -175,13 +174,13 @@ where /// Creates a new empty `Pool`. pub fn new( local_id: TPeerId, - executor: Option>, + manager_config: ManagerConfig, limits: PoolLimits ) -> Self { Pool { local_id, limits, - manager: Manager::new(executor), + manager: Manager::new(manager_config), established: Default::default(), pending: Default::default(), } diff --git a/core/src/network.rs b/core/src/network.rs index 73240abd..1e89dcd7 100644 --- a/core/src/network.rs +++ b/core/src/network.rs @@ -43,6 +43,7 @@ use crate::{ ListenersStream, PendingConnectionError, Substream, + manager::ManagerConfig, pool::{Pool, PoolEvent, PoolLimits}, }, muxing::StreamMuxer, @@ -57,6 +58,7 @@ use std::{ error, fmt, hash::Hash, + num::NonZeroUsize, pin::Pin, task::{Context, Poll}, }; @@ -154,7 +156,7 @@ where Network { local_peer_id, listeners: ListenersStream::new(transport), - pool: Pool::new(pool_local_id, config.executor, config.pool_limits), + pool: Pool::new(pool_local_id, config.manager_config, config.pool_limits), dialing: Default::default(), } } @@ -598,17 +600,21 @@ pub struct NetworkInfo { /// The (optional) configuration for a [`Network`]. /// -/// The default configuration specifies no dedicated task executor -/// and no connection limits. +/// The default configuration specifies no dedicated task executor, no +/// connection limits, a connection event buffer size of 32, and a +/// `notify_handler` buffer size of 8. #[derive(Default)] pub struct NetworkConfig { - executor: Option>, + /// Note that the `ManagerConfig`s task command buffer always provides + /// one "free" slot per task. Thus the given total `notify_handler_buffer_size` + /// exposed for configuration on the `Network` is reduced by one. + manager_config: ManagerConfig, pool_limits: PoolLimits, } impl NetworkConfig { pub fn set_executor(&mut self, e: Box) -> &mut Self { - self.executor = Some(e); + self.manager_config.executor = Some(e); self } @@ -625,7 +631,30 @@ impl NetworkConfig { } pub fn executor(&self) -> Option<&Box> { - self.executor.as_ref() + self.manager_config.executor.as_ref() + } + + /// Sets the maximum number of events sent to a connection's background task + /// that may be buffered, if the task cannot keep up with their consumption and + /// delivery to the connection handler. + /// + /// When the buffer for a particular connection is full, `notify_handler` will no + /// longer be able to deliver events to the associated `ConnectionHandler`, + /// thus exerting back-pressure on the connection and peer API. + pub fn set_notify_handler_buffer_size(&mut self, n: NonZeroUsize) -> &mut Self { + self.manager_config.task_command_buffer_size = n.get() - 1; + self + } + + /// Sets the maximum number of buffered connection events (beyond a guaranteed + /// buffer of 1 event per connection). + /// + /// When the buffer is full, the background tasks of all connections will stall. + /// In this way, the consumers of network events exert back-pressure on + /// the network connection I/O. + pub fn set_connection_event_buffer_size(&mut self, n: usize) -> &mut Self { + self.manager_config.task_event_buffer_size = n; + self } pub fn set_incoming_limit(&mut self, n: usize) -> &mut Self { diff --git a/swarm/src/lib.rs b/swarm/src/lib.rs index d1d3ea6a..f848f64f 100644 --- a/swarm/src/lib.rs +++ b/swarm/src/lib.rs @@ -123,7 +123,7 @@ use registry::{Addresses, AddressIntoIter}; use smallvec::SmallVec; use std::{error, fmt, hash::Hash, io, ops::{Deref, DerefMut}, pin::Pin, task::{Context, Poll}}; use std::collections::HashSet; -use std::num::NonZeroU32; +use std::num::{NonZeroU32, NonZeroUsize}; use upgrade::UpgradeInfoSend as _; /// Contains the state of the network, plus the way it should behave. @@ -1002,6 +1002,48 @@ where TBehaviour: NetworkBehaviour, self } + /// Configures the number of events from the [`NetworkBehaviour`] in + /// destination to the [`ProtocolsHandler`] that can be buffered before + /// the [`Swarm`] has to wait. An individual buffer with this number of + /// events exists for each individual connection. + /// + /// The ideal value depends on the executor used, the CPU speed, and the + /// volume of events. If this value is too low, then the [`Swarm`] will + /// be sleeping more often than necessary. Increasing this value increases + /// the overall memory usage. + pub fn notify_handler_buffer_size(mut self, n: NonZeroUsize) -> Self { + self.network_config.set_notify_handler_buffer_size(n); + self + } + + /// Configures the number of extra events from the [`ProtocolsHandler`] in + /// destination to the [`NetworkBehaviour`] that can be buffered before + /// the [`ProtocolsHandler`] has to go to sleep. + /// + /// There exists a buffer of events received from [`ProtocolsHandler`]s + /// that the [`NetworkBehaviour`] has yet to process. This buffer is + /// shared between all instances of [`ProtocolsHandler`]. Each instance of + /// [`ProtocolsHandler`] is guaranteed one slot in this buffer, meaning + /// that delivering an event for the first time is guaranteed to be + /// instantaneous. Any extra event delivery, however, must wait for that + /// first event to be delivered or for an "extra slot" to be available. + /// + /// This option configures the number of such "extra slots" in this + /// shared buffer. These extra slots are assigned in a first-come, + /// first-served basis. + /// + /// The ideal value depends on the executor used, the CPU speed, the + /// average number of connections, and the volume of events. If this value + /// is too low, then the [`ProtocolsHandler`]s will be sleeping more often + /// than necessary. Increasing this value increases the overall memory + /// usage, and more importantly the latency between the moment when an + /// event is emitted and the moment when it is received by the + /// [`NetworkBehaviour`]. + pub fn connection_event_buffer_size(mut self, n: usize) -> Self { + self.network_config.set_connection_event_buffer_size(n); + self + } + /// Configures a limit for the number of simultaneous incoming /// connection attempts. pub fn incoming_connection_limit(mut self, n: usize) -> Self { From 3a96ebf57ffa9f476ebbd150c6b989e0da6baaff Mon Sep 17 00:00:00 2001 From: Roman Borschel Date: Sat, 16 May 2020 10:43:09 +0200 Subject: [PATCH 13/25] More insight into Kademlia queries. (#1567) * [libp2p-kad] Provide more insight and control into Kademlia queries. More insight: The API allows iterating over the active queries and inspecting their state and execution statistics. More control: The API allows aborting queries prematurely at any time. To that end, API operations that initiate new queries return the query ID and multi-phase queries such as `put_record` retain the query ID across all phases, each phase being executed by a new (internal) query. * Cleanup * Cleanup * Update examples and re-exports. * Incorporate review feedback. * Update CHANGELOG * Update CHANGELOG Co-authored-by: Max Inden --- CHANGELOG.md | 8 + examples/distributed-key-value-store.rs | 49 +- examples/ipfs-kad.rs | 13 +- protocols/kad/src/behaviour.rs | 826 +++++++++++++++-------- protocols/kad/src/behaviour/test.rs | 127 +++- protocols/kad/src/lib.rs | 4 + protocols/kad/src/query.rs | 178 ++++- protocols/kad/src/query/peers/closest.rs | 52 +- protocols/kad/src/query/peers/fixed.rs | 34 +- 9 files changed, 925 insertions(+), 366 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b18df2c8..a926161b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,14 @@ has no effect. [PR 1536](https://github.com/libp2p/rust-libp2p/pull/1536) +- `libp2p-kad`: Provide more insight into, and control of, the execution of + queries. All query results are now wrapped in `KademliaEvent::QueryResult`. + As a side-effect of these changes and for as long as the record storage + API is not asynchronous, local storage errors on `put_record` are reported + synchronously in a `Result`, instead of being reported asynchronously by + an event. + [PR 1567](https://github.com/libp2p/rust-libp2p/pull/1567) + - `libp2p-tcp`: On listeners started with an IPv6 multi-address the socket option `IPV6_V6ONLY` is set to true. Instead of relying on IPv4-mapped IPv6 address support, two listeners can be started if IPv4 and IPv6 should both diff --git a/examples/distributed-key-value-store.rs b/examples/distributed-key-value-store.rs index 5bf21855..014418d3 100644 --- a/examples/distributed-key-value-store.rs +++ b/examples/distributed-key-value-store.rs @@ -32,7 +32,15 @@ use async_std::{io, task}; use futures::prelude::*; use libp2p::kad::record::store::MemoryStore; -use libp2p::kad::{record::Key, Kademlia, KademliaEvent, PutRecordOk, Quorum, Record}; +use libp2p::kad::{ + record::Key, + Kademlia, + KademliaEvent, + PutRecordOk, + QueryResult, + Quorum, + Record +}; use libp2p::{ NetworkBehaviour, PeerId, @@ -76,26 +84,29 @@ fn main() -> Result<(), Box> { // Called when `kademlia` produces an event. fn inject_event(&mut self, message: KademliaEvent) { match message { - KademliaEvent::GetRecordResult(Ok(result)) => { - for Record { key, value, .. } in result.records { + KademliaEvent::QueryResult { result, .. } => match result { + QueryResult::GetRecord(Ok(ok)) => { + for Record { key, value, .. } in ok.records { + println!( + "Got record {:?} {:?}", + std::str::from_utf8(key.as_ref()).unwrap(), + std::str::from_utf8(&value).unwrap(), + ); + } + } + QueryResult::GetRecord(Err(err)) => { + eprintln!("Failed to get record: {:?}", err); + } + QueryResult::PutRecord(Ok(PutRecordOk { key })) => { println!( - "Got record {:?} {:?}", - std::str::from_utf8(key.as_ref()).unwrap(), - std::str::from_utf8(&value).unwrap(), + "Successfully put record {:?}", + std::str::from_utf8(key.as_ref()).unwrap() ); } - } - KademliaEvent::GetRecordResult(Err(err)) => { - eprintln!("Failed to get record: {:?}", err); - } - KademliaEvent::PutRecordResult(Ok(PutRecordOk { key })) => { - println!( - "Successfully put record {:?}", - std::str::from_utf8(key.as_ref()).unwrap() - ); - } - KademliaEvent::PutRecordResult(Err(err)) => { - eprintln!("Failed to put record: {:?}", err); + QueryResult::PutRecord(Err(err)) => { + eprintln!("Failed to put record: {:?}", err); + } + _ => {} } _ => {} } @@ -188,7 +199,7 @@ fn handle_input_line(kademlia: &mut Kademlia, line: String) { publisher: None, expires: None, }; - kademlia.put_record(record, Quorum::One); + kademlia.put_record(record, Quorum::One).expect("Failed to store record locally."); } _ => { eprintln!("expected GET or PUT"); diff --git a/examples/ipfs-kad.rs b/examples/ipfs-kad.rs index bb1738e5..ec48435d 100644 --- a/examples/ipfs-kad.rs +++ b/examples/ipfs-kad.rs @@ -30,7 +30,13 @@ use libp2p::{ identity, build_development_transport }; -use libp2p::kad::{Kademlia, KademliaConfig, KademliaEvent, GetClosestPeersError}; +use libp2p::kad::{ + Kademlia, + KademliaConfig, + KademliaEvent, + GetClosestPeersError, + QueryResult, +}; use libp2p::kad::record::store::MemoryStore; use std::{env, error::Error, time::Duration}; @@ -91,7 +97,10 @@ fn main() -> Result<(), Box> { task::block_on(async move { loop { let event = swarm.next().await; - if let KademliaEvent::GetClosestPeersResult(result) = event { + if let KademliaEvent::QueryResult { + result: QueryResult::GetClosestPeers(result), + .. + } = event { match result { Ok(ok) => if !ok.peers.is_empty() { diff --git a/protocols/kad/src/behaviour.rs b/protocols/kad/src/behaviour.rs index 4401309e..6bfcdba7 100644 --- a/protocols/kad/src/behaviour.rs +++ b/protocols/kad/src/behaviour.rs @@ -44,10 +44,14 @@ use log::{info, debug, warn}; use smallvec::SmallVec; use std::{borrow::{Borrow, Cow}, error, iter, time::Duration}; use std::collections::{HashSet, VecDeque}; +use std::fmt; use std::num::NonZeroUsize; use std::task::{Context, Poll}; +use std::vec; use wasm_timer::Instant; +pub use crate::query::QueryStats; + /// Network behaviour that handles Kademlia. pub struct Kademlia { /// The Kademlia routing table. @@ -286,6 +290,46 @@ where } } + /// Gets an iterator over immutable references to all running queries. + pub fn iter_queries<'a>(&'a self) -> impl Iterator> { + self.queries.iter().filter_map(|query| + if !query.is_finished() { + Some(QueryRef { query }) + } else { + None + }) + } + + /// Gets an iterator over mutable references to all running queries. + pub fn iter_queries_mut<'a>(&'a mut self) -> impl Iterator> { + self.queries.iter_mut().filter_map(|query| + if !query.is_finished() { + Some(QueryMut { query }) + } else { + None + }) + } + + /// Gets an immutable reference to a running query, if it exists. + pub fn query<'a>(&'a self, id: &QueryId) -> Option> { + self.queries.get(id).and_then(|query| + if !query.is_finished() { + Some(QueryRef { query }) + } else { + None + }) + } + + /// Gets a mutable reference to a running query, if it exists. + pub fn query_mut<'a>(&'a mut self, id: &QueryId) -> Option> { + self.queries.get_mut(id).and_then(|query| + if !query.is_finished() { + Some(QueryMut { query }) + } else { + None + }) + } + /// Adds a known listen address of a peer participating in the DHT to the /// routing table. /// @@ -359,10 +403,11 @@ where self.kbuckets.iter().map(|entry| entry.node.key.preimage()) } - /// Performs a lookup for the closest peers to the given key. + /// Initiates an iterative query for the closest peers to the given key. /// - /// The result of this operation is delivered in [`KademliaEvent::GetClosestPeersResult`]. - pub fn get_closest_peers(&mut self, key: K) + /// The result of the query is delivered in a + /// [`KademliaEvent::QueryResult{QueryResult::GetClosestPeers}`]. + pub fn get_closest_peers(&mut self, key: K) -> QueryId where K: Borrow<[u8]> + Clone { @@ -370,13 +415,14 @@ where let target = kbucket::Key::new(key); let peers = self.kbuckets.closest_keys(&target); let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); + self.queries.add_iter_closest(target.clone(), peers, inner) } /// Performs a lookup for a record in the DHT. /// - /// The result of this operation is delivered in [`KademliaEvent::GetRecordResult`]. - pub fn get_record(&mut self, key: &record::Key, quorum: Quorum) { + /// The result of this operation is delivered in a + /// [`KademliaEvent::QueryResult{QueryResult::GetRecord}`]. + pub fn get_record(&mut self, key: &record::Key, quorum: Quorum) -> QueryId { let quorum = quorum.eval(self.queries.config().replication_factor); let mut records = Vec::with_capacity(quorum.get()); @@ -385,25 +431,30 @@ where self.store.remove(key) } else { records.push(record.into_owned()); - if quorum.get() == 1 { - self.queued_events.push_back(NetworkBehaviourAction::GenerateEvent( - KademliaEvent::GetRecordResult(Ok(GetRecordOk { records })) - )); - return; - } } } + let done = records.len() >= quorum.get(); let target = kbucket::Key::new(key.clone()); let info = QueryInfo::GetRecord { key: key.clone(), records, quorum, cache_at: None }; let peers = self.kbuckets.closest_keys(&target); let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); + let id = self.queries.add_iter_closest(target.clone(), peers, inner); // (*) + + // Instantly finish the query if we already have enough records. + if done { + self.queries.get_mut(&id).expect("by (*)").finish(); + } + + id } /// Stores a record in the DHT. /// - /// The result of this operation is delivered in [`KademliaEvent::PutRecordResult`]. + /// Returns `Ok` if a record has been stored locally, providing the + /// `QueryId` of the initial query that replicates the record in the DHT. + /// The result of the query is eventually reported as a + /// [`KademliaEvent::QueryResult{QueryResult::PutRecord}`]. /// /// The record is always stored locally with the given expiration. If the record's /// expiration is `None`, the common case, it does not expire in local storage @@ -415,28 +466,23 @@ where /// does not update the record's expiration in local storage, thus a given record /// with an explicit expiration will always expire at that instant and until then /// is subject to regular (re-)replication and (re-)publication. - pub fn put_record(&mut self, mut record: Record, quorum: Quorum) { + pub fn put_record(&mut self, mut record: Record, quorum: Quorum) -> Result { record.publisher = Some(self.kbuckets.local_key().preimage().clone()); - if let Err(err) = self.store.put(record.clone()) { - self.queued_events.push_back(NetworkBehaviourAction::GenerateEvent( - KademliaEvent::PutRecordResult(Err( - PutRecordError::LocalStorageError { - key: record.key, - cause: err, - } - )) - )); - } else { - record.expires = record.expires.or_else(|| - self.record_ttl.map(|ttl| Instant::now() + ttl)); - let quorum = quorum.eval(self.queries.config().replication_factor); - let target = kbucket::Key::new(record.key.clone()); - let peers = self.kbuckets.closest_keys(&target); - let context = PutRecordContext::Publish; - let info = QueryInfo::PreparePutRecord { record, quorum, context }; - let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); - } + self.store.put(record.clone())?; + record.expires = record.expires.or_else(|| + self.record_ttl.map(|ttl| Instant::now() + ttl)); + let quorum = quorum.eval(self.queries.config().replication_factor); + let target = kbucket::Key::new(record.key.clone()); + let peers = self.kbuckets.closest_keys(&target); + let context = PutRecordContext::Publish; + let info = QueryInfo::PutRecord { + context, + record, + quorum, + phase: PutRecordPhase::GetClosestPeers + }; + let inner = QueryInner::new(info); + Ok(self.queries.add_iter_closest(target.clone(), peers, inner)) } /// Removes the record with the given key from _local_ storage, @@ -471,18 +517,28 @@ where /// refreshed by initiating an additional bootstrapping query for each such /// bucket with random keys. /// - /// The result(s) of this operation are delivered in [`KademliaEvent::BootstrapResult`], - /// with one event per bootstrapping query. + /// Returns `Ok` if bootstrapping has been initiated with a self-lookup, providing the + /// `QueryId` for the entire bootstrapping process. The progress of bootstrapping is + /// reported via [`KademliaEvent::QueryResult{QueryResult::Bootstrap}`] events, + /// with one such event per bootstrapping query. + /// + /// Returns `Err` if bootstrapping is impossible due an empty routing table. /// /// > **Note**: Bootstrapping requires at least one node of the DHT to be known. /// > See [`Kademlia::add_address`]. - pub fn bootstrap(&mut self) { + pub fn bootstrap(&mut self) -> Result { let local_key = self.kbuckets.local_key().clone(); - let info = QueryInfo::Bootstrap { peer: local_key.preimage().clone() }; + let info = QueryInfo::Bootstrap { + peer: local_key.preimage().clone(), + remaining: None + }; let peers = self.kbuckets.closest_keys(&local_key).collect::>(); - // TODO: Emit error if `peers` is empty? BootstrapError::NoPeers? - let inner = QueryInner::new(info); - self.queries.add_iter_closest(local_key, peers, inner); + if peers.is_empty() { + Err(NoKnownPeers()) + } else { + let inner = QueryInner::new(info); + Ok(self.queries.add_iter_closest(local_key, peers, inner)) + } } /// Establishes the local node as a provider of a value for the given key. @@ -491,6 +547,9 @@ where /// identity of the local node to the peers closest to the key, thus establishing /// the local node as a provider. /// + /// Returns `Ok` if a provider record has been stored locally, providing the + /// `QueryId` of the initial query that announces the local node as a provider. + /// /// The publication of the provider records is periodically repeated as per the /// configured interval, to renew the expiry and account for changes to the DHT /// topology. A provider record may be removed from local storage and @@ -503,23 +562,21 @@ where /// of the libp2p Kademlia provider API. /// /// The results of the (repeated) provider announcements sent by this node are - /// delivered in [`AddProviderResult`]. - pub fn start_providing(&mut self, key: record::Key) { + /// reported via [`KademliaEvent::QueryResult{QueryResult::AddProvider}`]. + pub fn start_providing(&mut self, key: record::Key) -> Result { let record = ProviderRecord::new(key.clone(), self.kbuckets.local_key().preimage().clone()); - if let Err(err) = self.store.add_provider(record) { - self.queued_events.push_back(NetworkBehaviourAction::GenerateEvent( - KademliaEvent::StartProvidingResult(Err( - AddProviderError::LocalStorageError { key, cause: err } - )) - )); - } else { - let target = kbucket::Key::new(key.clone()); - let peers = self.kbuckets.closest_keys(&target); - let context = AddProviderContext::Publish; - let info = QueryInfo::PrepareAddProvider { key, context }; - let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); - } + self.store.add_provider(record)?; + let target = kbucket::Key::new(key.clone()); + let peers = self.kbuckets.closest_keys(&target); + let context = AddProviderContext::Publish; + let info = QueryInfo::AddProvider { + context, + key, + phase: AddProviderPhase::GetClosestPeers + }; + let inner = QueryInner::new(info); + let id = self.queries.add_iter_closest(target.clone(), peers, inner); + Ok(id) } /// Stops the local node from announcing that it is a provider for the given key. @@ -532,8 +589,9 @@ where /// Performs a lookup for providers of a value to the given key. /// - /// The result of this operation is delivered in [`KademliaEvent::GetProvidersResult`]. - pub fn get_providers(&mut self, key: record::Key) { + /// The result of this operation is delivered in a + /// reported via [`KademliaEvent::QueryResult{QueryResult::GetProviders}`]. + pub fn get_providers(&mut self, key: record::Key) -> QueryId { let info = QueryInfo::GetProviders { key: key.clone(), providers: HashSet::new(), @@ -541,7 +599,7 @@ where let target = kbucket::Key::new(key); let peers = self.kbuckets.closest_keys(&target); let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); + self.queries.add_iter_closest(target.clone(), peers, inner) } /// Processes discovered peers from a successful request in an iterative `Query`. @@ -608,7 +666,11 @@ where /// Starts an iterative `ADD_PROVIDER` query for the given key. fn start_add_provider(&mut self, key: record::Key, context: AddProviderContext) { - let info = QueryInfo::PrepareAddProvider { key: key.clone(), context }; + let info = QueryInfo::AddProvider { + context, + key: key.clone(), + phase: AddProviderPhase::GetClosestPeers + }; let target = kbucket::Key::new(key); let peers = self.kbuckets.closest_keys(&target); let inner = QueryInner::new(info); @@ -620,7 +682,9 @@ where let quorum = quorum.eval(self.queries.config().replication_factor); let target = kbucket::Key::new(record.key.clone()); let peers = self.kbuckets.closest_keys(&target); - let info = QueryInfo::PreparePutRecord { record, quorum, context }; + let info = QueryInfo::PutRecord { + record, quorum, context, phase: PutRecordPhase::GetClosestPeers + }; let inner = QueryInner::new(info); self.queries.add_iter_closest(target.clone(), peers, inner); } @@ -696,17 +760,19 @@ where fn query_finished(&mut self, q: Query, params: &mut impl PollParameters) -> Option { - log::trace!("Query {:?} finished.", q.id()); + let query_id = q.id(); + log::trace!("Query {:?} finished.", query_id); let result = q.into_result(); match result.inner.info { - QueryInfo::Bootstrap { peer } => { + QueryInfo::Bootstrap { peer, remaining } => { let local_key = self.kbuckets.local_key().clone(); - if &peer == local_key.preimage() { + let mut remaining = remaining.unwrap_or_else(|| { + debug_assert_eq!(&peer, local_key.preimage()); // The lookup for the local key finished. To complete the bootstrap process, // a bucket refresh should be performed for every bucket farther away than // the first non-empty bucket (which are most likely no more than the last // few, i.e. farthest, buckets). - let targets = self.kbuckets.buckets() + self.kbuckets.buckets() .skip_while(|b| b.num_entries() == 0) .skip(1) // Skip the bucket with the closest neighbour. .map(|b| { @@ -732,71 +798,112 @@ where target = kbucket::Key::new(PeerId::random()); } target - }).collect::>(); + }).collect::>().into_iter() + }); - for target in targets { - let info = QueryInfo::Bootstrap { peer: target.clone().into_preimage() }; - let peers = self.kbuckets.closest_keys(&target); - let inner = QueryInner::new(info); - self.queries.add_iter_closest(target.clone(), peers, inner); - } + let num_remaining = remaining.len().saturating_sub(1) as u32; + + if let Some(target) = remaining.next() { + let info = QueryInfo::Bootstrap { + peer: target.clone().into_preimage(), + remaining: Some(remaining) + }; + let peers = self.kbuckets.closest_keys(&target); + let inner = QueryInner::new(info); + self.queries.continue_iter_closest(query_id, target.clone(), peers, inner); } - Some(KademliaEvent::BootstrapResult(Ok(BootstrapOk { peer }))) + + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::Bootstrap(Ok(BootstrapOk { peer, num_remaining })) + }) } QueryInfo::GetClosestPeers { key, .. } => { - Some(KademliaEvent::GetClosestPeersResult(Ok( - GetClosestPeersOk { key, peers: result.peers.collect() } - ))) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetClosestPeers(Ok( + GetClosestPeersOk { key, peers: result.peers.collect() } + )) + }) } QueryInfo::GetProviders { key, providers } => { - Some(KademliaEvent::GetProvidersResult(Ok( - GetProvidersOk { - key, - providers, - closest_peers: result.peers.collect() - } - ))) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetProviders(Ok( + GetProvidersOk { + key, + providers, + closest_peers: result.peers.collect() + } + )) + }) } - QueryInfo::PrepareAddProvider { key, context } => { + QueryInfo::AddProvider { + context, + key, + phase: AddProviderPhase::GetClosestPeers + } => { let provider_id = params.local_peer_id().clone(); let external_addresses = params.external_addresses().collect(); let inner = QueryInner::new(QueryInfo::AddProvider { - key, - provider_id, - external_addresses, context, + key, + phase: AddProviderPhase::AddProvider { + provider_id, + external_addresses, + get_closest_peers_stats: result.stats + } }); - self.queries.add_fixed(result.peers, inner); + self.queries.continue_fixed(query_id, result.peers, inner); None } - QueryInfo::AddProvider { key, context, .. } => { + QueryInfo::AddProvider { + context, + key, + phase: AddProviderPhase::AddProvider { get_closest_peers_stats, .. } + } => { match context { AddProviderContext::Publish => { - Some(KademliaEvent::StartProvidingResult(Ok( - AddProviderOk { key } - ))) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: get_closest_peers_stats.merge(result.stats), + result: QueryResult::StartProviding(Ok(AddProviderOk { key })) + }) } AddProviderContext::Republish => { - Some(KademliaEvent::RepublishProviderResult(Ok( - AddProviderOk { key } - ))) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: get_closest_peers_stats.merge(result.stats), + result: QueryResult::RepublishProvider(Ok(AddProviderOk { key })) + }) } } } QueryInfo::GetRecord { key, records, quorum, cache_at } => { - let result = if records.len() >= quorum.get() { // [not empty] + let results = if records.len() >= quorum.get() { // [not empty] if let Some(cache_key) = cache_at { // Cache the record at the closest node to the key that // did not return the record. let record = records.first().expect("[not empty]").clone(); let quorum = NonZeroUsize::new(1).expect("1 > 0"); let context = PutRecordContext::Cache; - let info = QueryInfo::PutRecord { record, quorum, context, num_results: 0 }; + let info = QueryInfo::PutRecord { + context, + record, + quorum, + phase: PutRecordPhase::PutRecord { + num_results: 0, + get_closest_peers_stats: QueryStats::empty() + } + }; let inner = QueryInner::new(info); self.queries.add_fixed(iter::once(cache_key.into_preimage()), inner); } @@ -809,18 +916,40 @@ where } else { Err(GetRecordError::QuorumFailed { key, records, quorum }) }; - Some(KademliaEvent::GetRecordResult(result)) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetRecord(results) + }) } - QueryInfo::PreparePutRecord { record, quorum, context } => { - let info = QueryInfo::PutRecord { record, quorum, context, num_results: 0 }; + QueryInfo::PutRecord { + context, + record, + quorum, + phase: PutRecordPhase::GetClosestPeers + } => { + let info = QueryInfo::PutRecord { + context, + record, + quorum, + phase: PutRecordPhase::PutRecord { + num_results: 0, + get_closest_peers_stats: result.stats + } + }; let inner = QueryInner::new(info); - self.queries.add_fixed(result.peers, inner); + self.queries.continue_fixed(query_id, result.peers, inner); None } - QueryInfo::PutRecord { record, quorum, num_results, context } => { - let result = |key: record::Key| { + QueryInfo::PutRecord { + context, + record, + quorum, + phase: PutRecordPhase::PutRecord { num_results, get_closest_peers_stats } + } => { + let mk_result = |key: record::Key| { if num_results >= quorum.get() { Ok(PutRecordOk { key }) } else { @@ -829,9 +958,17 @@ where }; match context { PutRecordContext::Publish => - Some(KademliaEvent::PutRecordResult(result(record.key))), + Some(KademliaEvent::QueryResult { + id: query_id, + stats: get_closest_peers_stats.merge(result.stats), + result: QueryResult::PutRecord(mk_result(record.key)) + }), PutRecordContext::Republish => - Some(KademliaEvent::RepublishRecordResult(result(record.key))), + Some(KademliaEvent::QueryResult { + id: query_id, + stats: get_closest_peers_stats.merge(result.stats), + result: QueryResult::RepublishRecord(mk_result(record.key)) + }), PutRecordContext::Replicate => { debug!("Record replicated: {:?}", record.key); None @@ -846,99 +983,138 @@ where } /// Handles a query that timed out. - fn query_timeout(&self, query: Query) -> Option { - log::trace!("Query {:?} timed out.", query.id()); + fn query_timeout(&mut self, query: Query) -> Option { + let query_id = query.id(); + log::trace!("Query {:?} timed out.", query_id); let result = query.into_result(); match result.inner.info { - QueryInfo::Bootstrap { peer } => - Some(KademliaEvent::BootstrapResult(Err( - BootstrapError::Timeout { peer }))), + QueryInfo::Bootstrap { peer, mut remaining } => { + let num_remaining = remaining.as_ref().map(|r| r.len().saturating_sub(1) as u32); - QueryInfo::PrepareAddProvider { key, context } => + if let Some(mut remaining) = remaining.take() { + // Continue with the next bootstrap query if `remaining` is not empty. + if let Some(target) = remaining.next() { + let info = QueryInfo::Bootstrap { + peer: target.clone().into_preimage(), + remaining: Some(remaining) + }; + let peers = self.kbuckets.closest_keys(&target); + let inner = QueryInner::new(info); + self.queries.continue_iter_closest(query_id, target.clone(), peers, inner); + } + } + + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::Bootstrap(Err( + BootstrapError::Timeout { peer, num_remaining } + )) + }) + } + + QueryInfo::AddProvider { context, key, .. } => Some(match context { AddProviderContext::Publish => - KademliaEvent::StartProvidingResult(Err( - AddProviderError::Timeout { key })), + KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::StartProviding(Err( + AddProviderError::Timeout { key } + )) + }, AddProviderContext::Republish => - KademliaEvent::RepublishProviderResult(Err( - AddProviderError::Timeout { key })), - }), - - QueryInfo::AddProvider { key, context, .. } => - Some(match context { - AddProviderContext::Publish => - KademliaEvent::StartProvidingResult(Err( - AddProviderError::Timeout { key })), - AddProviderContext::Republish => - KademliaEvent::RepublishProviderResult(Err( - AddProviderError::Timeout { key })), + KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::RepublishProvider(Err( + AddProviderError::Timeout { key } + )) + } }), QueryInfo::GetClosestPeers { key } => { - Some(KademliaEvent::GetClosestPeersResult(Err( - GetClosestPeersError::Timeout { - key, - peers: result.peers.collect() - }))) + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetClosestPeers(Err( + GetClosestPeersError::Timeout { + key, + peers: result.peers.collect() + } + )) + }) }, - QueryInfo::PreparePutRecord { record, quorum, context, .. } => { + QueryInfo::PutRecord { record, quorum, context, phase } => { let err = Err(PutRecordError::Timeout { key: record.key, - num_results: 0, - quorum + quorum, + num_results: match phase { + PutRecordPhase::GetClosestPeers => 0, + PutRecordPhase::PutRecord { num_results, .. } => num_results + } }); match context { PutRecordContext::Publish => - Some(KademliaEvent::PutRecordResult(err)), + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::PutRecord(err) + }), PutRecordContext::Republish => - Some(KademliaEvent::RepublishRecordResult(err)), - PutRecordContext::Replicate => { - warn!("Locating closest peers for replication failed: {:?}", err); - None + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::RepublishRecord(err) + }), + PutRecordContext::Replicate => match phase { + PutRecordPhase::GetClosestPeers => { + warn!("Locating closest peers for replication failed: {:?}", err); + None + } + PutRecordPhase::PutRecord { .. } => { + debug!("Replicating record failed: {:?}", err); + None + } } - PutRecordContext::Cache => - // Caching a record at the closest peer to a key that did not return - // a record is never preceded by a lookup for the closest peers, i.e. - // it is a direct query to a single peer. - unreachable!() - } - } - - QueryInfo::PutRecord { record, quorum, num_results, context } => { - let err = Err(PutRecordError::Timeout { - key: record.key, - num_results, - quorum - }); - match context { - PutRecordContext::Publish => - Some(KademliaEvent::PutRecordResult(err)), - PutRecordContext::Republish => - Some(KademliaEvent::RepublishRecordResult(err)), - PutRecordContext::Replicate => { - debug!("Replicatiing record failed: {:?}", err); - None - } - PutRecordContext::Cache => { - debug!("Caching record failed: {:?}", err); - None + PutRecordContext::Cache => match phase { + PutRecordPhase::GetClosestPeers => { + // Caching a record at the closest peer to a key that did not return + // a record is never preceded by a lookup for the closest peers, i.e. + // it is a direct query to a single peer. + unreachable!() + } + PutRecordPhase::PutRecord { .. } => { + debug!("Caching record failed: {:?}", err); + None + } } } } QueryInfo::GetRecord { key, records, quorum, .. } => - Some(KademliaEvent::GetRecordResult(Err( - GetRecordError::Timeout { key, records, quorum }))), + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetRecord(Err( + GetRecordError::Timeout { key, records, quorum } + )) + }), QueryInfo::GetProviders { key, providers } => - Some(KademliaEvent::GetProvidersResult(Err( - GetProvidersError::Timeout { - key, - providers, - closest_peers: result.peers.collect() - }))), - } + Some(KademliaEvent::QueryResult { + id: query_id, + stats: result.stats, + result: QueryResult::GetProviders(Err( + GetProvidersError::Timeout { + key, + providers, + closest_peers: result.peers.collect() + } + )) + }) + } } /// Processes a record received from a peer. @@ -1300,7 +1476,7 @@ where } = &mut query.inner.info { if let Some(record) = record { records.push(record); - if records.len() == quorum.get() { + if records.len() >= quorum.get() { query.finish() } } else if quorum.get() == 1 { @@ -1337,10 +1513,10 @@ where if let Some(query) = self.queries.get_mut(&user_data) { query.on_success(&source, vec![]); if let QueryInfo::PutRecord { - num_results, quorum, .. + phase: PutRecordPhase::PutRecord { num_results, .. }, quorum, .. } = &mut query.inner.info { *num_results += 1; - if *num_results == quorum.get() { + if *num_results >= quorum.get() { query.finish() } } @@ -1429,7 +1605,10 @@ where // better emit an event when the request has been sent (and report // an error if sending fails), instead of immediately reporting // "success" somewhat prematurely here. - if let QueryInfo::AddProvider { .. } = &query.inner.info { + if let QueryInfo::AddProvider { + phase: AddProviderPhase::AddProvider { .. }, + .. + } = &query.inner.info { query.on_success(&peer_id, vec![]) } if self.connected_peers.contains(&peer_id) { @@ -1488,29 +1667,15 @@ impl Quorum { /// See [`NetworkBehaviour::poll`]. #[derive(Debug)] pub enum KademliaEvent { - /// The result of [`Kademlia::bootstrap`]. - BootstrapResult(BootstrapResult), - - /// The result of [`Kademlia::get_closest_peers`]. - GetClosestPeersResult(GetClosestPeersResult), - - /// The result of [`Kademlia::get_providers`]. - GetProvidersResult(GetProvidersResult), - - /// The result of [`Kademlia::start_providing`]. - StartProvidingResult(AddProviderResult), - - /// The result of a (automatic) republishing of a provider record. - RepublishProviderResult(AddProviderResult), - - /// The result of [`Kademlia::get_record`]. - GetRecordResult(GetRecordResult), - - /// The result of [`Kademlia::put_record`]. - PutRecordResult(PutRecordResult), - - /// The result of a (automatic) republishing of a (value-)record. - RepublishRecordResult(PutRecordResult), + /// A query has produced a result. + QueryResult { + /// The ID of the query that finished. + id: QueryId, + /// The result of the query. + result: QueryResult, + /// Execution statistics from the query. + stats: QueryStats + }, /// A peer has been discovered during a query. Discovered { @@ -1543,6 +1708,34 @@ pub enum KademliaEvent { } } +/// The results of Kademlia queries. +#[derive(Debug)] +pub enum QueryResult { + /// The result of [`Kademlia::bootstrap`]. + Bootstrap(BootstrapResult), + + /// The result of [`Kademlia::get_closest_peers`]. + GetClosestPeers(GetClosestPeersResult), + + /// The result of [`Kademlia::get_providers`]. + GetProviders(GetProvidersResult), + + /// The result of [`Kademlia::start_providing`]. + StartProviding(AddProviderResult), + + /// The result of a (automatic) republishing of a provider record. + RepublishProvider(AddProviderResult), + + /// The result of [`Kademlia::get_record`]. + GetRecord(GetRecordResult), + + /// The result of [`Kademlia::put_record`]. + PutRecord(PutRecordResult), + + /// The result of a (automatic) republishing of a (value-)record. + RepublishRecord(PutRecordResult), +} + /// The result of [`Kademlia::get_record`]. pub type GetRecordResult = Result; @@ -1614,10 +1807,6 @@ pub enum PutRecordError { num_results: usize, quorum: NonZeroUsize }, - LocalStorageError { - key: record::Key, - cause: store::Error - } } impl PutRecordError { @@ -1626,7 +1815,6 @@ impl PutRecordError { match self { PutRecordError::QuorumFailed { key, .. } => key, PutRecordError::Timeout { key, .. } => key, - PutRecordError::LocalStorageError { key, .. } => key } } @@ -1636,7 +1824,6 @@ impl PutRecordError { match self { PutRecordError::QuorumFailed { key, .. } => key, PutRecordError::Timeout { key, .. } => key, - PutRecordError::LocalStorageError { key, .. } => key, } } } @@ -1647,13 +1834,17 @@ pub type BootstrapResult = Result; /// The successful result of [`Kademlia::bootstrap`]. #[derive(Debug, Clone)] pub struct BootstrapOk { - pub peer: PeerId + pub peer: PeerId, + pub num_remaining: u32, } /// The error result of [`Kademlia::bootstrap`]. #[derive(Debug, Clone)] pub enum BootstrapError { - Timeout { peer: PeerId } + Timeout { + peer: PeerId, + num_remaining: Option, + } } /// The result of [`Kademlia::get_closest_peers`]. @@ -1746,11 +1937,6 @@ pub enum AddProviderError { Timeout { key: record::Key, }, - /// The provider record could not be stored. - LocalStorageError { - key: record::Key, - cause: store::Error - } } impl AddProviderError { @@ -1758,16 +1944,13 @@ impl AddProviderError { pub fn key(&self) -> &record::Key { match self { AddProviderError::Timeout { key, .. } => key, - AddProviderError::LocalStorageError { key, .. } => key, } } /// Extracts the key for which the operation failed, - /// consuming the error. pub fn into_key(self) -> record::Key { match self { AddProviderError::Timeout { key, .. } => key, - AddProviderError::LocalStorageError { key, .. } => key, } } } @@ -1810,33 +1993,42 @@ impl QueryInner { } } +/// The context of a [`QueryInfo::AddProvider`] query. #[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum AddProviderContext { +pub enum AddProviderContext { Publish, Republish, } +/// The context of a [`QueryInfo::PutRecord`] query. #[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum PutRecordContext { +pub enum PutRecordContext { Publish, Republish, Replicate, Cache, } -/// The internal query state. -#[derive(Debug, Clone, PartialEq, Eq)] -enum QueryInfo { - /// A bootstrapping query. +/// Information about a running query. +#[derive(Debug, Clone)] +pub enum QueryInfo { + /// A query initiated by [`Kademlia::bootstrap`]. Bootstrap { /// The targeted peer ID. peer: PeerId, + /// The remaining random peer IDs to query, one per + /// bucket that still needs refreshing. + /// + /// This is `None` if the initial self-lookup has not + /// yet completed and `Some` with an exhausted iterator + /// if bootstrapping is complete. + remaining: Option>> }, - /// A query to find the closest peers to a key. + /// A query initiated by [`Kademlia::get_closest_peers`]. GetClosestPeers { key: Vec }, - /// A query for the providers of a key. + /// A query initiated by [`Kademlia::get_providers`]. GetProviders { /// The key for which to search for providers. key: record::Key, @@ -1844,49 +2036,39 @@ enum QueryInfo { providers: HashSet, }, - /// A query that searches for the closest closest nodes to a key to be - /// used in a subsequent `AddProvider` query. - PrepareAddProvider { - key: record::Key, - context: AddProviderContext, - }, - - /// A query that advertises the local node as a provider for a key. + /// A (repeated) query initiated by [`Kademlia::start_providing`]. AddProvider { + /// The record key. key: record::Key, - provider_id: PeerId, - external_addresses: Vec, + /// The current phase of the query. + phase: AddProviderPhase, + /// The execution context of the query. context: AddProviderContext, }, - /// A query that searches for the closest closest nodes to a key to be used - /// in a subsequent `PutValue` query. - PreparePutRecord { - record: Record, - quorum: NonZeroUsize, - context: PutRecordContext, - }, - - /// A query that replicates a record to other nodes. + /// A (repeated) query initiated by [`Kademlia::put_record`]. PutRecord { record: Record, + /// The expected quorum of responses w.r.t. the replication factor. quorum: NonZeroUsize, - num_results: usize, + /// The current phase of the query. + phase: PutRecordPhase, + /// The execution context of the query. context: PutRecordContext, }, - /// A query that searches for values for a key. + /// A query initiated by [`Kademlia::get_record`]. GetRecord { /// The key to look for. key: record::Key, - /// The records found. + /// The records found so far. records: Vec, /// The number of records to look for. quorum: NonZeroUsize, /// The closest peer to `key` that did not return a record. /// /// When a record is found in a standard Kademlia query (quorum == 1), - /// it is cached at this peer. + /// it is cached at this peer as soon as a record is found. cache_at: Option>, }, } @@ -1896,7 +2078,7 @@ impl QueryInfo { /// context of a query. fn to_request(&self, query_id: QueryId) -> KademliaHandlerIn { match &self { - QueryInfo::Bootstrap { peer } => KademliaHandlerIn::FindNodeReq { + QueryInfo::Bootstrap { peer, .. } => KademliaHandlerIn::FindNodeReq { key: peer.clone().into_bytes(), user_data: query_id, }, @@ -1908,35 +2090,135 @@ impl QueryInfo { key: key.clone(), user_data: query_id, }, - QueryInfo::PrepareAddProvider { key, .. } => KademliaHandlerIn::FindNodeReq { - key: key.to_vec(), - user_data: query_id, - }, - QueryInfo::AddProvider { - key, - provider_id, - external_addresses, - .. - } => KademliaHandlerIn::AddProvider { - key: key.clone(), - provider: crate::protocol::KadPeer { - node_id: provider_id.clone(), - multiaddrs: external_addresses.clone(), - connection_ty: crate::protocol::KadConnectionType::Connected, + QueryInfo::AddProvider { key, phase, .. } => match phase { + AddProviderPhase::GetClosestPeers => KademliaHandlerIn::FindNodeReq { + key: key.to_vec(), + user_data: query_id, + }, + AddProviderPhase::AddProvider { provider_id, external_addresses, .. } => { + KademliaHandlerIn::AddProvider { + key: key.clone(), + provider: crate::protocol::KadPeer { + node_id: provider_id.clone(), + multiaddrs: external_addresses.clone(), + connection_ty: crate::protocol::KadConnectionType::Connected, + } + } } }, QueryInfo::GetRecord { key, .. } => KademliaHandlerIn::GetRecord { key: key.clone(), user_data: query_id, }, - QueryInfo::PreparePutRecord { record, .. } => KademliaHandlerIn::FindNodeReq { - key: record.key.to_vec(), - user_data: query_id, - }, - QueryInfo::PutRecord { record, .. } => KademliaHandlerIn::PutRecord { - record: record.clone(), - user_data: query_id + QueryInfo::PutRecord { record, phase, .. } => match phase { + PutRecordPhase::GetClosestPeers => KademliaHandlerIn::FindNodeReq { + key: record.key.to_vec(), + user_data: query_id, + }, + PutRecordPhase::PutRecord { .. } => KademliaHandlerIn::PutRecord { + record: record.clone(), + user_data: query_id + } } } } } + +/// The phases of a [`QueryInfo::AddProvider`] query. +#[derive(Debug, Clone)] +pub enum AddProviderPhase { + /// The query is searching for the closest nodes to the record key. + GetClosestPeers, + + /// The query advertises the local node as a provider for the key to + /// the closest nodes to the key. + AddProvider { + /// The local peer ID that is advertised as a provider. + provider_id: PeerId, + /// The external addresses of the provider being advertised. + external_addresses: Vec, + /// Query statistics from the finished `GetClosestPeers` phase. + get_closest_peers_stats: QueryStats, + }, +} + +/// The phases of a [`QueryInfo::PutRecord`] query. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PutRecordPhase { + /// The query is searching for the closest nodes to the record key. + GetClosestPeers, + + /// The query is replicating the record to the closest nodes to the key. + PutRecord { + /// The number of successful replication requests so far. + num_results: usize, + /// Query statistics from the finished `GetClosestPeers` phase. + get_closest_peers_stats: QueryStats, + }, +} + +/// A mutable reference to a running query. +pub struct QueryMut<'a> { + query: &'a mut Query, +} + +impl<'a> QueryMut<'a> { + pub fn id(&self) -> QueryId { + self.query.id() + } + + /// Gets information about the type and state of the query. + pub fn info(&self) -> &QueryInfo { + &self.query.inner.info + } + + /// Gets execution statistics about the query. + /// + /// For a multi-phase query such as `put_record`, these are the + /// statistics of the current phase. + pub fn stats(&self) -> &QueryStats { + self.query.stats() + } + + /// Finishes the query asap, without waiting for the + /// regular termination conditions. + pub fn finish(&mut self) { + self.query.finish() + } +} + +/// An immutable reference to a running query. +pub struct QueryRef<'a> { + query: &'a Query, +} + +impl<'a> QueryRef<'a> { + pub fn id(&self) -> QueryId { + self.query.id() + } + + /// Gets information about the type and state of the query. + pub fn info(&self) -> &QueryInfo { + &self.query.inner.info + } + + /// Gets execution statistics about the query. + /// + /// For a multi-phase query such as `put_record`, these are the + /// statistics of the current phase. + pub fn stats(&self) -> &QueryStats { + self.query.stats() + } +} + +/// An operation failed to due no known peers in the routing table. +#[derive(Debug, Clone)] +pub struct NoKnownPeers(); + +impl fmt::Display for NoKnownPeers { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "No known peers.") + } +} + +impl std::error::Error for NoKnownPeers {} diff --git a/protocols/kad/src/behaviour/test.rs b/protocols/kad/src/behaviour/test.rs index afeda26a..6e64f67a 100644 --- a/protocols/kad/src/behaviour/test.rs +++ b/protocols/kad/src/behaviour/test.rs @@ -148,10 +148,11 @@ fn bootstrap() { .collect::>(); let swarm_ids: Vec<_> = swarms.iter().map(Swarm::local_peer_id).cloned().collect(); - swarms[0].bootstrap(); + let qid = swarms[0].bootstrap().unwrap(); // Expected known peers let expected_known = swarm_ids.iter().skip(1).cloned().collect::>(); + let mut first = true; // Run test block_on( @@ -159,14 +160,23 @@ fn bootstrap() { for (i, swarm) in swarms.iter_mut().enumerate() { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::BootstrapResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::Bootstrap(Ok(ok)), .. + })) => { + assert_eq!(id, qid); assert_eq!(i, 0); - assert_eq!(ok.peer, swarm_ids[0]); - let known = swarm.kbuckets.iter() - .map(|e| e.node.key.preimage().clone()) - .collect::>(); - assert_eq!(expected_known, known); - return Poll::Ready(()) + if first { + // Bootstrapping must start with a self-lookup. + assert_eq!(ok.peer, swarm_ids[0]); + } + first = false; + if ok.num_remaining == 0 { + let known = swarm.kbuckets.iter() + .map(|e| e.node.key.preimage().clone()) + .collect::>(); + assert_eq!(expected_known, known); + return Poll::Ready(()) + } } // Ignore any other event. Poll::Ready(Some(_)) => (), @@ -206,7 +216,17 @@ fn query_iter() { // propagate forwards through the list of peers. let search_target = PeerId::random(); let search_target_key = kbucket::Key::new(search_target.clone()); - swarms[0].get_closest_peers(search_target.clone()); + let qid = swarms[0].get_closest_peers(search_target.clone()); + + match swarms[0].query(&qid) { + Some(q) => match q.info() { + QueryInfo::GetClosestPeers { key } => { + assert_eq!(&key[..], search_target.borrow() as &[u8]) + }, + i => panic!("Unexpected query info: {:?}", i) + } + None => panic!("Query not found: {:?}", qid) + } // Set up expectations. let expected_swarm_id = swarm_ids[0].clone(); @@ -220,7 +240,10 @@ fn query_iter() { for (i, swarm) in swarms.iter_mut().enumerate() { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetClosestPeersResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::GetClosestPeers(Ok(ok)), .. + })) => { + assert_eq!(id, qid); assert_eq!(&ok.key[..], search_target.as_bytes()); assert_eq!(swarm_ids[i], expected_swarm_id); assert_eq!(swarm.queries.size(), 0); @@ -270,7 +293,9 @@ fn unresponsive_not_returned_direct() { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetClosestPeersResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + result: QueryResult::GetClosestPeers(Ok(ok)), .. + })) => { assert_eq!(&ok.key[..], search_target.as_bytes()); assert_eq!(ok.peers.len(), 0); return Poll::Ready(()); @@ -318,7 +343,9 @@ fn unresponsive_not_returned_indirect() { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetClosestPeersResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + result: QueryResult::GetClosestPeers(Ok(ok)), .. + })) => { assert_eq!(&ok.key[..], search_target.as_bytes()); assert_eq!(ok.peers.len(), 1); assert_eq!(ok.peers[0], first_peer_id); @@ -354,14 +381,17 @@ fn get_record_not_found() { let mut swarms = swarms.into_iter().map(|(_addr, swarm)| swarm).collect::>(); let target_key = record::Key::from(random_multihash()); - swarms[0].get_record(&target_key, Quorum::One); + let qid = swarms[0].get_record(&target_key, Quorum::One); block_on( poll_fn(move |ctx| { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetRecordResult(Err(e)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::GetRecord(Err(e)), .. + })) => { + assert_eq!(id, qid); if let GetRecordError::NotFound { key, closest_peers, } = e { assert_eq!(key, target_key); assert_eq!(closest_peers.len(), 2); @@ -426,8 +456,23 @@ fn put_record() { }) .collect::>(); + // Initiate put_record queries. + let mut qids = HashSet::new(); for r in records.values() { - swarms[0].put_record(r.clone(), Quorum::All); + let qid = swarms[0].put_record(r.clone(), Quorum::All).unwrap(); + match swarms[0].query(&qid) { + Some(q) => match q.info() { + QueryInfo::PutRecord { phase, record, .. } => { + assert_eq!(phase, &PutRecordPhase::GetClosestPeers); + assert_eq!(record.key, r.key); + assert_eq!(record.value, r.value); + assert!(record.expires.is_some()); + qids.insert(qid); + }, + i => panic!("Unexpected query info: {:?}", i) + } + None => panic!("Query not found: {:?}", qid) + } } // Each test run republishes all records once. @@ -441,8 +486,17 @@ fn put_record() { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::PutRecordResult(res))) | - Poll::Ready(Some(KademliaEvent::RepublishRecordResult(res))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::PutRecord(res), stats + })) | + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::RepublishRecord(res), stats + })) => { + assert!(qids.is_empty() || qids.remove(&id)); + assert!(stats.duration().is_some()); + assert!(stats.num_successes() >= replication_factor.get() as u32); + assert!(stats.num_requests() >= stats.num_successes()); + assert_eq!(stats.num_failures(), 0); match res { Err(e) => panic!("{:?}", e), Ok(ok) => { @@ -541,7 +595,7 @@ fn put_record() { } #[test] -fn get_value() { +fn get_record() { let mut swarms = build_nodes(3); // Let first peer know of second peer and second peer know of third peer. @@ -556,14 +610,17 @@ fn get_value() { let record = Record::new(random_multihash(), vec![4,5,6]); swarms[1].store.put(record.clone()).unwrap(); - swarms[0].get_record(&record.key, Quorum::One); + let qid = swarms[0].get_record(&record.key, Quorum::One); block_on( poll_fn(move |ctx| { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetRecordResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::GetRecord(Ok(ok)), .. + })) => { + assert_eq!(id, qid); assert_eq!(ok.records.len(), 1); assert_eq!(ok.records.first(), Some(&record)); return Poll::Ready(()); @@ -582,7 +639,7 @@ fn get_value() { } #[test] -fn get_value_many() { +fn get_record_many() { // TODO: Randomise let num_nodes = 12; let mut swarms = build_connected_nodes(num_nodes, 3).into_iter() @@ -597,14 +654,17 @@ fn get_value_many() { } let quorum = Quorum::N(NonZeroUsize::new(num_results).unwrap()); - swarms[0].get_record(&record.key, quorum); + let qid = swarms[0].get_record(&record.key, quorum); block_on( poll_fn(move |ctx| { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::GetRecordResult(Ok(ok)))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::GetRecord(Ok(ok)), .. + })) => { + assert_eq!(id, qid); assert_eq!(ok.records.len(), num_results); assert_eq!(ok.records.first(), Some(&record)); return Poll::Ready(()); @@ -661,8 +721,10 @@ fn add_provider() { let mut results = Vec::new(); // Initiate the first round of publishing. + let mut qids = HashSet::new(); for k in &keys { - swarms[0].start_providing(k.clone()); + let qid = swarms[0].start_providing(k.clone()).unwrap(); + qids.insert(qid); } block_on( @@ -671,8 +733,13 @@ fn add_provider() { for swarm in &mut swarms { loop { match swarm.poll_next_unpin(ctx) { - Poll::Ready(Some(KademliaEvent::StartProvidingResult(res))) | - Poll::Ready(Some(KademliaEvent::RepublishProviderResult(res))) => { + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::StartProviding(res), .. + })) | + Poll::Ready(Some(KademliaEvent::QueryResult { + id, result: QueryResult::RepublishProvider(res), .. + })) => { + assert!(qids.is_empty() || qids.remove(&id)); match res { Err(e) => panic!(e), Ok(ok) => { @@ -773,7 +840,7 @@ fn exceed_jobs_max_queries() { let (_addr, mut swarm) = build_node(); let num = JOBS_MAX_QUERIES + 1; for _ in 0 .. num { - swarm.bootstrap(); + swarm.get_closest_peers(PeerId::random()); } assert_eq!(swarm.queries.size(), num); @@ -783,8 +850,10 @@ fn exceed_jobs_max_queries() { for _ in 0 .. num { // There are no other nodes, so the queries finish instantly. if let Poll::Ready(Some(e)) = swarm.poll_next_unpin(ctx) { - if let KademliaEvent::BootstrapResult(r) = e { - assert!(r.is_ok(), "Unexpected error") + if let KademliaEvent::QueryResult { + result: QueryResult::GetClosestPeers(Ok(r)), .. + } = e { + assert!(r.peers.is_empty()) } else { panic!("Unexpected event: {:?}", e) } diff --git a/protocols/kad/src/lib.rs b/protocols/kad/src/lib.rs index 5da1fe07..aa13374f 100644 --- a/protocols/kad/src/lib.rs +++ b/protocols/kad/src/lib.rs @@ -41,6 +41,10 @@ mod dht_proto { pub use addresses::Addresses; pub use behaviour::{Kademlia, KademliaConfig, KademliaEvent, Quorum}; pub use behaviour::{ + QueryResult, + QueryInfo, + QueryStats, + BootstrapResult, BootstrapOk, BootstrapError, diff --git a/protocols/kad/src/query.rs b/protocols/kad/src/query.rs index 71d95f0d..706ca622 100644 --- a/protocols/kad/src/query.rs +++ b/protocols/kad/src/query.rs @@ -91,13 +91,38 @@ impl QueryPool { where I: IntoIterator { + let id = self.next_query_id(); + self.continue_fixed(id, peers, inner); + id + } + + /// Continues an earlier query with a fixed set of peers, reusing + /// the given query ID, which must be from a query that finished + /// earlier. + pub fn continue_fixed(&mut self, id: QueryId, peers: I, inner: TInner) + where + I: IntoIterator + { + assert!(!self.queries.contains_key(&id)); let parallelism = self.config.replication_factor.get(); let peer_iter = QueryPeerIter::Fixed(FixedPeersIter::new(peers, parallelism)); - self.add(peer_iter, inner) + let query = Query::new(id, peer_iter, inner); + self.queries.insert(id, query); } /// Adds a query to the pool that iterates towards the closest peers to the target. pub fn add_iter_closest(&mut self, target: T, peers: I, inner: TInner) -> QueryId + where + T: Into, + I: IntoIterator> + { + let id = self.next_query_id(); + self.continue_iter_closest(id, target, peers, inner); + id + } + + /// Adds a query to the pool that iterates towards the closest peers to the target. + pub fn continue_iter_closest(&mut self, id: QueryId, target: T, peers: I, inner: TInner) where T: Into, I: IntoIterator> @@ -107,14 +132,13 @@ impl QueryPool { .. ClosestPeersIterConfig::default() }; let peer_iter = QueryPeerIter::Closest(ClosestPeersIter::with_config(cfg, target, peers)); - self.add(peer_iter, inner) - } - - fn add(&mut self, peer_iter: QueryPeerIter, inner: TInner) -> QueryId { - let id = QueryId(self.next_id); - self.next_id = self.next_id.wrapping_add(1); let query = Query::new(id, peer_iter, inner); self.queries.insert(id, query); + } + + fn next_query_id(&mut self) -> QueryId { + let id = QueryId(self.next_id); + self.next_id = self.next_id.wrapping_add(1); id } @@ -135,7 +159,7 @@ impl QueryPool { let mut waiting = None; for (&query_id, query) in self.queries.iter_mut() { - query.started = query.started.or(Some(now)); + query.stats.start = query.stats.start.or(Some(now)); match query.next(now) { PeersIterState::Finished => { finished = Some(query_id); @@ -147,7 +171,7 @@ impl QueryPool { break } PeersIterState::Waiting(None) | PeersIterState::WaitingAtCapacity => { - let elapsed = now - query.started.unwrap_or(now); + let elapsed = now - query.stats.start.unwrap_or(now); if elapsed >= self.config.timeout { timeout = Some(query_id); break @@ -162,12 +186,14 @@ impl QueryPool { } if let Some(query_id) = finished { - let query = self.queries.remove(&query_id).expect("s.a."); + let mut query = self.queries.remove(&query_id).expect("s.a."); + query.stats.end = Some(now); return QueryPoolState::Finished(query) } if let Some(query_id) = timeout { - let query = self.queries.remove(&query_id).expect("s.a."); + let mut query = self.queries.remove(&query_id).expect("s.a."); + query.stats.end = Some(now); return QueryPoolState::Timeout(query) } @@ -205,9 +231,8 @@ pub struct Query { id: QueryId, /// The peer iterator that drives the query state. peer_iter: QueryPeerIter, - /// The instant when the query started (i.e. began waiting for the first - /// result from a peer). - started: Option, + /// Execution statistics of the query. + stats: QueryStats, /// The opaque inner query state. pub inner: TInner, } @@ -221,7 +246,7 @@ enum QueryPeerIter { impl Query { /// Creates a new query without starting it. fn new(id: QueryId, peer_iter: QueryPeerIter, inner: TInner) -> Self { - Query { id, inner, peer_iter, started: None } + Query { id, inner, peer_iter, stats: QueryStats::empty() } } /// Gets the unique ID of the query. @@ -229,11 +254,19 @@ impl Query { self.id } + /// Gets the current execution statistics of the query. + pub fn stats(&self) -> &QueryStats { + &self.stats + } + /// Informs the query that the attempt to contact `peer` failed. pub fn on_failure(&mut self, peer: &PeerId) { - match &mut self.peer_iter { + let updated = match &mut self.peer_iter { QueryPeerIter::Closest(iter) => iter.on_failure(peer), QueryPeerIter::Fixed(iter) => iter.on_failure(peer) + }; + if updated { + self.stats.failure += 1; } } @@ -244,9 +277,12 @@ impl Query { where I: IntoIterator { - match &mut self.peer_iter { + let updated = match &mut self.peer_iter { QueryPeerIter::Closest(iter) => iter.on_success(peer, new_peers), QueryPeerIter::Fixed(iter) => iter.on_success(peer) + }; + if updated { + self.stats.success += 1; } } @@ -260,10 +296,16 @@ impl Query { /// Advances the state of the underlying peer iterator. fn next(&mut self, now: Instant) -> PeersIterState { - match &mut self.peer_iter { + let state = match &mut self.peer_iter { QueryPeerIter::Closest(iter) => iter.next(now), QueryPeerIter::Fixed(iter) => iter.next() + }; + + if let PeersIterState::Waiting(Some(_)) = state { + self.stats.requests += 1; } + + state } /// Finishes the query prematurely. @@ -277,13 +319,24 @@ impl Query { } } + /// Checks whether the query has finished. + /// + /// A finished query is eventually reported by `QueryPool::next()` and + /// removed from the pool. + pub fn is_finished(&self) -> bool { + match &self.peer_iter { + QueryPeerIter::Closest(iter) => iter.is_finished(), + QueryPeerIter::Fixed(iter) => iter.is_finished() + } + } + /// Consumes the query, producing the final `QueryResult`. pub fn into_result(self) -> QueryResult> { let peers = match self.peer_iter { QueryPeerIter::Closest(iter) => Either::Left(iter.into_result()), QueryPeerIter::Fixed(iter) => Either::Right(iter.into_result()) }; - QueryResult { inner: self.inner, peers } + QueryResult { peers, inner: self.inner, stats: self.stats } } } @@ -292,5 +345,90 @@ pub struct QueryResult { /// The opaque inner query state. pub inner: TInner, /// The successfully contacted peers. - pub peers: TPeers + pub peers: TPeers, + /// The collected query statistics. + pub stats: QueryStats +} + +/// Execution statistics of a query. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct QueryStats { + requests: u32, + success: u32, + failure: u32, + start: Option, + end: Option +} + +impl QueryStats { + pub fn empty() -> Self { + QueryStats { + requests: 0, + success: 0, + failure: 0, + start: None, + end: None, + } + } + + /// Gets the total number of requests initiated by the query. + pub fn num_requests(&self) -> u32 { + self.requests + } + + /// Gets the number of successful requests. + pub fn num_successes(&self) -> u32 { + self.success + } + + /// Gets the number of failed requests. + pub fn num_failures(&self) -> u32 { + self.failure + } + + /// Gets the number of pending requests. + /// + /// > **Note**: A query can finish while still having pending + /// > requests, if the termination conditions are already met. + pub fn num_pending(&self) -> u32 { + self.requests - (self.success + self.failure) + } + + /// Gets the duration of the query. + /// + /// If the query has not yet finished, the duration is measured from the + /// start of the query to the current instant. + /// + /// If the query did not yet start (i.e. yield the first peer to contact), + /// `None` is returned. + pub fn duration(&self) -> Option { + if let Some(s) = self.start { + if let Some(e) = self.end { + Some(e - s) + } else { + Some(Instant::now() - s) + } + } else { + None + } + } + + /// Merges these stats with the given stats of another query, + /// e.g. to accumulate statistics from a multi-phase query. + /// + /// Counters are merged cumulatively while the instants for + /// start and end of the queries are taken as the minimum and + /// maximum, respectively. + pub fn merge(self, other: QueryStats) -> Self { + QueryStats { + requests: self.requests + other.requests, + success: self.success + other.success, + failure: self.failure + other.failure, + start: match (self.start, other.start) { + (Some(a), Some(b)) => Some(std::cmp::min(a, b)), + (a, b) => a.or(b) + }, + end: std::cmp::max(self.end, other.end) + } + } } diff --git a/protocols/kad/src/query/peers/closest.rs b/protocols/kad/src/query/peers/closest.rs index 8879c58c..dda9b716 100644 --- a/protocols/kad/src/query/peers/closest.rs +++ b/protocols/kad/src/query/peers/closest.rs @@ -122,8 +122,7 @@ impl ClosestPeersIter { } } - /// Callback for delivering the result of a successful request to a peer - /// that the iterator is waiting on. + /// Callback for delivering the result of a successful request to a peer. /// /// Delivering results of requests back to the iterator allows the iterator to make /// progress. The iterator is said to make progress either when the given @@ -131,18 +130,20 @@ impl ClosestPeersIter { /// or when the iterator did not yet accumulate `num_results` closest peers and /// `closer_peers` contains a new peer, regardless of its distance to the target. /// - /// After calling this function, `next` should eventually be called again - /// to advance the state of the iterator. + /// If the iterator is currently waiting for a result from `peer`, + /// the iterator state is updated and `true` is returned. In that + /// case, after calling this function, `next` should eventually be + /// called again to obtain the new state of the iterator. /// /// If the iterator is finished, it is not currently waiting for a /// result from `peer`, or a result for `peer` has already been reported, - /// calling this function has no effect. - pub fn on_success(&mut self, peer: &PeerId, closer_peers: I) + /// calling this function has no effect and `false` is returned. + pub fn on_success(&mut self, peer: &PeerId, closer_peers: I) -> bool where I: IntoIterator { if let State::Finished = self.state { - return + return false } let key = Key::from(peer.clone()); @@ -150,7 +151,7 @@ impl ClosestPeersIter { // Mark the peer as succeeded. match self.closest_peers.entry(distance) { - Entry::Vacant(..) => return, + Entry::Vacant(..) => return false, Entry::Occupied(mut e) => match e.get().state { PeerState::Waiting(..) => { debug_assert!(self.num_waiting > 0); @@ -162,7 +163,7 @@ impl ClosestPeersIter { } PeerState::NotContacted | PeerState::Failed - | PeerState::Succeeded => return + | PeerState::Succeeded => return false } } @@ -199,28 +200,31 @@ impl ClosestPeersIter { State::Stalled } State::Finished => State::Finished - } + }; + + true } - /// Callback for informing the iterator about a failed request to a peer - /// that the iterator is waiting on. + /// Callback for informing the iterator about a failed request to a peer. /// - /// After calling this function, `next` should eventually be called again - /// to advance the state of the iterator. + /// If the iterator is currently waiting for a result from `peer`, + /// the iterator state is updated and `true` is returned. In that + /// case, after calling this function, `next` should eventually be + /// called again to obtain the new state of the iterator. /// /// If the iterator is finished, it is not currently waiting for a /// result from `peer`, or a result for `peer` has already been reported, - /// calling this function has no effect. - pub fn on_failure(&mut self, peer: &PeerId) { + /// calling this function has no effect and `false` is returned. + pub fn on_failure(&mut self, peer: &PeerId) -> bool { if let State::Finished = self.state { - return + return false } let key = Key::from(peer.clone()); let distance = key.distance(&self.target); match self.closest_peers.entry(distance) { - Entry::Vacant(_) => return, + Entry::Vacant(_) => return false, Entry::Occupied(mut e) => match e.get().state { PeerState::Waiting(_) => { debug_assert!(self.num_waiting > 0); @@ -230,9 +234,13 @@ impl ClosestPeersIter { PeerState::Unresponsive => { e.get_mut().state = PeerState::Failed } - _ => {} + PeerState::NotContacted + | PeerState::Failed + | PeerState::Succeeded => return false } } + + true } /// Returns the list of peers for which the iterator is currently waiting @@ -343,7 +351,7 @@ impl ClosestPeersIter { } /// Checks whether the iterator has finished. - pub fn finished(&self) -> bool { + pub fn is_finished(&self) -> bool { self.state == State::Finished } @@ -649,7 +657,7 @@ mod tests { match iter.next(now) { PeersIterState::Waiting(Some(p)) => { let peer2 = p.into_owned(); - iter.on_success(&peer2, closer.clone()) + assert!(iter.on_success(&peer2, closer.clone())) } PeersIterState::Finished => {} _ => panic!("Unexpectedly iter state."), @@ -689,7 +697,7 @@ mod tests { Peer { state, .. } => panic!("Unexpected peer state: {:?}", state) } - let finished = iter.finished(); + let finished = iter.is_finished(); iter.on_success(&peer, iter::empty()); let closest = iter.into_result().collect::>(); diff --git a/protocols/kad/src/query/peers/fixed.rs b/protocols/kad/src/query/peers/fixed.rs index 402a4c2b..edb86ef4 100644 --- a/protocols/kad/src/query/peers/fixed.rs +++ b/protocols/kad/src/query/peers/fixed.rs @@ -39,6 +39,7 @@ pub struct FixedPeersIter { state: State, } +#[derive(Debug, PartialEq, Eq)] enum State { Waiting { num_waiting: usize }, Finished @@ -71,22 +72,46 @@ impl FixedPeersIter { } } - pub fn on_success(&mut self, peer: &PeerId) { + /// Callback for delivering the result of a successful request to a peer. + /// + /// If the iterator is currently waiting for a result from `peer`, + /// the iterator state is updated and `true` is returned. In that + /// case, after calling this function, `next` should eventually be + /// called again to obtain the new state of the iterator. + /// + /// If the iterator is finished, it is not currently waiting for a + /// result from `peer`, or a result for `peer` has already been reported, + /// calling this function has no effect and `false` is returned. + pub fn on_success(&mut self, peer: &PeerId) -> bool { if let State::Waiting { num_waiting } = &mut self.state { if let Some(state @ PeerState::Waiting) = self.peers.get_mut(peer) { *state = PeerState::Succeeded; *num_waiting -= 1; + return true } } + false } - pub fn on_failure(&mut self, peer: &PeerId) { + /// Callback for informing the iterator about a failed request to a peer. + /// + /// If the iterator is currently waiting for a result from `peer`, + /// the iterator state is updated and `true` is returned. In that + /// case, after calling this function, `next` should eventually be + /// called again to obtain the new state of the iterator. + /// + /// If the iterator is finished, it is not currently waiting for a + /// result from `peer`, or a result for `peer` has already been reported, + /// calling this function has no effect and `false` is returned. + pub fn on_failure(&mut self, peer: &PeerId) -> bool { if let State::Waiting { num_waiting } = &mut self.state { if let Some(state @ PeerState::Waiting) = self.peers.get_mut(peer) { *state = PeerState::Failed; *num_waiting -= 1; + return true } } + false } pub fn is_waiting(&self, peer: &PeerId) -> bool { @@ -99,6 +124,11 @@ impl FixedPeersIter { } } + /// Checks whether the iterator has finished. + pub fn is_finished(&self) -> bool { + self.state == State::Finished + } + pub fn next(&mut self) -> PeersIterState { match &mut self.state { State::Finished => return PeersIterState::Finished, From 82156deac531b764f5cb5bf9936a973aa1b34322 Mon Sep 17 00:00:00 2001 From: Mikhail Zabaluev Date: Mon, 18 May 2020 11:21:55 +0300 Subject: [PATCH 14/25] Support /dns protocol in multiaddr (#1575) * Add /dns protocol support to multiaddr The /dns protocol has been added to the spec and has had a de-facto meaning for years. See https://github.com/multiformats/multiaddr/pull/100 This adds address parsing and encoding support for /dns to the multiaddr format library. * Cover Dns protocol in multiaddr property tests * transports/dns: Support the /dns protocol * Support /dns protocol in address translation * Translate an FQDN URL into a /dns multiaddr * transports/websocket: Support /dns multiaddr * Use the /dns protocol in websocket redirects The whole thing with back-translating from an redirect URL looks a bit baroque, but at least now the transport does not completely ignore IPv6 addresses resolved from a hostname in a redirect URL. * Add CHANGELOG entry Co-authored-by: Pierre Krieger --- CHANGELOG.md | 5 +++ core/src/translation.rs | 7 ++++- misc/multiaddr/src/from_url.rs | 14 ++++----- misc/multiaddr/src/protocol.rs | 19 ++++++++++++ misc/multiaddr/tests/lib.rs | 49 +++++++++++++++--------------- transports/dns/src/lib.rs | 12 ++++++-- transports/websocket/src/framed.rs | 4 ++- 7 files changed, 74 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a926161b..551ab1c3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,11 @@ - `libp2p-core`: Make the number of events buffered to/from tasks configurable. [PR 1574](https://github.com/libp2p/rust-libp2p/pull/1574) +- `libp2p-dns`, `parity-multiaddr`: Added support for the `/dns` multiaddr + protocol. Additionally, the `multiaddr::from_url` function will now use + `/dns` instead of `/dns4`. + [PR 1575](https://github.com/libp2p/rust-libp2p/pull/1575) + - `libp2p-noise`: Added the `X25519Spec` protocol suite which uses libp2p-noise-spec compliant signatures on static keys as well as the `/noise` protocol upgrade, hence providing a libp2p-noise-spec compliant diff --git a/core/src/translation.rs b/core/src/translation.rs index 5dab82a0..70700ca6 100644 --- a/core/src/translation.rs +++ b/core/src/translation.rs @@ -37,9 +37,14 @@ use multiaddr::{Multiaddr, Protocol}; /// If the first [`Protocol`]s are not IP addresses, `None` is returned instead. pub fn address_translation(original: &Multiaddr, observed: &Multiaddr) -> Option { original.replace(0, move |proto| match proto { - Protocol::Ip4(_) | Protocol::Ip6(_) | Protocol::Dns4(_) | Protocol::Dns6(_) => match observed.iter().next() { + Protocol::Ip4(_) + | Protocol::Ip6(_) + | Protocol::Dns(_) + | Protocol::Dns4(_) + | Protocol::Dns6(_) => match observed.iter().next() { x @ Some(Protocol::Ip4(_)) => x, x @ Some(Protocol::Ip6(_)) => x, + x @ Some(Protocol::Dns(_)) => x, x @ Some(Protocol::Dns4(_)) => x, x @ Some(Protocol::Dns6(_)) => x, _ => None, diff --git a/misc/multiaddr/src/from_url.rs b/misc/multiaddr/src/from_url.rs index 454672da..738ed20b 100644 --- a/misc/multiaddr/src/from_url.rs +++ b/misc/multiaddr/src/from_url.rs @@ -70,7 +70,7 @@ fn from_url_inner_http_ws(url: url::Url, lossy: bool) -> std::result::Result() { Protocol::from(ip) } else { - Protocol::Dns4(hostname.into()) + Protocol::Dns(hostname.into()) } } else { return Err(FromUrlErr::BadUrl); @@ -185,31 +185,31 @@ mod tests { #[test] fn dns_addr_ws() { let addr = from_url("ws://example.com").unwrap(); - assert_eq!(addr, "/dns4/example.com/tcp/80/ws".parse().unwrap()); + assert_eq!(addr, "/dns/example.com/tcp/80/ws".parse().unwrap()); } #[test] fn dns_addr_http() { let addr = from_url("http://example.com").unwrap(); - assert_eq!(addr, "/dns4/example.com/tcp/80/http".parse().unwrap()); + assert_eq!(addr, "/dns/example.com/tcp/80/http".parse().unwrap()); } #[test] fn dns_addr_wss() { let addr = from_url("wss://example.com").unwrap(); - assert_eq!(addr, "/dns4/example.com/tcp/443/wss".parse().unwrap()); + assert_eq!(addr, "/dns/example.com/tcp/443/wss".parse().unwrap()); } #[test] fn dns_addr_https() { let addr = from_url("https://example.com").unwrap(); - assert_eq!(addr, "/dns4/example.com/tcp/443/https".parse().unwrap()); + assert_eq!(addr, "/dns/example.com/tcp/443/https".parse().unwrap()); } #[test] fn bad_hostname() { let addr = from_url("wss://127.0.0.1x").unwrap(); - assert_eq!(addr, "/dns4/127.0.0.1x/tcp/443/wss".parse().unwrap()); + assert_eq!(addr, "/dns/127.0.0.1x/tcp/443/wss".parse().unwrap()); } #[test] @@ -223,7 +223,7 @@ mod tests { #[test] fn dns_and_port() { let addr = from_url("http://example.com:1000").unwrap(); - assert_eq!(addr, "/dns4/example.com/tcp/1000/http".parse().unwrap()); + assert_eq!(addr, "/dns/example.com/tcp/1000/http".parse().unwrap()); } #[test] diff --git a/misc/multiaddr/src/protocol.rs b/misc/multiaddr/src/protocol.rs index 6491d0d8..1ff0a8a6 100644 --- a/misc/multiaddr/src/protocol.rs +++ b/misc/multiaddr/src/protocol.rs @@ -17,6 +17,7 @@ use unsigned_varint::{encode, decode}; use crate::onion_addr::Onion3Addr; const DCCP: u32 = 33; +const DNS: u32 = 53; const DNS4: u32 = 54; const DNS6: u32 = 55; const DNSADDR: u32 = 56; @@ -66,6 +67,7 @@ const PATH_SEGMENT_ENCODE_SET: &percent_encoding::AsciiSet = &percent_encoding:: #[derive(PartialEq, Eq, Clone, Debug)] pub enum Protocol<'a> { Dccp(u16), + Dns(Cow<'a, str>), Dns4(Cow<'a, str>), Dns6(Cow<'a, str>), Dnsaddr(Cow<'a, str>), @@ -125,6 +127,10 @@ impl<'a> Protocol<'a> { let s = iter.next().ok_or(Error::InvalidProtocolString)?; Ok(Protocol::Ip6(Ipv6Addr::from_str(s)?)) } + "dns" => { + let s = iter.next().ok_or(Error::InvalidProtocolString)?; + Ok(Protocol::Dns(Cow::Borrowed(s))) + } "dns4" => { let s = iter.next().ok_or(Error::InvalidProtocolString)?; Ok(Protocol::Dns4(Cow::Borrowed(s))) @@ -206,6 +212,11 @@ impl<'a> Protocol<'a> { let num = rdr.read_u16::()?; Ok((Protocol::Dccp(num), rest)) } + DNS => { + let (n, input) = decode::usize(input)?; + let (data, rest) = split_at(n, input)?; + Ok((Protocol::Dns(Cow::Borrowed(str::from_utf8(data)?)), rest)) + } DNS4 => { let (n, input) = decode::usize(input)?; let (data, rest) = split_at(n, input)?; @@ -345,6 +356,12 @@ impl<'a> Protocol<'a> { w.write_all(encode::u32(SCTP, &mut buf))?; w.write_u16::(*port)? } + Protocol::Dns(s) => { + w.write_all(encode::u32(DNS, &mut buf))?; + let bytes = s.as_bytes(); + w.write_all(encode::usize(bytes.len(), &mut encode::usize_buffer()))?; + w.write_all(&bytes)? + } Protocol::Dns4(s) => { w.write_all(encode::u32(DNS4, &mut buf))?; let bytes = s.as_bytes(); @@ -421,6 +438,7 @@ impl<'a> Protocol<'a> { use self::Protocol::*; match self { Dccp(a) => Dccp(a), + Dns(cow) => Dns(Cow::Owned(cow.into_owned())), Dns4(cow) => Dns4(Cow::Owned(cow.into_owned())), Dns6(cow) => Dns6(Cow::Owned(cow.into_owned())), Dnsaddr(cow) => Dnsaddr(Cow::Owned(cow.into_owned())), @@ -454,6 +472,7 @@ impl<'a> fmt::Display for Protocol<'a> { use self::Protocol::*; match self { Dccp(port) => write!(f, "/dccp/{}", port), + Dns(s) => write!(f, "/dns/{}", s), Dns4(s) => write!(f, "/dns4/{}", s), Dns6(s) => write!(f, "/dns6/{}", s), Dnsaddr(s) => write!(f, "/dnsaddr/{}", s), diff --git a/misc/multiaddr/tests/lib.rs b/misc/multiaddr/tests/lib.rs index dafeb215..d0928022 100644 --- a/misc/multiaddr/tests/lib.rs +++ b/misc/multiaddr/tests/lib.rs @@ -76,36 +76,37 @@ struct Proto(Protocol<'static>); impl Arbitrary for Proto { fn arbitrary(g: &mut G) -> Self { use Protocol::*; - match g.gen_range(0, 24) { // TODO: Add Protocol::Quic + match g.gen_range(0, 25) { // TODO: Add Protocol::Quic 0 => Proto(Dccp(g.gen())), - 1 => Proto(Dns4(Cow::Owned(SubString::arbitrary(g).0))), - 2 => Proto(Dns6(Cow::Owned(SubString::arbitrary(g).0))), - 3 => Proto(Http), - 4 => Proto(Https), - 5 => Proto(Ip4(Ipv4Addr::arbitrary(g))), - 6 => Proto(Ip6(Ipv6Addr::arbitrary(g))), - 7 => Proto(P2pWebRtcDirect), - 8 => Proto(P2pWebRtcStar), - 9 => Proto(P2pWebSocketStar), - 10 => Proto(Memory(g.gen())), + 1 => Proto(Dns(Cow::Owned(SubString::arbitrary(g).0))), + 2 => Proto(Dns4(Cow::Owned(SubString::arbitrary(g).0))), + 3 => Proto(Dns6(Cow::Owned(SubString::arbitrary(g).0))), + 4 => Proto(Http), + 5 => Proto(Https), + 6 => Proto(Ip4(Ipv4Addr::arbitrary(g))), + 7 => Proto(Ip6(Ipv6Addr::arbitrary(g))), + 8 => Proto(P2pWebRtcDirect), + 9 => Proto(P2pWebRtcStar), + 10 => Proto(P2pWebSocketStar), + 11 => Proto(Memory(g.gen())), // TODO: impl Arbitrary for Multihash: - 11 => Proto(P2p(multihash("QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC"))), - 12 => Proto(P2pCircuit), - 13 => Proto(Quic), - 14 => Proto(Sctp(g.gen())), - 15 => Proto(Tcp(g.gen())), - 16 => Proto(Udp(g.gen())), - 17 => Proto(Udt), - 18 => Proto(Unix(Cow::Owned(SubString::arbitrary(g).0))), - 19 => Proto(Utp), - 20 => Proto(Ws("/".into())), - 21 => Proto(Wss("/".into())), - 22 => { + 12 => Proto(P2p(multihash("QmcgpsyWgH8Y8ajJz1Cu72KnS5uo2Aa2LpzU7kinSupNKC"))), + 13 => Proto(P2pCircuit), + 14 => Proto(Quic), + 15 => Proto(Sctp(g.gen())), + 16 => Proto(Tcp(g.gen())), + 17 => Proto(Udp(g.gen())), + 18 => Proto(Udt), + 19 => Proto(Unix(Cow::Owned(SubString::arbitrary(g).0))), + 20 => Proto(Utp), + 21 => Proto(Ws("/".into())), + 22 => Proto(Wss("/".into())), + 23 => { let mut a = [0; 10]; g.fill(&mut a); Proto(Onion(Cow::Owned(a), g.gen_range(1, std::u16::MAX))) }, - 23 => { + 24 => { let mut a = [0; 35]; g.fill_bytes(&mut a); Proto(Onion3((a, g.gen_range(1, std::u16::MAX)).into())) diff --git a/transports/dns/src/lib.rs b/transports/dns/src/lib.rs index 7c0a372c..82e54151 100644 --- a/transports/dns/src/lib.rs +++ b/transports/dns/src/lib.rs @@ -124,6 +124,7 @@ where // As an optimization, we immediately pass through if no component of the address contain // a DNS protocol. let contains_dns = addr.iter().any(|cmp| match cmp { + Protocol::Dns(_) => true, Protocol::Dns4(_) => true, Protocol::Dns6(_) => true, _ => false, @@ -139,7 +140,7 @@ where trace!("Dialing address with DNS: {}", addr); let resolve_futs = addr.iter() .map(|cmp| match cmp { - Protocol::Dns4(ref name) | Protocol::Dns6(ref name) => { + Protocol::Dns(ref name) | Protocol::Dns4(ref name) | Protocol::Dns6(ref name) => { let name = name.to_string(); let to_resolve = format!("{}:0", name); let (tx, rx) = oneshot::channel(); @@ -151,7 +152,12 @@ where }); }); - let is_dns4 = if let Protocol::Dns4(_) = cmp { true } else { false }; + let (dns4, dns6) = match cmp { + Protocol::Dns(_) => (true, true), + Protocol::Dns4(_) => (true, false), + Protocol::Dns6(_) => (false, true), + _ => unreachable!(), + }; async move { let list = rx.await @@ -166,7 +172,7 @@ where list.into_iter() .filter_map(|addr| { - if (is_dns4 && addr.is_ipv4()) || (!is_dns4 && addr.is_ipv6()) { + if (dns4 && addr.is_ipv4()) || (dns6 && addr.is_ipv6()) { Some(Protocol::from(addr)) } else { None diff --git a/transports/websocket/src/framed.rs b/transports/websocket/src/framed.rs index 191636a0..182b81ab 100644 --- a/transports/websocket/src/framed.rs +++ b/transports/websocket/src/framed.rs @@ -353,6 +353,8 @@ fn host_and_dnsname(addr: &Multiaddr) -> Result<(String, Option Ok((format!("{}:{}", ip, port), None)), + (Some(Protocol::Dns(h)), Some(Protocol::Tcp(port))) => + Ok((format!("{}:{}", &h, port), Some(tls::dns_name_ref(&h)?.to_owned()))), (Some(Protocol::Dns4(h)), Some(Protocol::Tcp(port))) => Ok((format!("{}:{}", &h, port), Some(tls::dns_name_ref(&h)?.to_owned()))), (Some(Protocol::Dns6(h)), Some(Protocol::Tcp(port))) => @@ -371,7 +373,7 @@ fn location_to_multiaddr(location: &str) -> Result> { let mut a = Multiaddr::empty(); match url.host() { Some(url::Host::Domain(h)) => { - a.push(Protocol::Dns4(h.into())) + a.push(Protocol::Dns(h.into())) } Some(url::Host::Ipv4(ip)) => { a.push(Protocol::Ip4(ip)) From bbc67351d375426ba6d387da714a017f1b2ea8e9 Mon Sep 17 00:00:00 2001 From: Thomas Eizinger Date: Mon, 18 May 2020 18:37:42 +1000 Subject: [PATCH 15/25] Extend feature-flags to allow choosing runtime for libp2p-tcp (#1471) * Extend feature-flags to allow choosing runtime for libp2p-tcp * Added CHANGELOG entry Co-authored-by: Pierre Krieger --- CHANGELOG.md | 5 +++++ Cargo.toml | 5 +++-- src/lib.rs | 18 +++++++++--------- transports/tcp/Cargo.toml | 3 --- 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 551ab1c3..410a2106 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,11 @@ an event. [PR 1567](https://github.com/libp2p/rust-libp2p/pull/1567) +- `libp2p-tcp`, `libp2p`: Made the `libp2p-tcp/async-std` feature flag + disabled by default, and split the `libp2p/tcp` feature in two: + `tcp-async-std` and `tcp-tokio`. `tcp-async-std` is still enabled by default. + [PR 1471](https://github.com/libp2p/rust-libp2p/pull/1471) + - `libp2p-tcp`: On listeners started with an IPv6 multi-address the socket option `IPV6_V6ONLY` is set to true. Instead of relying on IPv4-mapped IPv6 address support, two listeners can be started if IPv4 and IPv6 should both diff --git a/Cargo.toml b/Cargo.toml index fe5b101c..555cd5fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,7 +25,7 @@ default = [ "pnet", "secio", "secp256k1", - "tcp", + "tcp-async-std", "uds", "wasm-ext", "websocket", @@ -44,7 +44,8 @@ ping = ["libp2p-ping"] plaintext = ["libp2p-plaintext"] pnet = ["libp2p-pnet"] secio = ["libp2p-secio"] -tcp = ["libp2p-tcp"] +tcp-async-std = ["libp2p-tcp", "libp2p-tcp/async-std"] +tcp-tokio = ["libp2p-tcp", "libp2p-tcp/tokio"] uds = ["libp2p-uds"] wasm-ext = ["libp2p-wasm-ext"] websocket = ["libp2p-websocket"] diff --git a/src/lib.rs b/src/lib.rs index ded22023..7f24d409 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -85,7 +85,7 @@ //! Example ([`secio`] + [`yamux`] Protocol Upgrade): //! //! ```rust -//! # #[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "secio", feature = "yamux"))] { +//! # #[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "secio", feature = "yamux"))] { //! use libp2p::{Transport, core::upgrade, tcp::TcpConfig, secio::SecioConfig, identity::Keypair, yamux}; //! let tcp = TcpConfig::new(); //! let secio = SecioConfig::new(Keypair::generate_ed25519()); @@ -217,8 +217,8 @@ pub use libp2p_plaintext as plaintext; pub use libp2p_secio as secio; #[doc(inline)] pub use libp2p_swarm as swarm; -#[cfg(feature = "tcp")] -#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] +#[cfg(any(feature = "tcp-async-std", feature = "tcp-tokio-std"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp-async-std", feature = "tcp-tokio-std"))))] #[cfg(not(any(target_os = "emscripten", target_os = "unknown")))] #[doc(inline)] pub use libp2p_tcp as tcp; @@ -266,8 +266,8 @@ pub use self::transport_ext::TransportExt; /// /// > **Note**: This `Transport` is not suitable for production usage, as its implementation /// > reserves the right to support additional protocols or remove deprecated protocols. -#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))] -#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))))] +#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))] +#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))))] pub fn build_development_transport(keypair: identity::Keypair) -> std::io::Result> + Send + Sync), Error = impl std::error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> { @@ -280,8 +280,8 @@ pub fn build_development_transport(keypair: identity::Keypair) /// and mplex or yamux as the multiplexing layer. /// /// > **Note**: If you ever need to express the type of this `Transport`. -#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))] -#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))))] +#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))] +#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux"))))] pub fn build_tcp_ws_secio_mplex_yamux(keypair: identity::Keypair) -> std::io::Result> + Send + Sync), Error = impl std::error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> { @@ -306,8 +306,8 @@ pub fn build_tcp_ws_secio_mplex_yamux(keypair: identity::Keypair) /// and mplex or yamux as the multiplexing layer. /// /// > **Note**: If you ever need to express the type of this `Transport`. -#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux", feature = "pnet"))] -#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux", feature = "pnet"))))] +#[cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux", feature = "pnet"))] +#[cfg_attr(docsrs, doc(cfg(all(not(any(target_os = "emscripten", target_os = "unknown")), feature = "tcp-async-std", feature = "websocket", feature = "secio", feature = "mplex", feature = "yamux", feature = "pnet"))))] pub fn build_tcp_ws_pnet_secio_mplex_yamux(keypair: identity::Keypair, psk: PreSharedKey) -> std::io::Result> + Send + Sync), Error = impl std::error::Error + Send, Listener = impl Send, Dial = impl Send, ListenerUpgrade = impl Send> + Clone> { diff --git a/transports/tcp/Cargo.toml b/transports/tcp/Cargo.toml index 1133941c..b3c81f71 100644 --- a/transports/tcp/Cargo.toml +++ b/transports/tcp/Cargo.toml @@ -19,6 +19,3 @@ libp2p-core = { version = "0.18.0", path = "../../core" } log = "0.4.1" socket2 = "0.3.12" tokio = { version = "0.2", default-features = false, features = ["tcp"], optional = true } - -[features] -default = ["async-std"] From 2fbd1a55031bc18e3bcb7816b83ca9ea462b7ee1 Mon Sep 17 00:00:00 2001 From: Kevin Zhang Date: Mon, 18 May 2020 02:14:31 -0700 Subject: [PATCH 16/25] Keep consistency with Upgrade Inbound by using PING_SIZE (#1578) although 32 is prefect fine in our case, it would be consistent to use the const value PING_SIZE. Co-authored-by: Pierre Krieger --- protocols/ping/src/protocol.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/protocols/ping/src/protocol.rs b/protocols/ping/src/protocol.rs index 9bd5682f..fab16b36 100644 --- a/protocols/ping/src/protocol.rs +++ b/protocols/ping/src/protocol.rs @@ -83,7 +83,7 @@ where type Future = BoxFuture<'static, Result>; fn upgrade_outbound(self, mut socket: TSocket, _: Self::Info) -> Self::Future { - let payload: [u8; 32] = thread_rng().sample(distributions::Standard); + let payload: [u8; PING_SIZE] = thread_rng().sample(distributions::Standard); debug!("Preparing ping payload {:?}", payload); async move { socket.write_all(&payload).await?; From 162a0be0c2dd08be76f8fbf5a58d12ac45267339 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 18 May 2020 12:42:51 +0200 Subject: [PATCH 17/25] Display the identity hash of PeerIds when relevant (#1576) * Display the identity hash of PeerIds when relevant * Add CHANGELOG entry --- CHANGELOG.md | 5 +++++ core/src/peer_id.rs | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 410a2106..4c0332a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,11 @@ attempts per peer, with a configurable limit. [PR 1506](https://github.com/libp2p/rust-libp2p/pull/1506) +- `libp2p-core`: `PeerId`s that use the identity hashing will now be properly + displayed using the string representation of an identity multihash, rather + than the canonical SHA 256 representation. + [PR 1576](https://github.com/libp2p/rust-libp2p/pull/1576) + - `libp2p-core`: Updated to multihash 0.11.0. [PR 1566](https://github.com/libp2p/rust-libp2p/pull/1566) diff --git a/core/src/peer_id.rs b/core/src/peer_id.rs index 34b3ff6a..e5165857 100644 --- a/core/src/peer_id.rs +++ b/core/src/peer_id.rs @@ -147,7 +147,7 @@ impl PeerId { /// Returns a base-58 encoded string of this `PeerId`. pub fn to_base58(&self) -> String { - bs58::encode(self.borrow() as &[u8]).into_string() + bs58::encode(self.as_bytes()).into_string() } /// Checks whether the public key passed as parameter matches the public key of this `PeerId`. From 34faf94538e3464bb3e06995249c4d2a36bca759 Mon Sep 17 00:00:00 2001 From: Pierre Krieger Date: Mon, 18 May 2020 15:45:21 +0200 Subject: [PATCH 18/25] Publish 0.19.0 (#1579) --- CHANGELOG.md | 3 ++ Cargo.toml | 46 +++++++++++++++---------------- core/Cargo.toml | 10 +++---- misc/core-derive/Cargo.toml | 4 +-- misc/multiaddr/Cargo.toml | 2 +- misc/peer-id-generator/Cargo.toml | 2 +- muxers/mplex/Cargo.toml | 6 ++-- muxers/yamux/Cargo.toml | 4 +-- protocols/deflate/Cargo.toml | 6 ++-- protocols/floodsub/Cargo.toml | 6 ++-- protocols/gossipsub/Cargo.toml | 10 +++---- protocols/identify/Cargo.toml | 12 ++++---- protocols/kad/Cargo.toml | 10 +++---- protocols/mdns/Cargo.toml | 6 ++-- protocols/noise/Cargo.toml | 6 ++-- protocols/ping/Cargo.toml | 12 ++++---- protocols/plaintext/Cargo.toml | 4 +-- protocols/pnet/Cargo.toml | 2 +- protocols/secio/Cargo.toml | 8 +++--- swarm/Cargo.toml | 6 ++-- transports/dns/Cargo.toml | 4 +-- transports/tcp/Cargo.toml | 4 +-- transports/uds/Cargo.toml | 4 +-- transports/wasm-ext/Cargo.toml | 4 +-- transports/websocket/Cargo.toml | 6 ++-- 25 files changed, 95 insertions(+), 92 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c0332a6..f2e73fe1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Version ??? + +# Version 0.19.0 (2020-05-18) + - `libp2p-core`, `libp2p-swarm`: Added support for multiple dialing attempts per peer, with a configurable limit. [PR 1506](https://github.com/libp2p/rust-libp2p/pull/1506) diff --git a/Cargo.toml b/Cargo.toml index 555cd5fc..6e9a42e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p" edition = "2018" description = "Peer-to-peer networking library" -version = "0.18.1" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -55,36 +55,36 @@ secp256k1 = ["libp2p-core/secp256k1", "libp2p-secio/secp256k1"] [dependencies] bytes = "0.5" futures = "0.3.1" -multiaddr = { package = "parity-multiaddr", version = "0.8.0", path = "misc/multiaddr" } +multiaddr = { package = "parity-multiaddr", version = "0.9.0", path = "misc/multiaddr" } multihash = "0.11.0" lazy_static = "1.2" -libp2p-mplex = { version = "0.18.0", path = "muxers/mplex", optional = true } -libp2p-identify = { version = "0.18.0", path = "protocols/identify", optional = true } -libp2p-kad = { version = "0.18.0", path = "protocols/kad", optional = true } -libp2p-floodsub = { version = "0.18.0", path = "protocols/floodsub", optional = true } -libp2p-gossipsub = { version = "0.18.0", path = "./protocols/gossipsub", optional = true } -libp2p-ping = { version = "0.18.0", path = "protocols/ping", optional = true } -libp2p-plaintext = { version = "0.18.0", path = "protocols/plaintext", optional = true } -libp2p-pnet = { version = "0.18.0", path = "protocols/pnet", optional = true } -libp2p-core = { version = "0.18.0", path = "core" } -libp2p-core-derive = { version = "0.18.0", path = "misc/core-derive" } -libp2p-secio = { version = "0.18.0", path = "protocols/secio", default-features = false, optional = true } -libp2p-swarm = { version = "0.18.1", path = "swarm" } -libp2p-uds = { version = "0.18.0", path = "transports/uds", optional = true } -libp2p-wasm-ext = { version = "0.18.0", path = "transports/wasm-ext", optional = true } -libp2p-yamux = { version = "0.18.0", path = "muxers/yamux", optional = true } -libp2p-noise = { version = "0.18.0", path = "protocols/noise", optional = true } +libp2p-mplex = { version = "0.19.0", path = "muxers/mplex", optional = true } +libp2p-identify = { version = "0.19.0", path = "protocols/identify", optional = true } +libp2p-kad = { version = "0.19.0", path = "protocols/kad", optional = true } +libp2p-floodsub = { version = "0.19.0", path = "protocols/floodsub", optional = true } +libp2p-gossipsub = { version = "0.19.0", path = "./protocols/gossipsub", optional = true } +libp2p-ping = { version = "0.19.0", path = "protocols/ping", optional = true } +libp2p-plaintext = { version = "0.19.0", path = "protocols/plaintext", optional = true } +libp2p-pnet = { version = "0.19.0", path = "protocols/pnet", optional = true } +libp2p-core = { version = "0.19.0", path = "core" } +libp2p-core-derive = { version = "0.19.0", path = "misc/core-derive" } +libp2p-secio = { version = "0.19.0", path = "protocols/secio", default-features = false, optional = true } +libp2p-swarm = { version = "0.19.0", path = "swarm" } +libp2p-uds = { version = "0.19.0", path = "transports/uds", optional = true } +libp2p-wasm-ext = { version = "0.19.0", path = "transports/wasm-ext", optional = true } +libp2p-yamux = { version = "0.19.0", path = "muxers/yamux", optional = true } +libp2p-noise = { version = "0.19.0", path = "protocols/noise", optional = true } parking_lot = "0.10.0" pin-project = "0.4.6" smallvec = "1.0" wasm-timer = "0.2.4" [target.'cfg(not(any(target_os = "emscripten", target_os = "unknown")))'.dependencies] -libp2p-deflate = { version = "0.18.0", path = "protocols/deflate", optional = true } -libp2p-dns = { version = "0.18.0", path = "transports/dns", optional = true } -libp2p-mdns = { version = "0.18.0", path = "protocols/mdns", optional = true } -libp2p-tcp = { version = "0.18.0", path = "transports/tcp", optional = true } -libp2p-websocket = { version = "0.18.0", path = "transports/websocket", optional = true } +libp2p-deflate = { version = "0.19.0", path = "protocols/deflate", optional = true } +libp2p-dns = { version = "0.19.0", path = "transports/dns", optional = true } +libp2p-mdns = { version = "0.19.0", path = "protocols/mdns", optional = true } +libp2p-tcp = { version = "0.19.0", path = "transports/tcp", optional = true } +libp2p-websocket = { version = "0.19.0", path = "transports/websocket", optional = true } [dev-dependencies] async-std = "1.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index 5c64bc99..c2f8166b 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-core" edition = "2018" description = "Core traits and structs of libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -20,7 +20,7 @@ futures-timer = "3" lazy_static = "1.2" libsecp256k1 = { version = "0.3.1", optional = true } log = "0.4" -multiaddr = { package = "parity-multiaddr", version = "0.8.0", path = "../misc/multiaddr" } +multiaddr = { package = "parity-multiaddr", version = "0.9.0", path = "../misc/multiaddr" } multihash = "0.11.0" multistream-select = { version = "0.8.0", path = "../misc/multistream-select" } parking_lot = "0.10.0" @@ -40,9 +40,9 @@ ring = { version = "0.16.9", features = ["alloc", "std"], default-features = fal [dev-dependencies] async-std = "1.0" -libp2p-mplex = { version = "0.18.0", path = "../muxers/mplex" } -libp2p-secio = { version = "0.18.0", path = "../protocols/secio" } -libp2p-tcp = { version = "0.18.0", path = "../transports/tcp" } +libp2p-mplex = { version = "0.19.0", path = "../muxers/mplex" } +libp2p-secio = { version = "0.19.0", path = "../protocols/secio" } +libp2p-tcp = { version = "0.19.0", path = "../transports/tcp" } quickcheck = "0.9.0" wasm-timer = "0.2" diff --git a/misc/core-derive/Cargo.toml b/misc/core-derive/Cargo.toml index 166156cd..b5521838 100644 --- a/misc/core-derive/Cargo.toml +++ b/misc/core-derive/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-core-derive" edition = "2018" description = "Procedural macros of libp2p-core" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -17,4 +17,4 @@ syn = { version = "1.0.8", default-features = false, features = ["clone-impls", quote = "1.0" [dev-dependencies] -libp2p = { version = "0.18.0", path = "../.." } +libp2p = { version = "0.19.0", path = "../.." } diff --git a/misc/multiaddr/Cargo.toml b/misc/multiaddr/Cargo.toml index 56bb11e7..38f920e6 100644 --- a/misc/multiaddr/Cargo.toml +++ b/misc/multiaddr/Cargo.toml @@ -6,7 +6,7 @@ description = "Implementation of the multiaddr format" homepage = "https://github.com/libp2p/rust-libp2p" keywords = ["multiaddr", "ipfs"] license = "MIT" -version = "0.8.0" +version = "0.9.0" [dependencies] arrayref = "0.3" diff --git a/misc/peer-id-generator/Cargo.toml b/misc/peer-id-generator/Cargo.toml index f3efec74..1813bcbd 100644 --- a/misc/peer-id-generator/Cargo.toml +++ b/misc/peer-id-generator/Cargo.toml @@ -11,5 +11,5 @@ categories = ["network-programming", "asynchronous"] publish = false [dependencies] -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } num_cpus = "1.8" diff --git a/muxers/mplex/Cargo.toml b/muxers/mplex/Cargo.toml index 4be96b7e..23398a67 100644 --- a/muxers/mplex/Cargo.toml +++ b/muxers/mplex/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-mplex" edition = "2018" description = "Mplex multiplexing protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -14,11 +14,11 @@ bytes = "0.5" fnv = "1.0" futures = "0.3.1" futures_codec = "0.3.4" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4" parking_lot = "0.10" unsigned-varint = { version = "0.3", features = ["futures-codec"] } [dev-dependencies] async-std = "1.0" -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } diff --git a/muxers/yamux/Cargo.toml b/muxers/yamux/Cargo.toml index 4255bea4..2901556a 100644 --- a/muxers/yamux/Cargo.toml +++ b/muxers/yamux/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-yamux" edition = "2018" description = "Yamux multiplexing protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,7 +11,7 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } parking_lot = "0.10" thiserror = "1.0" yamux = "0.4.5" diff --git a/protocols/deflate/Cargo.toml b/protocols/deflate/Cargo.toml index 0d2552f5..ab7f0018 100644 --- a/protocols/deflate/Cargo.toml +++ b/protocols/deflate/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-deflate" edition = "2018" description = "Deflate encryption protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,11 +11,11 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } flate2 = "1.0" [dev-dependencies] async-std = "1.0" -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } rand = "0.7" quickcheck = "0.9" diff --git a/protocols/floodsub/Cargo.toml b/protocols/floodsub/Cargo.toml index 4c8fa279..45701703 100644 --- a/protocols/floodsub/Cargo.toml +++ b/protocols/floodsub/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-floodsub" edition = "2018" description = "Floodsub protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -13,8 +13,8 @@ categories = ["network-programming", "asynchronous"] cuckoofilter = "0.3.2" fnv = "1.0" futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } prost = "0.6.1" rand = "0.7" smallvec = "1.0" diff --git a/protocols/gossipsub/Cargo.toml b/protocols/gossipsub/Cargo.toml index 8f1b297d..8d91104d 100644 --- a/protocols/gossipsub/Cargo.toml +++ b/protocols/gossipsub/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-gossipsub" edition = "2018" description = "Gossipsub protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Age Manning "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -10,8 +10,8 @@ keywords = ["peer-to-peer", "libp2p", "networking"] categories = ["network-programming", "asynchronous"] [dependencies] -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } bytes = "0.5.4" byteorder = "1.3.2" fnv = "1.0.6" @@ -30,8 +30,8 @@ prost = "0.6.1" [dev-dependencies] async-std = "1.4.0" env_logger = "0.7.1" -libp2p-plaintext = { version = "0.18.0", path = "../plaintext" } -libp2p-yamux = { version = "0.18.0", path = "../../muxers/yamux" } +libp2p-plaintext = { version = "0.19.0", path = "../plaintext" } +libp2p-yamux = { version = "0.19.0", path = "../../muxers/yamux" } quickcheck = "0.9.2" [build-dependencies] diff --git a/protocols/identify/Cargo.toml b/protocols/identify/Cargo.toml index 31fb5e9a..77f3dec9 100644 --- a/protocols/identify/Cargo.toml +++ b/protocols/identify/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-identify" edition = "2018" description = "Nodes identifcation protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,8 +11,8 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } log = "0.4.1" prost = "0.6.1" smallvec = "1.0" @@ -20,9 +20,9 @@ wasm-timer = "0.2" [dev-dependencies] async-std = "1.0" -libp2p-mplex = { version = "0.18.0", path = "../../muxers/mplex" } -libp2p-secio = { version = "0.18.0", path = "../../protocols/secio" } -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } +libp2p-mplex = { version = "0.19.0", path = "../../muxers/mplex" } +libp2p-secio = { version = "0.19.0", path = "../../protocols/secio" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } [build-dependencies] prost-build = "0.6" diff --git a/protocols/kad/Cargo.toml b/protocols/kad/Cargo.toml index b9734d18..02c5cf67 100644 --- a/protocols/kad/Cargo.toml +++ b/protocols/kad/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-kad" edition = "2018" description = "Kademlia protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -17,8 +17,8 @@ fnv = "1.0" futures_codec = "0.3.4" futures = "0.3.1" log = "0.4" -libp2p-core = { version = "0.18.0", path = "../../core" } -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } multihash = "0.11.0" prost = "0.6.1" rand = "0.7.2" @@ -30,8 +30,8 @@ unsigned-varint = { version = "0.3", features = ["futures-codec"] } void = "1.0" [dev-dependencies] -libp2p-secio = { version = "0.18.0", path = "../secio" } -libp2p-yamux = { version = "0.18.0", path = "../../muxers/yamux" } +libp2p-secio = { version = "0.19.0", path = "../secio" } +libp2p-yamux = { version = "0.19.0", path = "../../muxers/yamux" } quickcheck = "0.9.0" [build-dependencies] diff --git a/protocols/mdns/Cargo.toml b/protocols/mdns/Cargo.toml index a0684422..f61572d1 100644 --- a/protocols/mdns/Cargo.toml +++ b/protocols/mdns/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "libp2p-mdns" edition = "2018" -version = "0.18.0" +version = "0.19.0" description = "Implementation of the libp2p mDNS discovery method" authors = ["Parity Technologies "] license = "MIT" @@ -16,8 +16,8 @@ dns-parser = "0.8" either = "1.5.3" futures = "0.3.1" lazy_static = "1.2" -libp2p-core = { version = "0.18.0", path = "../../core" } -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } log = "0.4" net2 = "0.2" rand = "0.7" diff --git a/protocols/noise/Cargo.toml b/protocols/noise/Cargo.toml index 50392188..1f812f1e 100644 --- a/protocols/noise/Cargo.toml +++ b/protocols/noise/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "libp2p-noise" description = "Cryptographic handshake protocol using the noise framework." -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,7 +11,7 @@ edition = "2018" curve25519-dalek = "2.0.0" futures = "0.3.1" lazy_static = "1.2" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4" prost = "0.6.1" rand = "0.7.2" @@ -28,7 +28,7 @@ snow = { version = "0.6.1", features = ["default-resolver"], default-features = [dev-dependencies] env_logger = "0.7.1" -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } quickcheck = "0.9.0" sodiumoxide = "^0.2.5" diff --git a/protocols/ping/Cargo.toml b/protocols/ping/Cargo.toml index b9323e81..0b6a9960 100644 --- a/protocols/ping/Cargo.toml +++ b/protocols/ping/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-ping" edition = "2018" description = "Ping protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,8 +11,8 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } -libp2p-swarm = { version = "0.18.0", path = "../../swarm" } +libp2p-core = { version = "0.19.0", path = "../../core" } +libp2p-swarm = { version = "0.19.0", path = "../../swarm" } log = "0.4.1" rand = "0.7.2" void = "1.0" @@ -20,7 +20,7 @@ wasm-timer = "0.2" [dev-dependencies] async-std = "1.0" -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } -libp2p-secio = { version = "0.18.0", path = "../../protocols/secio" } -libp2p-yamux = { version = "0.18.0", path = "../../muxers/yamux" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } +libp2p-secio = { version = "0.19.0", path = "../../protocols/secio" } +libp2p-yamux = { version = "0.19.0", path = "../../muxers/yamux" } quickcheck = "0.9.0" diff --git a/protocols/plaintext/Cargo.toml b/protocols/plaintext/Cargo.toml index 07321552..e00b5ac4 100644 --- a/protocols/plaintext/Cargo.toml +++ b/protocols/plaintext/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-plaintext" edition = "2018" description = "Plaintext encryption dummy protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -13,7 +13,7 @@ categories = ["network-programming", "asynchronous"] bytes = "0.5" futures = "0.3.1" futures_codec = "0.3.4" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.8" prost = "0.6.1" rw-stream-sink = "0.2.0" diff --git a/protocols/pnet/Cargo.toml b/protocols/pnet/Cargo.toml index 6897cfb4..de5ea5cf 100644 --- a/protocols/pnet/Cargo.toml +++ b/protocols/pnet/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-pnet" edition = "2018" description = "Private swarm support for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" diff --git a/protocols/secio/Cargo.toml b/protocols/secio/Cargo.toml index a5be9059..a24f4990 100644 --- a/protocols/secio/Cargo.toml +++ b/protocols/secio/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-secio" edition = "2018" description = "Secio encryption protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -16,7 +16,7 @@ ctr = "0.3" futures = "0.3.1" hmac = "0.7.0" lazy_static = "1.2.0" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.6" prost = "0.6.1" pin-project = "0.4.6" @@ -48,8 +48,8 @@ aes-all = ["aesni"] [dev-dependencies] async-std = "1.0" criterion = "0.3" -libp2p-mplex = { version = "0.18.0", path = "../../muxers/mplex" } -libp2p-tcp = { version = "0.18.0", path = "../../transports/tcp" } +libp2p-mplex = { version = "0.19.0", path = "../../muxers/mplex" } +libp2p-tcp = { version = "0.19.0", path = "../../transports/tcp" } [[bench]] name = "bench" diff --git a/swarm/Cargo.toml b/swarm/Cargo.toml index 7feaa30a..46785665 100644 --- a/swarm/Cargo.toml +++ b/swarm/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-swarm" edition = "2018" description = "The libp2p swarm" -version = "0.18.1" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,7 +11,7 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../core" } +libp2p-core = { version = "0.19.0", path = "../core" } log = "0.4" rand = "0.7" smallvec = "1.0" @@ -19,6 +19,6 @@ wasm-timer = "0.2" void = "1" [dev-dependencies] -libp2p-mplex = { version = "0.18.0", path = "../muxers/mplex" } +libp2p-mplex = { version = "0.19.0", path = "../muxers/mplex" } quickcheck = "0.9.0" rand = "0.7.2" diff --git a/transports/dns/Cargo.toml b/transports/dns/Cargo.toml index 11d1958f..acd122f1 100644 --- a/transports/dns/Cargo.toml +++ b/transports/dns/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-dns" edition = "2018" description = "DNS transport implementation for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -10,6 +10,6 @@ keywords = ["peer-to-peer", "libp2p", "networking"] categories = ["network-programming", "asynchronous"] [dependencies] -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.1" futures = "0.3.1" diff --git a/transports/tcp/Cargo.toml b/transports/tcp/Cargo.toml index b3c81f71..8b6a5af4 100644 --- a/transports/tcp/Cargo.toml +++ b/transports/tcp/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-tcp" edition = "2018" description = "TCP/IP transport protocol for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -15,7 +15,7 @@ futures = "0.3.1" futures-timer = "3.0" get_if_addrs = "0.5.3" ipnet = "2.0.0" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.1" socket2 = "0.3.12" tokio = { version = "0.2", default-features = false, features = ["tcp"], optional = true } diff --git a/transports/uds/Cargo.toml b/transports/uds/Cargo.toml index ee83f852..0fcfa6ff 100644 --- a/transports/uds/Cargo.toml +++ b/transports/uds/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-uds" edition = "2018" description = "Unix domain sockets transport for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -11,7 +11,7 @@ categories = ["network-programming", "asynchronous"] [target.'cfg(all(unix, not(any(target_os = "emscripten", target_os = "unknown"))))'.dependencies] async-std = { version = "1.0", optional = true } -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.1" futures = "0.3.1" tokio = { version = "0.2", default-features = false, features = ["uds"], optional = true } diff --git a/transports/wasm-ext/Cargo.toml b/transports/wasm-ext/Cargo.toml index 4661e8b8..c04d00b7 100644 --- a/transports/wasm-ext/Cargo.toml +++ b/transports/wasm-ext/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "libp2p-wasm-ext" -version = "0.18.0" +version = "0.19.0" authors = ["Pierre Krieger "] edition = "2018" description = "Allows passing in an external transport in a WASM environment" @@ -12,7 +12,7 @@ categories = ["network-programming", "asynchronous"] [dependencies] futures = "0.3.1" js-sys = "0.3.19" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } parity-send-wrapper = "0.1.0" wasm-bindgen = "0.2.42" wasm-bindgen-futures = "0.4.4" diff --git a/transports/websocket/Cargo.toml b/transports/websocket/Cargo.toml index 33d0419b..7dfe6c75 100644 --- a/transports/websocket/Cargo.toml +++ b/transports/websocket/Cargo.toml @@ -2,7 +2,7 @@ name = "libp2p-websocket" edition = "2018" description = "WebSocket transport for libp2p" -version = "0.18.0" +version = "0.19.0" authors = ["Parity Technologies "] license = "MIT" repository = "https://github.com/libp2p/rust-libp2p" @@ -14,7 +14,7 @@ async-tls = "0.7.0" bytes = "0.5" either = "1.5.3" futures = "0.3.1" -libp2p-core = { version = "0.18.0", path = "../../core" } +libp2p-core = { version = "0.19.0", path = "../../core" } log = "0.4.8" quicksink = "0.1" rustls = "0.17.0" @@ -25,4 +25,4 @@ webpki = "0.21" webpki-roots = "0.18" [dev-dependencies] -libp2p-tcp = { version = "0.18.0", path = "../tcp" } +libp2p-tcp = { version = "0.19.0", path = "../tcp" } From e6915be539f792de46e2aaf4af1dffa54554dea9 Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Tue, 19 May 2020 19:23:44 +0300 Subject: [PATCH 19/25] 0.19 WIP --- protocols/kad/src/behaviour.rs | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/protocols/kad/src/behaviour.rs b/protocols/kad/src/behaviour.rs index fc85b5d0..5fd36ded 100644 --- a/protocols/kad/src/behaviour.rs +++ b/protocols/kad/src/behaviour.rs @@ -572,12 +572,14 @@ where bs58::encode(target.preimage().as_ref()).into_string(), // peer id bs58::encode(target.as_ref()).into_string(), // sha256 ); + let provider_key = self.kbuckets.local_public_key(); let peers = self.kbuckets.closest_keys(&target); let context = AddProviderContext::Publish; let info = QueryInfo::AddProvider { context, key, - phase: AddProviderPhase::GetClosestPeers + phase: AddProviderPhase::GetClosestPeers, + provider_key }; let inner = QueryInner::new(info); let id = self.queries.add_iter_closest(target.clone(), peers, inner); @@ -693,10 +695,12 @@ where /// Starts an iterative `ADD_PROVIDER` query for the given key. fn start_add_provider(&mut self, key: record::Key, context: AddProviderContext) { + let provider_key = self.kbuckets.local_public_key(); let info = QueryInfo::AddProvider { context, key: key.clone(), - phase: AddProviderPhase::GetClosestPeers + phase: AddProviderPhase::GetClosestPeers, + provider_key }; let target = kbucket::Key::new(key); let peers = self.kbuckets.closest_keys(&target); @@ -908,7 +912,8 @@ where QueryInfo::AddProvider { context, key, - phase: AddProviderPhase::GetClosestPeers + phase: AddProviderPhase::GetClosestPeers, + .. } => { let provider_id = params.local_peer_id().clone(); let external_addresses = params.external_addresses().collect(); @@ -920,7 +925,7 @@ where provider_id, external_addresses, get_closest_peers_stats: result.stats - } + }, provider_key }); self.queries.continue_fixed(query_id, result.peers, inner); @@ -930,7 +935,8 @@ where QueryInfo::AddProvider { context, key, - phase: AddProviderPhase::AddProvider { get_closest_peers_stats, .. } + phase: AddProviderPhase::AddProvider { get_closest_peers_stats, .. }, + .. } => { log::info!("AddProvider finished {:?}!", context); match context { @@ -2263,16 +2269,16 @@ impl QueryInfo { key: key.clone(), user_data: query_id, }, - QueryInfo::AddProvider { key, phase, .. } => match phase { + QueryInfo::AddProvider { key, phase, provider_key, .. } => match phase { AddProviderPhase::GetClosestPeers => KademliaHandlerIn::FindNodeReq { key: key.to_vec(), user_data: query_id, }, AddProviderPhase::AddProvider { - provider_id, - external_addresses, - provider_key, - ..} => { + provider_id, + external_addresses, + .. + } => { KademliaHandlerIn::AddProvider { key: key.clone(), provider: crate::protocol::KadPeer { From 27c30a9f2d331eed2dde4710ceccd7e448a08288 Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Tue, 19 May 2020 19:36:25 +0300 Subject: [PATCH 20/25] arqada => fluence --- Cargo.toml | 4 +--- protocols/kad/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index d0eb8507..cc294af9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,13 +116,11 @@ members = [ "transports/wasm-ext" ] -#[patch.'https://github.com/fluencelabs/rust-libp2p'] [patch.'ssh://git@github.com/fluencelabs/rust-libp2p.git'] libp2p-core = { path = "core" } -# NOTE: this is required because trust-graph depends on libp2p-core = 0.17.0, +# NOTE: this is required because trust-graph depends on libp2p-core, # and patches it to git only in patch section, which apparently isn't # visible via dependency mechanics (i.e., not visible HERE) [patch.crates-io] libp2p-core = { path = "core" } -#trust-graph = { path = "../arqada/janus/trust-graph" } diff --git a/protocols/kad/Cargo.toml b/protocols/kad/Cargo.toml index 9f3ccc8d..493754e5 100644 --- a/protocols/kad/Cargo.toml +++ b/protocols/kad/Cargo.toml @@ -31,7 +31,7 @@ void = "1.0" bs58 = "0.3.0" derivative = "2.0.2" -trust-graph = { git = "ssh://git@github.com/fluencelabs/arqada.git", branch = "memory_store_set" } +trust-graph = { git = "ssh://git@github.com/fluencelabs/fluence.git", branch = "libp2p_0.19.0" } [dev-dependencies] libp2p-secio = { version = "0.19.0", path = "../secio" } From d78805364d8136650d4cd6d020ffc5315bcdd520 Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Tue, 19 May 2020 21:45:41 +0300 Subject: [PATCH 21/25] ssh => https --- Cargo.toml | 2 +- protocols/kad/Cargo.toml | 2 +- protocols/kad/src/behaviour.rs | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cc294af9..a998af69 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -116,7 +116,7 @@ members = [ "transports/wasm-ext" ] -[patch.'ssh://git@github.com/fluencelabs/rust-libp2p.git'] +[patch.'https://github.com/fluencelabs/rust-libp2p'] libp2p-core = { path = "core" } # NOTE: this is required because trust-graph depends on libp2p-core, diff --git a/protocols/kad/Cargo.toml b/protocols/kad/Cargo.toml index 493754e5..81cc50b6 100644 --- a/protocols/kad/Cargo.toml +++ b/protocols/kad/Cargo.toml @@ -31,7 +31,7 @@ void = "1.0" bs58 = "0.3.0" derivative = "2.0.2" -trust-graph = { git = "ssh://git@github.com/fluencelabs/fluence.git", branch = "libp2p_0.19.0" } +trust-graph = { git = "https://github.com/fluencelabs/fluence", branch = "libp2p_0.19.0" } [dev-dependencies] libp2p-secio = { version = "0.19.0", path = "../secio" } diff --git a/protocols/kad/src/behaviour.rs b/protocols/kad/src/behaviour.rs index 5fd36ded..1e432822 100644 --- a/protocols/kad/src/behaviour.rs +++ b/protocols/kad/src/behaviour.rs @@ -1,4 +1,5 @@ // Copyright 2018 Parity Technologies (UK) Ltd. +#![allow(clippy::needless_lifetimes)] // // Permission is hereby granted, free of charge, to any person obtaining a // copy of this software and associated documentation files (the "Software"), From f6a4f61fbb64c9cef7e9338e6b7fc46adac8abee Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Tue, 19 May 2020 21:48:26 +0300 Subject: [PATCH 22/25] disable ipfs-kad & distributed-key-value-store examples --- examples/distributed-key-value-store.rs | 410 ++++++++++++------------ examples/ipfs-kad.rs | 252 +++++++-------- 2 files changed, 331 insertions(+), 331 deletions(-) diff --git a/examples/distributed-key-value-store.rs b/examples/distributed-key-value-store.rs index 014418d3..39a4d046 100644 --- a/examples/distributed-key-value-store.rs +++ b/examples/distributed-key-value-store.rs @@ -1,208 +1,208 @@ -// Copyright 20l9 Parity Technologies (UK) Ltd. +// // Copyright 20l9 Parity Technologies (UK) Ltd. +// // +// // Permission is hereby granted, free of charge, to any person obtaining a +// // copy of this software and associated documentation files (the "Software"), +// // to deal in the Software without restriction, including without limitation +// // the rights to use, copy, modify, merge, publish, distribute, sublicense, +// // and/or sell copies of the Software, and to permit persons to whom the +// // Software is furnished to do so, subject to the following conditions: +// // +// // The above copyright notice and this permission notice shall be included in +// // all copies or substantial portions of the Software. +// // +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// // DEALINGS IN THE SOFTWARE. // -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: +// //! A basic key value store demonstrating libp2p and the mDNS and Kademlia protocols. +// //! +// //! 1. Using two terminal windows, start two instances. If you local network +// //! allows mDNS, they will automatically connect. +// //! +// //! 2. Type `PUT my-key my-value` in terminal one and hit return. +// //! +// //! 3. Type `GET my-key` in terminal two and hit return. +// //! +// //! 4. Close with Ctrl-c. // -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. +// use async_std::{io, task}; +// use futures::prelude::*; +// use libp2p::kad::record::store::MemoryStore; +// use libp2p::kad::{ +// record::Key, +// Kademlia, +// KademliaEvent, +// PutRecordOk, +// QueryResult, +// Quorum, +// Record +// }; +// use libp2p::{ +// NetworkBehaviour, +// PeerId, +// Swarm, +// build_development_transport, +// identity, +// mdns::{Mdns, MdnsEvent}, +// swarm::NetworkBehaviourEventProcess +// }; +// use std::{error::Error, task::{Context, Poll}}; // -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -//! A basic key value store demonstrating libp2p and the mDNS and Kademlia protocols. -//! -//! 1. Using two terminal windows, start two instances. If you local network -//! allows mDNS, they will automatically connect. -//! -//! 2. Type `PUT my-key my-value` in terminal one and hit return. -//! -//! 3. Type `GET my-key` in terminal two and hit return. -//! -//! 4. Close with Ctrl-c. - -use async_std::{io, task}; -use futures::prelude::*; -use libp2p::kad::record::store::MemoryStore; -use libp2p::kad::{ - record::Key, - Kademlia, - KademliaEvent, - PutRecordOk, - QueryResult, - Quorum, - Record -}; -use libp2p::{ - NetworkBehaviour, - PeerId, - Swarm, - build_development_transport, - identity, - mdns::{Mdns, MdnsEvent}, - swarm::NetworkBehaviourEventProcess -}; -use std::{error::Error, task::{Context, Poll}}; - -fn main() -> Result<(), Box> { - env_logger::init(); - - // Create a random key for ourselves. - let local_key = identity::Keypair::generate_ed25519(); - let local_peer_id = PeerId::from(local_key.public()); - - // Set up a an encrypted DNS-enabled TCP Transport over the Mplex protocol. - let transport = build_development_transport(local_key)?; - - // We create a custom network behaviour that combines Kademlia and mDNS. - #[derive(NetworkBehaviour)] - struct MyBehaviour { - kademlia: Kademlia, - mdns: Mdns - } - - impl NetworkBehaviourEventProcess for MyBehaviour { - // Called when `mdns` produces an event. - fn inject_event(&mut self, event: MdnsEvent) { - if let MdnsEvent::Discovered(list) = event { - for (peer_id, multiaddr) in list { - self.kademlia.add_address(&peer_id, multiaddr); - } - } - } - } - - impl NetworkBehaviourEventProcess for MyBehaviour { - // Called when `kademlia` produces an event. - fn inject_event(&mut self, message: KademliaEvent) { - match message { - KademliaEvent::QueryResult { result, .. } => match result { - QueryResult::GetRecord(Ok(ok)) => { - for Record { key, value, .. } in ok.records { - println!( - "Got record {:?} {:?}", - std::str::from_utf8(key.as_ref()).unwrap(), - std::str::from_utf8(&value).unwrap(), - ); - } - } - QueryResult::GetRecord(Err(err)) => { - eprintln!("Failed to get record: {:?}", err); - } - QueryResult::PutRecord(Ok(PutRecordOk { key })) => { - println!( - "Successfully put record {:?}", - std::str::from_utf8(key.as_ref()).unwrap() - ); - } - QueryResult::PutRecord(Err(err)) => { - eprintln!("Failed to put record: {:?}", err); - } - _ => {} - } - _ => {} - } - } - } - - // Create a swarm to manage peers and events. - let mut swarm = { - // Create a Kademlia behaviour. - let store = MemoryStore::new(local_peer_id.clone()); - let kademlia = Kademlia::new(local_peer_id.clone(), store); - let mdns = Mdns::new()?; - let behaviour = MyBehaviour { kademlia, mdns }; - Swarm::new(transport, behaviour, local_peer_id) - }; - - // Read full lines from stdin - let mut stdin = io::BufReader::new(io::stdin()).lines(); - - // Listen on all interfaces and whatever port the OS assigns. - Swarm::listen_on(&mut swarm, "/ip4/0.0.0.0/tcp/0".parse()?)?; - - // Kick it off. - let mut listening = false; - task::block_on(future::poll_fn(move |cx: &mut Context| { - loop { - match stdin.try_poll_next_unpin(cx)? { - Poll::Ready(Some(line)) => handle_input_line(&mut swarm.kademlia, line), - Poll::Ready(None) => panic!("Stdin closed"), - Poll::Pending => break - } - } - loop { - match swarm.poll_next_unpin(cx) { - Poll::Ready(Some(event)) => println!("{:?}", event), - Poll::Ready(None) => return Poll::Ready(Ok(())), - Poll::Pending => { - if !listening { - if let Some(a) = Swarm::listeners(&swarm).next() { - println!("Listening on {:?}", a); - listening = true; - } - } - break - } - } - } - Poll::Pending - })) -} - -fn handle_input_line(kademlia: &mut Kademlia, line: String) { - let mut args = line.split(" "); - - match args.next() { - Some("GET") => { - let key = { - match args.next() { - Some(key) => Key::new(&key), - None => { - eprintln!("Expected key"); - return; - } - } - }; - kademlia.get_record(&key, Quorum::One); - } - Some("PUT") => { - let key = { - match args.next() { - Some(key) => Key::new(&key), - None => { - eprintln!("Expected key"); - return; - } - } - }; - let value = { - match args.next() { - Some(value) => value.as_bytes().to_vec(), - None => { - eprintln!("Expected value"); - return; - } - } - }; - let record = Record { - key, - value, - publisher: None, - expires: None, - }; - kademlia.put_record(record, Quorum::One).expect("Failed to store record locally."); - } - _ => { - eprintln!("expected GET or PUT"); - } - } -} +// fn main() -> Result<(), Box> { +// env_logger::init(); +// +// // Create a random key for ourselves. +// let local_key = identity::Keypair::generate_ed25519(); +// let local_peer_id = PeerId::from(local_key.public()); +// +// // Set up a an encrypted DNS-enabled TCP Transport over the Mplex protocol. +// let transport = build_development_transport(local_key)?; +// +// // We create a custom network behaviour that combines Kademlia and mDNS. +// #[derive(NetworkBehaviour)] +// struct MyBehaviour { +// kademlia: Kademlia, +// mdns: Mdns +// } +// +// impl NetworkBehaviourEventProcess for MyBehaviour { +// // Called when `mdns` produces an event. +// fn inject_event(&mut self, event: MdnsEvent) { +// if let MdnsEvent::Discovered(list) = event { +// for (peer_id, multiaddr) in list { +// self.kademlia.add_address(&peer_id, multiaddr); +// } +// } +// } +// } +// +// impl NetworkBehaviourEventProcess for MyBehaviour { +// // Called when `kademlia` produces an event. +// fn inject_event(&mut self, message: KademliaEvent) { +// match message { +// KademliaEvent::QueryResult { result, .. } => match result { +// QueryResult::GetRecord(Ok(ok)) => { +// for Record { key, value, .. } in ok.records { +// println!( +// "Got record {:?} {:?}", +// std::str::from_utf8(key.as_ref()).unwrap(), +// std::str::from_utf8(&value).unwrap(), +// ); +// } +// } +// QueryResult::GetRecord(Err(err)) => { +// eprintln!("Failed to get record: {:?}", err); +// } +// QueryResult::PutRecord(Ok(PutRecordOk { key })) => { +// println!( +// "Successfully put record {:?}", +// std::str::from_utf8(key.as_ref()).unwrap() +// ); +// } +// QueryResult::PutRecord(Err(err)) => { +// eprintln!("Failed to put record: {:?}", err); +// } +// _ => {} +// } +// _ => {} +// } +// } +// } +// +// // Create a swarm to manage peers and events. +// let mut swarm = { +// // Create a Kademlia behaviour. +// let store = MemoryStore::new(local_peer_id.clone()); +// let kademlia = Kademlia::new(local_peer_id.clone(), store); +// let mdns = Mdns::new()?; +// let behaviour = MyBehaviour { kademlia, mdns }; +// Swarm::new(transport, behaviour, local_peer_id) +// }; +// +// // Read full lines from stdin +// let mut stdin = io::BufReader::new(io::stdin()).lines(); +// +// // Listen on all interfaces and whatever port the OS assigns. +// Swarm::listen_on(&mut swarm, "/ip4/0.0.0.0/tcp/0".parse()?)?; +// +// // Kick it off. +// let mut listening = false; +// task::block_on(future::poll_fn(move |cx: &mut Context| { +// loop { +// match stdin.try_poll_next_unpin(cx)? { +// Poll::Ready(Some(line)) => handle_input_line(&mut swarm.kademlia, line), +// Poll::Ready(None) => panic!("Stdin closed"), +// Poll::Pending => break +// } +// } +// loop { +// match swarm.poll_next_unpin(cx) { +// Poll::Ready(Some(event)) => println!("{:?}", event), +// Poll::Ready(None) => return Poll::Ready(Ok(())), +// Poll::Pending => { +// if !listening { +// if let Some(a) = Swarm::listeners(&swarm).next() { +// println!("Listening on {:?}", a); +// listening = true; +// } +// } +// break +// } +// } +// } +// Poll::Pending +// })) +// } +// +// fn handle_input_line(kademlia: &mut Kademlia, line: String) { +// let mut args = line.split(" "); +// +// match args.next() { +// Some("GET") => { +// let key = { +// match args.next() { +// Some(key) => Key::new(&key), +// None => { +// eprintln!("Expected key"); +// return; +// } +// } +// }; +// kademlia.get_record(&key, Quorum::One); +// } +// Some("PUT") => { +// let key = { +// match args.next() { +// Some(key) => Key::new(&key), +// None => { +// eprintln!("Expected key"); +// return; +// } +// } +// }; +// let value = { +// match args.next() { +// Some(value) => value.as_bytes().to_vec(), +// None => { +// eprintln!("Expected value"); +// return; +// } +// } +// }; +// let record = Record { +// key, +// value, +// publisher: None, +// expires: None, +// }; +// kademlia.put_record(record, Quorum::One).expect("Failed to store record locally."); +// } +// _ => { +// eprintln!("expected GET or PUT"); +// } +// } +// } diff --git a/examples/ipfs-kad.rs b/examples/ipfs-kad.rs index ec48435d..8d45f213 100644 --- a/examples/ipfs-kad.rs +++ b/examples/ipfs-kad.rs @@ -1,129 +1,129 @@ -// Copyright 2018 Parity Technologies (UK) Ltd. +// // Copyright 2018 Parity Technologies (UK) Ltd. +// // +// // Permission is hereby granted, free of charge, to any person obtaining a +// // copy of this software and associated documentation files (the "Software"), +// // to deal in the Software without restriction, including without limitation +// // the rights to use, copy, modify, merge, publish, distribute, sublicense, +// // and/or sell copies of the Software, and to permit persons to whom the +// // Software is furnished to do so, subject to the following conditions: +// // +// // The above copyright notice and this permission notice shall be included in +// // all copies or substantial portions of the Software. +// // +// // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// // FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// // DEALINGS IN THE SOFTWARE. // -// Permission is hereby granted, free of charge, to any person obtaining a -// copy of this software and associated documentation files (the "Software"), -// to deal in the Software without restriction, including without limitation -// the rights to use, copy, modify, merge, publish, distribute, sublicense, -// and/or sell copies of the Software, and to permit persons to whom the -// Software is furnished to do so, subject to the following conditions: +// //! Demonstrates how to perform Kademlia queries on the IPFS network. +// //! +// //! You can pass as parameter a base58 peer ID to search for. If you don't pass any parameter, a +// //! peer ID will be generated randomly. // -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. +// use async_std::task; +// use libp2p::{ +// Swarm, +// PeerId, +// identity, +// build_development_transport +// }; +// use libp2p::kad::{ +// Kademlia, +// KademliaConfig, +// KademliaEvent, +// GetClosestPeersError, +// QueryResult, +// }; +// use libp2p::kad::record::store::MemoryStore; +// use std::{env, error::Error, time::Duration}; // -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -// DEALINGS IN THE SOFTWARE. - -//! Demonstrates how to perform Kademlia queries on the IPFS network. -//! -//! You can pass as parameter a base58 peer ID to search for. If you don't pass any parameter, a -//! peer ID will be generated randomly. - -use async_std::task; -use libp2p::{ - Swarm, - PeerId, - identity, - build_development_transport -}; -use libp2p::kad::{ - Kademlia, - KademliaConfig, - KademliaEvent, - GetClosestPeersError, - QueryResult, -}; -use libp2p::kad::record::store::MemoryStore; -use std::{env, error::Error, time::Duration}; - -fn main() -> Result<(), Box> { - env_logger::init(); - - // Create a random key for ourselves. - let local_key = identity::Keypair::generate_ed25519(); - let local_peer_id = PeerId::from(local_key.public()); - - // Set up a an encrypted DNS-enabled TCP Transport over the Mplex protocol - let transport = build_development_transport(local_key)?; - - // Create a swarm to manage peers and events. - let mut swarm = { - // Create a Kademlia behaviour. - let mut cfg = KademliaConfig::default(); - cfg.set_query_timeout(Duration::from_secs(5 * 60)); - let store = MemoryStore::new(local_peer_id.clone()); - let mut behaviour = Kademlia::with_config(local_peer_id.clone(), store, cfg); - - // TODO: the /dnsaddr/ scheme is not supported (https://github.com/libp2p/rust-libp2p/issues/967) - /*behaviour.add_address(&"QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); - behaviour.add_address(&"QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); - behaviour.add_address(&"QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); - behaviour.add_address(&"QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap());*/ - - // The only address that currently works. - behaviour.add_address(&"QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ".parse()?, "/ip4/104.131.131.82/tcp/4001".parse()?); - - // The following addresses always fail signature verification, possibly due to - // RSA keys with < 2048 bits. - // behaviour.add_address(&"QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM".parse().unwrap(), "/ip4/104.236.179.241/tcp/4001".parse().unwrap()); - // behaviour.add_address(&"QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu".parse().unwrap(), "/ip4/128.199.219.111/tcp/4001".parse().unwrap()); - // behaviour.add_address(&"QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64".parse().unwrap(), "/ip4/104.236.76.40/tcp/4001".parse().unwrap()); - // behaviour.add_address(&"QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd".parse().unwrap(), "/ip4/178.62.158.247/tcp/4001".parse().unwrap()); - - // The following addresses are permanently unreachable: - // Other(Other(A(Transport(A(Underlying(Os { code: 101, kind: Other, message: "Network is unreachable" })))))) - // behaviour.add_address(&"QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM".parse().unwrap(), "/ip6/2604:a880:1:20::203:d001/tcp/4001".parse().unwrap()); - // behaviour.add_address(&"QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu".parse().unwrap(), "/ip6/2400:6180:0:d0::151:6001/tcp/4001".parse().unwrap()); - // behaviour.add_address(&"QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64".parse().unwrap(), "/ip6/2604:a880:800:10::4a:5001/tcp/4001".parse().unwrap()); - // behaviour.add_address(&"QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd".parse().unwrap(), "/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001".parse().unwrap()); - Swarm::new(transport, behaviour, local_peer_id) - }; - - // Order Kademlia to search for a peer. - let to_search: PeerId = if let Some(peer_id) = env::args().nth(1) { - peer_id.parse()? - } else { - identity::Keypair::generate_ed25519().public().into() - }; - - println!("Searching for the closest peers to {:?}", to_search); - swarm.get_closest_peers(to_search); - - // Kick it off! - task::block_on(async move { - loop { - let event = swarm.next().await; - if let KademliaEvent::QueryResult { - result: QueryResult::GetClosestPeers(result), - .. - } = event { - match result { - Ok(ok) => - if !ok.peers.is_empty() { - println!("Query finished with closest peers: {:#?}", ok.peers) - } else { - // The example is considered failed as there - // should always be at least 1 reachable peer. - println!("Query finished with no closest peers.") - } - Err(GetClosestPeersError::Timeout { peers, .. }) => - if !peers.is_empty() { - println!("Query timed out with closest peers: {:#?}", peers) - } else { - // The example is considered failed as there - // should always be at least 1 reachable peer. - println!("Query timed out with no closest peers."); - } - }; - - break; - } - } - - Ok(()) - }) -} +// fn main() -> Result<(), Box> { +// env_logger::init(); +// +// // Create a random key for ourselves. +// let local_key = identity::Keypair::generate_ed25519(); +// let local_peer_id = PeerId::from(local_key.public()); +// +// // Set up a an encrypted DNS-enabled TCP Transport over the Mplex protocol +// let transport = build_development_transport(local_key)?; +// +// // Create a swarm to manage peers and events. +// let mut swarm = { +// // Create a Kademlia behaviour. +// let mut cfg = KademliaConfig::default(); +// cfg.set_query_timeout(Duration::from_secs(5 * 60)); +// let store = MemoryStore::new(local_peer_id.clone()); +// let mut behaviour = Kademlia::with_config(local_peer_id.clone(), store, cfg); +// +// // TODO: the /dnsaddr/ scheme is not supported (https://github.com/libp2p/rust-libp2p/issues/967) +// /*behaviour.add_address(&"QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); +// behaviour.add_address(&"QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); +// behaviour.add_address(&"QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap()); +// behaviour.add_address(&"QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt".parse().unwrap(), "/dnsaddr/bootstrap.libp2p.io".parse().unwrap());*/ +// +// // The only address that currently works. +// behaviour.add_address(&"QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ".parse()?, "/ip4/104.131.131.82/tcp/4001".parse()?); +// +// // The following addresses always fail signature verification, possibly due to +// // RSA keys with < 2048 bits. +// // behaviour.add_address(&"QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM".parse().unwrap(), "/ip4/104.236.179.241/tcp/4001".parse().unwrap()); +// // behaviour.add_address(&"QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu".parse().unwrap(), "/ip4/128.199.219.111/tcp/4001".parse().unwrap()); +// // behaviour.add_address(&"QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64".parse().unwrap(), "/ip4/104.236.76.40/tcp/4001".parse().unwrap()); +// // behaviour.add_address(&"QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd".parse().unwrap(), "/ip4/178.62.158.247/tcp/4001".parse().unwrap()); +// +// // The following addresses are permanently unreachable: +// // Other(Other(A(Transport(A(Underlying(Os { code: 101, kind: Other, message: "Network is unreachable" })))))) +// // behaviour.add_address(&"QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM".parse().unwrap(), "/ip6/2604:a880:1:20::203:d001/tcp/4001".parse().unwrap()); +// // behaviour.add_address(&"QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu".parse().unwrap(), "/ip6/2400:6180:0:d0::151:6001/tcp/4001".parse().unwrap()); +// // behaviour.add_address(&"QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64".parse().unwrap(), "/ip6/2604:a880:800:10::4a:5001/tcp/4001".parse().unwrap()); +// // behaviour.add_address(&"QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd".parse().unwrap(), "/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001".parse().unwrap()); +// Swarm::new(transport, behaviour, local_peer_id) +// }; +// +// // Order Kademlia to search for a peer. +// let to_search: PeerId = if let Some(peer_id) = env::args().nth(1) { +// peer_id.parse()? +// } else { +// identity::Keypair::generate_ed25519().public().into() +// }; +// +// println!("Searching for the closest peers to {:?}", to_search); +// swarm.get_closest_peers(to_search); +// +// // Kick it off! +// task::block_on(async move { +// loop { +// let event = swarm.next().await; +// if let KademliaEvent::QueryResult { +// result: QueryResult::GetClosestPeers(result), +// .. +// } = event { +// match result { +// Ok(ok) => +// if !ok.peers.is_empty() { +// println!("Query finished with closest peers: {:#?}", ok.peers) +// } else { +// // The example is considered failed as there +// // should always be at least 1 reachable peer. +// println!("Query finished with no closest peers.") +// } +// Err(GetClosestPeersError::Timeout { peers, .. }) => +// if !peers.is_empty() { +// println!("Query timed out with closest peers: {:#?}", peers) +// } else { +// // The example is considered failed as there +// // should always be at least 1 reachable peer. +// println!("Query timed out with no closest peers."); +// } +// }; +// +// break; +// } +// } +// +// Ok(()) +// }) +// } From 982e23ad393de9d534111b246528ef12eef346fe Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Tue, 19 May 2020 21:49:51 +0300 Subject: [PATCH 23/25] fn main() {} --- examples/distributed-key-value-store.rs | 2 ++ examples/ipfs-kad.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/examples/distributed-key-value-store.rs b/examples/distributed-key-value-store.rs index 39a4d046..8266744e 100644 --- a/examples/distributed-key-value-store.rs +++ b/examples/distributed-key-value-store.rs @@ -206,3 +206,5 @@ // } // } // } + +fn main() {} diff --git a/examples/ipfs-kad.rs b/examples/ipfs-kad.rs index 8d45f213..665ee705 100644 --- a/examples/ipfs-kad.rs +++ b/examples/ipfs-kad.rs @@ -127,3 +127,5 @@ // Ok(()) // }) // } + +fn main() {} \ No newline at end of file From b4681cca12822f5045b89a354d879cb8394a09af Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Tue, 19 May 2020 23:10:33 +0300 Subject: [PATCH 24/25] fix wasm build --- Cargo.toml | 1 + protocols/kad/src/record.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index a998af69..26d2bd78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,3 +124,4 @@ libp2p-core = { path = "core" } # visible via dependency mechanics (i.e., not visible HERE) [patch.crates-io] libp2p-core = { path = "core" } +wasm-timer = { git = "https://github.com/fluencelabs/wasm-timer", branch = "saturating_duration_since" } diff --git a/protocols/kad/src/record.rs b/protocols/kad/src/record.rs index ec13bfb1..e371d58e 100644 --- a/protocols/kad/src/record.rs +++ b/protocols/kad/src/record.rs @@ -70,7 +70,7 @@ impl From for Key { } /// A record stored in the DHT. -#[derive(Clone, Debug, Eq, PartialEq, Hash)] +#[derive(Clone, Debug, Eq, PartialEq)] pub struct Record { /// Key of the record. pub key: Key, From 0fea4a19865996dfc19bf00db228a856092f63c5 Mon Sep 17 00:00:00 2001 From: folex <0xdxdy@gmail.com> Date: Tue, 19 May 2020 23:17:33 +0300 Subject: [PATCH 25/25] fix wasm build: imports --- protocols/kad/src/protocol.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/protocols/kad/src/protocol.rs b/protocols/kad/src/protocol.rs index 3dc49ba2..a440cf3b 100644 --- a/protocols/kad/src/protocol.rs +++ b/protocols/kad/src/protocol.rs @@ -26,24 +26,22 @@ //! to poll the underlying transport for incoming messages, and the `Sink` component //! is used to send messages to remote peers. -use std::{borrow::Cow, convert::TryFrom, time::Duration}; -use std::{io, iter}; - use bytes::BytesMut; use codec::UviBytes; -use futures::prelude::*; -use futures_codec::Framed; -use prost::Message; -use unsigned_varint::codec; -use derivative::Derivative; -use std::time::Instant; - -use libp2p_core::{Multiaddr, PeerId}; -use libp2p_core::identity::ed25519::PublicKey; -use libp2p_core::upgrade::{InboundUpgrade, OutboundUpgrade, UpgradeInfo}; - use crate::dht_proto as proto; use crate::record::{self, Record}; +use futures::prelude::*; +use futures_codec::Framed; +use libp2p_core::{Multiaddr, PeerId}; +use libp2p_core::upgrade::{InboundUpgrade, OutboundUpgrade, UpgradeInfo}; +use prost::Message; +use std::{borrow::Cow, convert::TryFrom, time::Duration}; +use std::{io, iter}; +use unsigned_varint::codec; +use wasm_timer::Instant; + +use derivative::Derivative; +use libp2p_core::identity::ed25519::PublicKey; /// The protocol name used for negotiating with multistream-select. pub const DEFAULT_PROTO_NAME: &[u8] = b"/ipfs/kad/1.0.0";