From 143bc986c3583e0815beade1376dbead39cae223 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Sat, 11 Mar 2023 17:25:00 +0200 Subject: [PATCH 01/98] WIP: Introduce `ProtocolController` --- client/network/src/service.rs | 2 +- client/peerset/src/lib.rs | 1 + client/peerset/src/protocol_controller.rs | 576 ++++++++++++++++++++++ 3 files changed, 578 insertions(+), 1 deletion(-) create mode 100644 client/peerset/src/protocol_controller.rs diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 4a05393618cfb..8c09997a4a584 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -620,7 +620,7 @@ where } /// Returns the list of reserved peers. - pub fn reserved_peers(&self) -> impl Iterator { + fn reserved_peers(&self) -> impl Iterator { self.network_service.behaviour().user_protocol().reserved_peers() } } diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index e5393acbaa32f..5add3c518c0eb 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -33,6 +33,7 @@ //! will at all time try to maintain a connection with. mod peersstate; +mod protocol_controller; use futures::{channel::oneshot, prelude::*}; use log::{debug, error, trace}; diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs new file mode 100644 index 0000000000000..7cd1599be9514 --- /dev/null +++ b/client/peerset/src/protocol_controller.rs @@ -0,0 +1,576 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use futures::{FutureExt, StreamExt}; +use libp2p::PeerId; +use log::{error, trace}; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use std::{ + borrow::Cow, + collections::{ + hash_map::Entry, + HashMap, HashSet, + }, + time::{Duration, Instant}, +}; +use wasm_timer::Delay; + +use crate::{IncomingIndex, Message, SetConfig, SetId}; + +#[derive(Debug)] +enum Action { + AddReservedPeer(PeerId), + RemoveReservedPeer(PeerId), + SetReservedPeers(HashSet), + SetReservedOnly(bool), + AddPeer(PeerId), + RemovePeer(PeerId), + IncomingConnection(PeerId, IncomingIndex), +} + +/// Shared handle to [`ProtocolController`]. Distributed around the code. +pub struct ProtocolHandle { + to_controller: TracingUnboundedSender, +} + +impl ProtocolHandle { + /// Adds a new reserved peer. [`ProtocolController`] will make an effort + /// to always remain connected to this peer. + /// + /// Has no effect if the node was already a reserved peer. + /// + /// > **Note**: Keep in mind that the networking has to know an address for this node, + /// > otherwise it will not be able to connect to it. + pub fn add_reserved_peer(&self, peer_id: PeerId) { + let _ = self.to_controller.unbounded_send(Action::AddReservedPeer(peer_id)); + } + + /// Remove a previously-added reserved peer. + /// + /// Has no effect if the node was not a reserved peer. + pub fn remove_reserved_peer(&self, peer_id: PeerId) { + let _ = self.to_controller.unbounded_send(Action::RemoveReservedPeer(peer_id)); + } + + /// Set reserved peers to the new set. + pub fn set_reserved_peers(&self, peer_ids: HashSet) { + let _ = self.to_controller.unbounded_send(Action::SetReservedPeers(peer_ids)); + } + + /// Sets whether or not [`ProtocolController`] only has connections with nodes marked + /// as reserved for the given set. + pub fn set_reserved_only(&self, reserved: bool) { + let _ = self.to_controller.unbounded_send(Action::SetReservedOnly(reserved)); + } + + /// Add a peer to a set of peers we try to connect to. + pub fn add_peer(&self, peer_id: PeerId) { + let _ = self.to_controller.unbounded_send(Action::AddPeer(peer_id)); + } + + /// Remove a peer from a set of peers we try to connect to and disconnect a peer. + pub fn remove_peer(&self, peer_id: PeerId) { + let _ = self.to_controller.unbounded_send(Action::RemovePeer(peer_id)); + } + + /// Notify about incoming connection. [`ProtocolController`] will either accept or reject it. + pub fn incoming_connection(&self, peer_id: PeerId, incoming_index: IncomingIndex) { + let _ = self + .to_controller + .unbounded_send(Action::IncomingConnection(peer_id, incoming_index)); + } +} + +/// Status of a connection with a peer. +#[derive(Debug)] +enum ConnectionState { + /// We are connected through an ingoing connection. + In, + /// We are connected through an outgoing connection. + Out, + /// We are not connected. + NotConnected, +} + +impl ConnectionState { + /// Returns true if we are connected with the node. + fn is_connected(&self) -> bool { + matches!(self, ConnectionState::In | ConnectionState::Out) + } +} + +impl Default for ConnectionState { + fn default() -> ConnectionState { + ConnectionState::NotConnected + } +} + +/// Side of [`ProtocolHandle`] responsible for all the logic. Currently all instances are +/// owned by [`crate::Peerset`], but they should eventually be moved to corresponding protocols. +#[derive(Debug)] +pub struct ProtocolController { + /// Set id to use when sending connect/drop requests to `Notifications`. + // Will likely be replaced by `ProtocolName` in the future. + set_id: SetId, + /// Receiver for messages from [`ProtocolHandle`]. + from_handle: TracingUnboundedReceiver, + /// Number of occupied slots for incoming connections. + num_in: u32, + /// Number of occupied slots for outgoing connections. + num_out: u32, + /// Maximum number of slots for incoming connections. + max_in: u32, + /// Maximum number of slots for outgoing connections. + max_out: u32, + /// Reserved nodes. + reserved_nodes: HashSet, + /// Connect only to reserved nodes. + reserved_only: bool, + /// Peers and their connection states (including reserved nodes). + nodes: HashMap, + /// Next time to allocate slots. This is done once per second. + next_periodic_alloc_slots: Instant, + /// Outgoing channel for messages to `Notifications`. + to_notifications: TracingUnboundedSender, +} + +impl ProtocolController { + /// Construct new [`ProtocolController`]. + pub fn new( + set_id: SetId, + config: SetConfig, + to_notifications: TracingUnboundedSender, + ) -> (ProtocolHandle, ProtocolController) { + let (to_controller, from_handle) = tracing_unbounded("mpsc_protocol_controller", 10_000); + let handle = ProtocolHandle { to_controller }; + let nodes = config + .reserved_nodes + .union(&config.bootnodes.into_iter().collect()) + .map(|p| (*p, ConnectionState::NotConnected)) + .collect::>(); + let controller = ProtocolController { + set_id, + from_handle, + num_in: 0, + num_out: 0, + max_in: config.in_peers, + max_out: config.out_peers, + reserved_nodes: config.reserved_nodes, + reserved_only: config.reserved_only, + nodes, + next_periodic_alloc_slots: Instant::now(), + to_notifications, + }; + (handle, controller) + } + + /// Drive [`ProtocolController`]. This function returns when all instances of + /// [`ProtocolHandle`] are dropped. + pub async fn run(mut self) { + while self.next_action().await {} + } + + /// Perform one action. Returns `true` if it should be called again. + /// + /// Intended for tests only. Use `run` for driving [`ProtocolController`]. + pub async fn next_action(&mut self) -> bool { + let action = loop { + let mut next_alloc_slots = Delay::new_at(self.next_periodic_alloc_slots).fuse(); + futures::select! { + action = self.from_handle.next() => match action { + Some(action) => break action, + None => return false, + }, + _ = next_alloc_slots => { + self.alloc_slots(); + self.next_periodic_alloc_slots = Instant::now() + Duration::new(1, 0); + }, + } + }; + match action { + Action::AddReservedPeer(peer_id) => self.on_add_reserved_peer(peer_id), + Action::RemoveReservedPeer(peer_id) => self.on_remove_reserved_peer(peer_id), + Action::SetReservedPeers(peer_ids) => self.on_set_reserved_peers(peer_ids), + Action::SetReservedOnly(reserved_only) => self.on_set_reserved_only(reserved_only), + Action::AddPeer(peer_id) => self.on_add_peer(peer_id), + Action::RemovePeer(peer_id) => self.on_remove_peer(peer_id), + Action::IncomingConnection(peer_id, index) => + self.on_incoming_connection(peer_id, index), + } + true + } + + fn on_add_reserved_peer(&mut self, peer_id: PeerId) { + if !self.reserved_nodes.insert(peer_id) { + return + } + // Discount occupied slots or connect to the node. + match self.nodes.entry(peer_id).or_default() { + ConnectionState::In => self.num_in -= 1, + ConnectionState::Out => self.num_out -= 1, + ConnectionState::NotConnected => self.alloc_slots(), + } + } + + fn on_remove_reserved_peer(&mut self, peer_id: PeerId) { + if !self.reserved_nodes.remove(&peer_id) { + return + } + if let Entry::Occupied(mut node) = self.nodes.entry(peer_id) { + if self.reserved_only && node.get().is_connected() { + // If we are in reserved-only mode, the removed reserved node + // should be disconnected. + // Note that we don't need to update slots, so we drop the node manually here + // and not via [`Peer`] interface + *node.get_mut() = ConnectionState::NotConnected; + let _ = self + .to_notifications + .unbounded_send(Message::Drop { set_id: self.set_id, peer_id: *node.key() }); + trace!( + target: "peerset", + "Disconnecting previously reserved node {} on {:?}.", + node.key(), self.set_id + ); + } else { + // Otherwise, just count occupied slots for the node. + match node.get() { + ConnectionState::In => self.num_in += 1, + ConnectionState::Out => self.num_out += 1, + ConnectionState::NotConnected => {}, + } + } + } else { + error!( + target: "peerset", + "Invalid state: reserved node {} not in the list of known nodes.", + peer_id, + ); + } + } + + fn on_set_reserved_peers(&mut self, peer_ids: HashSet) { + // Determine the difference between the current group and the new list. + let to_insert = peer_ids.difference(&self.reserved_nodes).cloned().collect::>(); + let to_remove = self.reserved_nodes.difference(&peer_ids).cloned().collect::>(); + + for node in to_insert { + self.on_add_reserved_peer(node); + } + + for node in to_remove { + self.on_remove_reserved_peer(node); + } + } + + fn on_set_reserved_only(&mut self, reserved_only: bool) { + self.reserved_only = reserved_only; + + if reserved_only { + // Disconnect all non-reserved peers. + for peer_id in self.connected_peers().cloned().collect::>().iter() { + self.peer(&peer_id) + .into_connected() + .expect( + "We are enumerating connected peers, therefore the peer is connected; qed", + ) + .disconnect(); + } + } else { + // Try connecting to non-reserved peers. + self.alloc_slots(); + } + } + + fn on_add_peer(&mut self, peer_id: PeerId) { + if self.nodes.insert(peer_id, ConnectionState::NotConnected).is_none() { + self.alloc_slots(); + } + } + + fn on_remove_peer(&mut self, peer_id: PeerId) { + // Don't do anything if the node is reserved. + if self.reserved_nodes.contains(&peer_id) { + return + } + + self.peer(&peer_id).into_connected().map(|connected_peer| connected_peer.disconnect()); + self.nodes.remove(&peer_id); + } + + fn on_incoming_connection(&mut self, peer_id: PeerId, incoming_index: IncomingIndex) { + if self.reserved_only && !self.reserved_nodes.contains(&peer_id) { + let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); + return + } + + let not_connected = match self.peer(&peer_id) { + // If we're already connected, don't answer, as the docs mention. + Peer::Connected(_) => return, + Peer::NotConnected(peer) => peer, + // Adding incoming peer to our set of peers even before we decide whether to accept + // it is questionable, but this is how it was implemented in the original `Peerset`. + Peer::Unknown(peer) => peer.discover(), + }; + + if not_connected.is_banned() { + let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); + return; + } + + let _ = not_connected.try_accept_incoming(incoming_index); + } + + fn peer<'a>(&'a mut self, peer_id: &'a PeerId) -> Peer<'a> { + match self.nodes.get(peer_id) { + None => + Peer::Unknown(UnknownPeer { controller: self, peer_id: Cow::Borrowed(peer_id) }), + Some(ConnectionState::NotConnected) => Peer::NotConnected(NotConnectedPeer { + controller: self, + peer_id: Cow::Borrowed(peer_id), + }), + Some(ConnectionState::In) | Some(ConnectionState::Out) => + Peer::Connected(ConnectedPeer { controller: self, peer_id: Cow::Borrowed(peer_id) }), + } + } + + fn connected_peers(&self) -> impl Iterator { + self.nodes + .iter() + .filter(|(_, state)| state.is_connected()) + .map(|(peer_id, _)| peer_id) + } + + fn alloc_slots(&mut self) { + todo!( + "Count the number of slots we need to fill and request that number of peers from `Peerset`, + also passing the list of already connected peers to eliminate them from the list." + ); + } +} +/// Grants access to the state of a peer in the [`ProtocolController`]. +pub enum Peer<'a> { + /// We are connected to this node. + Connected(ConnectedPeer<'a>), + /// We are not connected to this node. + NotConnected(NotConnectedPeer<'a>), + /// We have never heard of this node, or it is not part of the set. + Unknown(UnknownPeer<'a>), +} + +impl<'a> Peer<'a> { + /// If we are the `Connected` variant, returns the inner [`ConnectedPeer`]. Returns `None` + /// otherwise. + fn into_connected(self) -> Option> { + match self { + Self::Connected(peer) => Some(peer), + Self::NotConnected(..) | Self::Unknown(..) => None, + } + } + + /// If we are the `NotConnected` variant, returns the inner [`NotConnectedPeer`]. Returns `None` + /// otherwise. + #[cfg(test)] // Feel free to remove this if this function is needed outside of tests + fn into_not_connected(self) -> Option> { + match self { + Self::NotConnected(peer) => Some(peer), + Self::Connected(..) | Self::Unknown(..) => None, + } + } + + /// If we are the `Unknown` variant, returns the inner [`UnknownPeer`]. Returns `None` + /// otherwise. + #[cfg(test)] // Feel free to remove this if this function is needed outside of tests + fn into_unknown(self) -> Option> { + match self { + Self::Unknown(peer) => Some(peer), + Self::Connected(..) | Self::NotConnected(..) => None, + } + } +} + +/// A peer that is connected to us. +pub struct ConnectedPeer<'a> { + controller: &'a mut ProtocolController, + peer_id: Cow<'a, PeerId>, +} + +impl<'a> ConnectedPeer<'a> { + /// Get the `PeerId` associated to this `ConnectedPeer`. + fn peer_id(&self) -> &PeerId { + &self.peer_id + } + + /// Destroys this `ConnectedPeer` and returns the `PeerId` inside of it. + fn into_peer_id(self) -> PeerId { + self.peer_id.into_owned() + } + + /// Switches the peer to "not connected". + fn disconnect(self) -> NotConnectedPeer<'a> { + let is_no_slot_occupy = self.controller.reserved_nodes.contains(&*self.peer_id); + if let Some(state) = self.controller.nodes.get_mut(&*self.peer_id) { + if !is_no_slot_occupy { + match state { + ConnectionState::In => self.controller.num_in -= 1, + ConnectionState::Out => self.controller.num_out -= 1, + ConnectionState::NotConnected => { + debug_assert!( + false, + "State inconsistency: disconnecting a disconnected node" + ) + }, + } + } + *state = ConnectionState::NotConnected; + let _ = self.controller.to_notifications.unbounded_send(Message::Drop { + set_id: self.controller.set_id, + peer_id: *self.peer_id, + }); + } else { + debug_assert!(false, "State inconsistency: disconnecting a disconnected node"); + } + + NotConnectedPeer { controller: self.controller, peer_id: self.peer_id } + } +} + +// A peer that is not connected to us. +pub struct NotConnectedPeer<'a> { + controller: &'a mut ProtocolController, + peer_id: Cow<'a, PeerId>, +} + +impl<'a> NotConnectedPeer<'a> { + /// Destroys this `NotConnectedPeer` and returns the `PeerId` inside of it. + fn into_peer_id(self) -> PeerId { + self.peer_id.into_owned() + } + + /// Tries to set the peer as connected as an outgoing connection. + /// + /// If there are enough slots available, switches the node to "connected" and returns `Ok`. If + /// the slots are full, the node stays "not connected" and we return `Err`. + /// + /// Non-slot-occupying nodes don't count towards the number of slots. + fn try_outgoing(self) -> Result, Self> { + let is_no_slot_occupy = self.controller.reserved_nodes.contains(&*self.peer_id); + + // Note that it is possible for num_out to be strictly superior to the max, in case we were + // connected to reserved node then marked them as not reserved. + if self.controller.num_out >= self.controller.max_out && !is_no_slot_occupy { + return Err(self) + } + + if let Some(state) = self.controller.nodes.get_mut(&*self.peer_id) { + debug_assert!( + matches!(state, ConnectionState::NotConnected), + "State inconsistency: try_outgoing on a connected node" + ); + if !is_no_slot_occupy { + self.controller.num_out += 1; + } + *state = ConnectionState::Out; + let _ = self.controller.to_notifications.unbounded_send(Message::Connect { + set_id: self.controller.set_id, + peer_id: *self.peer_id, + }); + } else { + debug_assert!(false, "State inconsistency: try_outgoing on an unknown node"); + } + + Ok(ConnectedPeer { controller: self.controller, peer_id: self.peer_id }) + } + + /// Tries to accept the peer as an incoming connection. + /// + /// If there are enough slots available, switches the node to "connected" and returns `Ok`. If + /// the slots are full, the node stays "not connected" and we return `Err`. + /// + /// Non-slot-occupying nodes don't count towards the number of slots. + fn try_accept_incoming( + self, + incoming_index: IncomingIndex, + ) -> Result, Self> { + let is_no_slot_occupy = self.controller.reserved_nodes.contains(&*self.peer_id); + + // Note that it is possible for num_in to be strictly superior to the max, in case we were + // connected to reserved node then marked them as not reserved. + if self.controller.num_in >= self.controller.max_in && !is_no_slot_occupy { + return Err(self) + } + + if let Some(state) = self.controller.nodes.get_mut(&*self.peer_id) { + debug_assert!( + matches!(state, ConnectionState::NotConnected), + "State inconsistency: try_accept_incoming on a connected node" + ); + if !is_no_slot_occupy { + self.controller.num_in += 1; + } + *state = ConnectionState::In; + let _ = + self.controller.to_notifications.unbounded_send(Message::Accept(incoming_index)); + } else { + debug_assert!(false, "State inconsistency: try_accept_incoming on an unknown node"); + } + + Ok(ConnectedPeer { controller: self.controller, peer_id: self.peer_id }) + } + + /// Removes the peer from the list of members of the set. + fn forget_peer(self) -> UnknownPeer<'a> { + if self.controller.nodes.remove(&*self.peer_id).is_none() { + debug_assert!(false, "State inconsistency: forget_peer on an unknown node"); + error!( + target: "peerset", + "State inconsistency with {} when forgetting peer", + self.peer_id + ); + }; + + UnknownPeer { controller: self.controller, peer_id: self.peer_id } + } + + fn is_banned(&self) -> bool { + todo!("request this info from `PeerSet`"); + } +} + +/// A peer that we have never heard of or that isn't part of the set. +pub struct UnknownPeer<'a> { + controller: &'a mut ProtocolController, + peer_id: Cow<'a, PeerId>, +} + +impl<'a> UnknownPeer<'a> { + /// Inserts the peer identity in our list. + /// + /// The node starts with a reputation of 0. You can adjust these default + /// values using the `NotConnectedPeer` that this method returns. + fn discover(self) -> NotConnectedPeer<'a> { + if self + .controller + .nodes + .insert(*self.peer_id, ConnectionState::NotConnected) + .is_some() + { + debug_assert!(false, "State inconsistency: discovered already known peer"); + } + NotConnectedPeer { controller: self.controller, peer_id: self.peer_id } + } +} From 5256f197654967b8e8a3b7709beb28c6982ce150 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 16 Mar 2023 10:56:12 +0200 Subject: [PATCH 02/98] Code review suggestions: docs and `trace`->`info` --- client/peerset/src/protocol_controller.rs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 7cd1599be9514..7bc0b4fb77b80 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -18,14 +18,11 @@ use futures::{FutureExt, StreamExt}; use libp2p::PeerId; -use log::{error, trace}; +use log::{error, info, trace}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{ borrow::Cow, - collections::{ - hash_map::Entry, - HashMap, HashSet, - }, + collections::{hash_map::Entry, HashMap, HashSet}, time::{Duration, Instant}, }; use wasm_timer::Delay; @@ -43,7 +40,8 @@ enum Action { IncomingConnection(PeerId, IncomingIndex), } -/// Shared handle to [`ProtocolController`]. Distributed around the code. +/// Shared handle to [`ProtocolController`]. Distributed around the code outside of the +/// protocol implementation. pub struct ProtocolHandle { to_controller: TracingUnboundedSender, } @@ -241,7 +239,7 @@ impl ProtocolController { let _ = self .to_notifications .unbounded_send(Message::Drop { set_id: self.set_id, peer_id: *node.key() }); - trace!( + info!( target: "peerset", "Disconnecting previously reserved node {} on {:?}.", node.key(), self.set_id @@ -308,7 +306,9 @@ impl ProtocolController { return } - self.peer(&peer_id).into_connected().map(|connected_peer| connected_peer.disconnect()); + self.peer(&peer_id) + .into_connected() + .map(|connected_peer| connected_peer.disconnect()); self.nodes.remove(&peer_id); } @@ -329,7 +329,7 @@ impl ProtocolController { if not_connected.is_banned() { let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); - return; + return } let _ = not_connected.try_accept_incoming(incoming_index); @@ -502,10 +502,7 @@ impl<'a> NotConnectedPeer<'a> { /// the slots are full, the node stays "not connected" and we return `Err`. /// /// Non-slot-occupying nodes don't count towards the number of slots. - fn try_accept_incoming( - self, - incoming_index: IncomingIndex, - ) -> Result, Self> { + fn try_accept_incoming(self, incoming_index: IncomingIndex) -> Result, Self> { let is_no_slot_occupy = self.controller.reserved_nodes.contains(&*self.peer_id); // Note that it is possible for num_in to be strictly superior to the max, in case we were From c333aacf603356988d3fb61c4512b465bf9411d8 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 16 Mar 2023 11:57:22 +0300 Subject: [PATCH 03/98] Apply suggestions from code review Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> --- client/peerset/src/protocol_controller.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 7bc0b4fb77b80..179edd691096b 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -81,7 +81,7 @@ impl ProtocolHandle { let _ = self.to_controller.unbounded_send(Action::AddPeer(peer_id)); } - /// Remove a peer from a set of peers we try to connect to and disconnect a peer. + /// Remove a peer from a set of peers we try to connect to and disconnect the peer. pub fn remove_peer(&self, peer_id: PeerId) { let _ = self.to_controller.unbounded_send(Action::RemovePeer(peer_id)); } @@ -97,7 +97,7 @@ impl ProtocolHandle { /// Status of a connection with a peer. #[derive(Debug)] enum ConnectionState { - /// We are connected through an ingoing connection. + /// We are connected through an incoming connection. In, /// We are connected through an outgoing connection. Out, @@ -200,6 +200,7 @@ impl ProtocolController { }, } }; + match action { Action::AddReservedPeer(peer_id) => self.on_add_reserved_peer(peer_id), Action::RemoveReservedPeer(peer_id) => self.on_remove_reserved_peer(peer_id), @@ -229,6 +230,7 @@ impl ProtocolController { if !self.reserved_nodes.remove(&peer_id) { return } + if let Entry::Occupied(mut node) = self.nodes.entry(peer_id) { if self.reserved_only && node.get().is_connected() { // If we are in reserved-only mode, the removed reserved node From b50f134d115635c7a5829968577fe7523ee1c4e2 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 16 Mar 2023 12:39:48 +0200 Subject: [PATCH 04/98] Rename `ConnectionState`->`PeerState` and reduce states to `Connected(_)` & `NotConnected` --- client/peerset/src/protocol_controller.rs | 74 +++++++++++------------ 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 179edd691096b..919b3c965f5f2 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -94,27 +94,32 @@ impl ProtocolHandle { } } +/// Direction of a connection +#[derive(Debug)] +enum Direction { + Inbound, + Outbound, +} + /// Status of a connection with a peer. #[derive(Debug)] -enum ConnectionState { - /// We are connected through an incoming connection. - In, - /// We are connected through an outgoing connection. - Out, +enum PeerState { + /// We are connected to the peer. + Connected(Direction), /// We are not connected. NotConnected, } -impl ConnectionState { +impl PeerState { /// Returns true if we are connected with the node. fn is_connected(&self) -> bool { - matches!(self, ConnectionState::In | ConnectionState::Out) + matches!(self, PeerState::Connected(_)) } } -impl Default for ConnectionState { - fn default() -> ConnectionState { - ConnectionState::NotConnected +impl Default for PeerState { + fn default() -> PeerState { + PeerState::NotConnected } } @@ -140,7 +145,7 @@ pub struct ProtocolController { /// Connect only to reserved nodes. reserved_only: bool, /// Peers and their connection states (including reserved nodes). - nodes: HashMap, + nodes: HashMap, /// Next time to allocate slots. This is done once per second. next_periodic_alloc_slots: Instant, /// Outgoing channel for messages to `Notifications`. @@ -159,8 +164,8 @@ impl ProtocolController { let nodes = config .reserved_nodes .union(&config.bootnodes.into_iter().collect()) - .map(|p| (*p, ConnectionState::NotConnected)) - .collect::>(); + .map(|p| (*p, PeerState::NotConnected)) + .collect::>(); let controller = ProtocolController { set_id, from_handle, @@ -220,9 +225,9 @@ impl ProtocolController { } // Discount occupied slots or connect to the node. match self.nodes.entry(peer_id).or_default() { - ConnectionState::In => self.num_in -= 1, - ConnectionState::Out => self.num_out -= 1, - ConnectionState::NotConnected => self.alloc_slots(), + PeerState::Connected(Direction::Inbound) => self.num_in -= 1, + PeerState::Connected(Direction::Outbound) => self.num_out -= 1, + PeerState::NotConnected => self.alloc_slots(), } } @@ -237,7 +242,7 @@ impl ProtocolController { // should be disconnected. // Note that we don't need to update slots, so we drop the node manually here // and not via [`Peer`] interface - *node.get_mut() = ConnectionState::NotConnected; + *node.get_mut() = PeerState::NotConnected; let _ = self .to_notifications .unbounded_send(Message::Drop { set_id: self.set_id, peer_id: *node.key() }); @@ -249,9 +254,9 @@ impl ProtocolController { } else { // Otherwise, just count occupied slots for the node. match node.get() { - ConnectionState::In => self.num_in += 1, - ConnectionState::Out => self.num_out += 1, - ConnectionState::NotConnected => {}, + PeerState::Connected(Direction::Inbound) => self.num_in += 1, + PeerState::Connected(Direction::Outbound) => self.num_out += 1, + PeerState::NotConnected => {}, } } } else { @@ -297,7 +302,7 @@ impl ProtocolController { } fn on_add_peer(&mut self, peer_id: PeerId) { - if self.nodes.insert(peer_id, ConnectionState::NotConnected).is_none() { + if self.nodes.insert(peer_id, PeerState::NotConnected).is_none() { self.alloc_slots(); } } @@ -341,11 +346,11 @@ impl ProtocolController { match self.nodes.get(peer_id) { None => Peer::Unknown(UnknownPeer { controller: self, peer_id: Cow::Borrowed(peer_id) }), - Some(ConnectionState::NotConnected) => Peer::NotConnected(NotConnectedPeer { + Some(PeerState::NotConnected) => Peer::NotConnected(NotConnectedPeer { controller: self, peer_id: Cow::Borrowed(peer_id), }), - Some(ConnectionState::In) | Some(ConnectionState::Out) => + Some(PeerState::Connected(_)) => Peer::Connected(ConnectedPeer { controller: self, peer_id: Cow::Borrowed(peer_id) }), } } @@ -428,9 +433,9 @@ impl<'a> ConnectedPeer<'a> { if let Some(state) = self.controller.nodes.get_mut(&*self.peer_id) { if !is_no_slot_occupy { match state { - ConnectionState::In => self.controller.num_in -= 1, - ConnectionState::Out => self.controller.num_out -= 1, - ConnectionState::NotConnected => { + PeerState::Connected(Direction::Inbound) => self.controller.num_in -= 1, + PeerState::Connected(Direction::Outbound) => self.controller.num_out -= 1, + PeerState::NotConnected => { debug_assert!( false, "State inconsistency: disconnecting a disconnected node" @@ -438,7 +443,7 @@ impl<'a> ConnectedPeer<'a> { }, } } - *state = ConnectionState::NotConnected; + *state = PeerState::NotConnected; let _ = self.controller.to_notifications.unbounded_send(Message::Drop { set_id: self.controller.set_id, peer_id: *self.peer_id, @@ -480,13 +485,13 @@ impl<'a> NotConnectedPeer<'a> { if let Some(state) = self.controller.nodes.get_mut(&*self.peer_id) { debug_assert!( - matches!(state, ConnectionState::NotConnected), + matches!(state, PeerState::NotConnected), "State inconsistency: try_outgoing on a connected node" ); if !is_no_slot_occupy { self.controller.num_out += 1; } - *state = ConnectionState::Out; + *state = PeerState::Connected(Direction::Outbound); let _ = self.controller.to_notifications.unbounded_send(Message::Connect { set_id: self.controller.set_id, peer_id: *self.peer_id, @@ -515,13 +520,13 @@ impl<'a> NotConnectedPeer<'a> { if let Some(state) = self.controller.nodes.get_mut(&*self.peer_id) { debug_assert!( - matches!(state, ConnectionState::NotConnected), + matches!(state, PeerState::NotConnected), "State inconsistency: try_accept_incoming on a connected node" ); if !is_no_slot_occupy { self.controller.num_in += 1; } - *state = ConnectionState::In; + *state = PeerState::Connected(Direction::Inbound); let _ = self.controller.to_notifications.unbounded_send(Message::Accept(incoming_index)); } else { @@ -562,12 +567,7 @@ impl<'a> UnknownPeer<'a> { /// The node starts with a reputation of 0. You can adjust these default /// values using the `NotConnectedPeer` that this method returns. fn discover(self) -> NotConnectedPeer<'a> { - if self - .controller - .nodes - .insert(*self.peer_id, ConnectionState::NotConnected) - .is_some() - { + if self.controller.nodes.insert(*self.peer_id, PeerState::NotConnected).is_some() { debug_assert!(false, "State inconsistency: discovered already known peer"); } NotConnectedPeer { controller: self.controller, peer_id: self.peer_id } From 2670615332d27eea5c2a3b95ecfd893579abde7d Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 16 Mar 2023 17:04:25 +0200 Subject: [PATCH 05/98] Get rid of `Peer` abstraction --- client/peerset/src/lib.rs | 11 + client/peerset/src/protocol_controller.rs | 416 ++++++++-------------- 2 files changed, 153 insertions(+), 274 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 5add3c518c0eb..eeada09856437 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -177,6 +177,16 @@ impl PeersetHandle { // The channel can only be closed if the peerset no longer exists. rx.await.map_err(|_| ()) } + + /// Checks whether the peer has a reputation value below [`BANNED_THRESHOLD`]. + pub fn is_banned(&self, peer_id: PeerId) -> bool { + todo!("will be implemented after `Peerset` is converted into shared struct"); + } + + /// Report disconnect to adjust peers reputation value. + pub fn report_disconnect(&self, peer_id: PeerId) { + todo!("will be implemented after `Peerset` is converted into shared struct"); + } } /// Message that can be sent by the peer set manager (PSM). @@ -765,6 +775,7 @@ impl Stream for Peerset { } /// Reason for calling [`Peerset::dropped`]. +#[derive(Debug)] pub enum DropReason { /// Substream or connection has been closed for an unknown reason. Unknown, diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 919b3c965f5f2..a5f9a81bbddce 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -21,13 +21,12 @@ use libp2p::PeerId; use log::{error, info, trace}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use std::{ - borrow::Cow, collections::{hash_map::Entry, HashMap, HashSet}, time::{Duration, Instant}, }; use wasm_timer::Delay; -use crate::{IncomingIndex, Message, SetConfig, SetId}; +use crate::{DropReason, IncomingIndex, Message, PeersetHandle, SetConfig, SetId}; #[derive(Debug)] enum Action { @@ -38,6 +37,7 @@ enum Action { AddPeer(PeerId), RemovePeer(PeerId), IncomingConnection(PeerId, IncomingIndex), + Dropped(PeerId, DropReason), } /// Shared handle to [`ProtocolController`]. Distributed around the code outside of the @@ -92,6 +92,11 @@ impl ProtocolHandle { .to_controller .unbounded_send(Action::IncomingConnection(peer_id, incoming_index)); } + + /// Notify that connection was dropped (eithere refused or disconnected). + pub fn dropped(&self, peer_id: PeerId, reason: DropReason) { + let _ = self.to_controller.unbounded_send(Action::Dropped(peer_id, reason)); + } } /// Direction of a connection @@ -150,6 +155,9 @@ pub struct ProtocolController { next_periodic_alloc_slots: Instant, /// Outgoing channel for messages to `Notifications`. to_notifications: TracingUnboundedSender, + /// Peerset handle for checking peer reputation values and getting connection candidates + /// with highest reputation. + peerset_handle: PeersetHandle, } impl ProtocolController { @@ -158,6 +166,7 @@ impl ProtocolController { set_id: SetId, config: SetConfig, to_notifications: TracingUnboundedSender, + peerset_handle: PeersetHandle, ) -> (ProtocolHandle, ProtocolController) { let (to_controller, from_handle) = tracing_unbounded("mpsc_protocol_controller", 10_000); let handle = ProtocolHandle { to_controller }; @@ -178,6 +187,7 @@ impl ProtocolController { nodes, next_periodic_alloc_slots: Instant::now(), to_notifications, + peerset_handle, }; (handle, controller) } @@ -215,6 +225,7 @@ impl ProtocolController { Action::RemovePeer(peer_id) => self.on_remove_peer(peer_id), Action::IncomingConnection(peer_id, index) => self.on_incoming_connection(peer_id, index), + Action::Dropped(peer_id, reason) => self.on_peer_dropped(peer_id, reason), } true } @@ -224,7 +235,7 @@ impl ProtocolController { return } // Discount occupied slots or connect to the node. - match self.nodes.entry(peer_id).or_default() { + match self.nodes.entry(peer_id).or_insert(PeerState::NotConnected) { PeerState::Connected(Direction::Inbound) => self.num_in -= 1, PeerState::Connected(Direction::Outbound) => self.num_out -= 1, PeerState::NotConnected => self.alloc_slots(), @@ -236,35 +247,42 @@ impl ProtocolController { return } - if let Entry::Occupied(mut node) = self.nodes.entry(peer_id) { - if self.reserved_only && node.get().is_connected() { - // If we are in reserved-only mode, the removed reserved node - // should be disconnected. - // Note that we don't need to update slots, so we drop the node manually here - // and not via [`Peer`] interface - *node.get_mut() = PeerState::NotConnected; - let _ = self - .to_notifications - .unbounded_send(Message::Drop { set_id: self.set_id, peer_id: *node.key() }); - info!( + match self.nodes.entry(peer_id) { + Entry::Occupied(mut node) => { + if !node.get().is_connected() { + return + } + + if self.reserved_only { + // If we are in reserved-only mode, the removed reserved node + // should be disconnected. + *node.get_mut() = PeerState::NotConnected; + let _ = self.to_notifications.unbounded_send(Message::Drop { + set_id: self.set_id, + peer_id: *node.key(), + }); + info!( + target: "peerset", + "Disconnecting previously reserved node {} on {:?}.", + node.key(), self.set_id + ); + } else { + // Otherwise, just count occupied slots for the node. + match node.get() { + PeerState::Connected(Direction::Inbound) => self.num_in += 1, + PeerState::Connected(Direction::Outbound) => self.num_out += 1, + PeerState::NotConnected => {}, + } + } + }, + Entry::Vacant(_) => { + debug_assert!(false, "Reserved node not in the list of known nodes."); + error!( target: "peerset", - "Disconnecting previously reserved node {} on {:?}.", - node.key(), self.set_id + "Invalid state: reserved node {} not in the list of known nodes.", + peer_id, ); - } else { - // Otherwise, just count occupied slots for the node. - match node.get() { - PeerState::Connected(Direction::Inbound) => self.num_in += 1, - PeerState::Connected(Direction::Outbound) => self.num_out += 1, - PeerState::NotConnected => {}, - } - } - } else { - error!( - target: "peerset", - "Invalid state: reserved node {} not in the list of known nodes.", - peer_id, - ); + }, } } @@ -287,13 +305,21 @@ impl ProtocolController { if reserved_only { // Disconnect all non-reserved peers. - for peer_id in self.connected_peers().cloned().collect::>().iter() { - self.peer(&peer_id) - .into_connected() - .expect( - "We are enumerating connected peers, therefore the peer is connected; qed", - ) - .disconnect(); + for (peer_id, state) in self.nodes.iter_mut() { + match state { + PeerState::Connected(d) => { + match d { + Direction::Inbound => self.num_in -= 1, + Direction::Outbound => self.num_out -= 1, + } + *state = PeerState::NotConnected; + let _ = self.to_notifications.unbounded_send(Message::Drop { + set_id: self.set_id, + peer_id: *peer_id, + }); + }, + PeerState::NotConnected => {}, + } } } else { // Try connecting to non-reserved peers. @@ -313,10 +339,27 @@ impl ProtocolController { return } - self.peer(&peer_id) - .into_connected() - .map(|connected_peer| connected_peer.disconnect()); - self.nodes.remove(&peer_id); + match self.nodes.remove(&peer_id) { + Some(state) => match state { + PeerState::Connected(d) => { + match d { + Direction::Inbound => self.num_in -= 1, + Direction::Outbound => self.num_out -= 1, + } + let _ = self + .to_notifications + .unbounded_send(Message::Drop { set_id: self.set_id, peer_id }); + }, + PeerState::NotConnected => {}, + }, + None => { + trace!( + target: "peerset", + "Trying to remove unknown peer {} from {:?}", + peer_id, self.set_id, + ); + }, + } } fn on_incoming_connection(&mut self, peer_id: PeerId, incoming_index: IncomingIndex) { @@ -325,41 +368,70 @@ impl ProtocolController { return } - let not_connected = match self.peer(&peer_id) { + // Adding incoming peer to our set of peers even before we decide whether to accept + // it is questionable, but this is how it was implemented in the original `Peerset`. + let state = self.nodes.entry(peer_id).or_insert(PeerState::NotConnected); + match state { // If we're already connected, don't answer, as the docs mention. - Peer::Connected(_) => return, - Peer::NotConnected(peer) => peer, - // Adding incoming peer to our set of peers even before we decide whether to accept - // it is questionable, but this is how it was implemented in the original `Peerset`. - Peer::Unknown(peer) => peer.discover(), - }; - - if not_connected.is_banned() { - let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); - return - } + PeerState::Connected(_) => return, + PeerState::NotConnected => { + if self.peerset_handle.is_banned(peer_id) { + let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); + return + } - let _ = not_connected.try_accept_incoming(incoming_index); - } + let is_no_slot_occupy = self.reserved_nodes.contains(&peer_id); + // Note that it is possible for num_in to be strictly superior to the max, in case + // we were connected to a reserved node then marked it as not reserved. + if self.num_in >= self.max_in && !is_no_slot_occupy { + let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); + return + } - fn peer<'a>(&'a mut self, peer_id: &'a PeerId) -> Peer<'a> { - match self.nodes.get(peer_id) { - None => - Peer::Unknown(UnknownPeer { controller: self, peer_id: Cow::Borrowed(peer_id) }), - Some(PeerState::NotConnected) => Peer::NotConnected(NotConnectedPeer { - controller: self, - peer_id: Cow::Borrowed(peer_id), - }), - Some(PeerState::Connected(_)) => - Peer::Connected(ConnectedPeer { controller: self, peer_id: Cow::Borrowed(peer_id) }), + if !is_no_slot_occupy { + self.num_in += 1; + } + *state = PeerState::Connected(Direction::Inbound); + let _ = self.to_notifications.unbounded_send(Message::Accept(incoming_index)); + }, } } - fn connected_peers(&self) -> impl Iterator { - self.nodes - .iter() - .filter(|(_, state)| state.is_connected()) - .map(|(peer_id, _)| peer_id) + fn on_peer_dropped(&mut self, peer_id: PeerId, reason: DropReason) { + match self.nodes.entry(peer_id) { + Entry::Occupied(mut node) => match node.get_mut() { + PeerState::Connected(d) => { + if !self.reserved_nodes.contains(&peer_id) { + match d { + Direction::Inbound => self.num_in -= 1, + Direction::Outbound => self.num_out -= 1, + } + } + self.peerset_handle.report_disconnect(peer_id); + if let DropReason::Refused = reason { + node.remove(); + } else { + *node.get_mut() = PeerState::NotConnected; + } + }, + PeerState::NotConnected => { + debug_assert!(false, "Received on_peer_dropped() for non-connected peer."); + error!( + target: "peerset", + "Received on_peer_dropped() for non-connected peer {} on {:?}.", + peer_id, self.set_id, + ) + }, + }, + Entry::Vacant(_) => { + debug_assert!(false, "Received on_peer_dropped() for unknown peer."); + error!( + target: "peerset", + "Received on_peer_dropped() for unknown peer {} on {:?}.", + peer_id, self.set_id, + ) + }, + } } fn alloc_slots(&mut self) { @@ -369,207 +441,3 @@ impl ProtocolController { ); } } -/// Grants access to the state of a peer in the [`ProtocolController`]. -pub enum Peer<'a> { - /// We are connected to this node. - Connected(ConnectedPeer<'a>), - /// We are not connected to this node. - NotConnected(NotConnectedPeer<'a>), - /// We have never heard of this node, or it is not part of the set. - Unknown(UnknownPeer<'a>), -} - -impl<'a> Peer<'a> { - /// If we are the `Connected` variant, returns the inner [`ConnectedPeer`]. Returns `None` - /// otherwise. - fn into_connected(self) -> Option> { - match self { - Self::Connected(peer) => Some(peer), - Self::NotConnected(..) | Self::Unknown(..) => None, - } - } - - /// If we are the `NotConnected` variant, returns the inner [`NotConnectedPeer`]. Returns `None` - /// otherwise. - #[cfg(test)] // Feel free to remove this if this function is needed outside of tests - fn into_not_connected(self) -> Option> { - match self { - Self::NotConnected(peer) => Some(peer), - Self::Connected(..) | Self::Unknown(..) => None, - } - } - - /// If we are the `Unknown` variant, returns the inner [`UnknownPeer`]. Returns `None` - /// otherwise. - #[cfg(test)] // Feel free to remove this if this function is needed outside of tests - fn into_unknown(self) -> Option> { - match self { - Self::Unknown(peer) => Some(peer), - Self::Connected(..) | Self::NotConnected(..) => None, - } - } -} - -/// A peer that is connected to us. -pub struct ConnectedPeer<'a> { - controller: &'a mut ProtocolController, - peer_id: Cow<'a, PeerId>, -} - -impl<'a> ConnectedPeer<'a> { - /// Get the `PeerId` associated to this `ConnectedPeer`. - fn peer_id(&self) -> &PeerId { - &self.peer_id - } - - /// Destroys this `ConnectedPeer` and returns the `PeerId` inside of it. - fn into_peer_id(self) -> PeerId { - self.peer_id.into_owned() - } - - /// Switches the peer to "not connected". - fn disconnect(self) -> NotConnectedPeer<'a> { - let is_no_slot_occupy = self.controller.reserved_nodes.contains(&*self.peer_id); - if let Some(state) = self.controller.nodes.get_mut(&*self.peer_id) { - if !is_no_slot_occupy { - match state { - PeerState::Connected(Direction::Inbound) => self.controller.num_in -= 1, - PeerState::Connected(Direction::Outbound) => self.controller.num_out -= 1, - PeerState::NotConnected => { - debug_assert!( - false, - "State inconsistency: disconnecting a disconnected node" - ) - }, - } - } - *state = PeerState::NotConnected; - let _ = self.controller.to_notifications.unbounded_send(Message::Drop { - set_id: self.controller.set_id, - peer_id: *self.peer_id, - }); - } else { - debug_assert!(false, "State inconsistency: disconnecting a disconnected node"); - } - - NotConnectedPeer { controller: self.controller, peer_id: self.peer_id } - } -} - -// A peer that is not connected to us. -pub struct NotConnectedPeer<'a> { - controller: &'a mut ProtocolController, - peer_id: Cow<'a, PeerId>, -} - -impl<'a> NotConnectedPeer<'a> { - /// Destroys this `NotConnectedPeer` and returns the `PeerId` inside of it. - fn into_peer_id(self) -> PeerId { - self.peer_id.into_owned() - } - - /// Tries to set the peer as connected as an outgoing connection. - /// - /// If there are enough slots available, switches the node to "connected" and returns `Ok`. If - /// the slots are full, the node stays "not connected" and we return `Err`. - /// - /// Non-slot-occupying nodes don't count towards the number of slots. - fn try_outgoing(self) -> Result, Self> { - let is_no_slot_occupy = self.controller.reserved_nodes.contains(&*self.peer_id); - - // Note that it is possible for num_out to be strictly superior to the max, in case we were - // connected to reserved node then marked them as not reserved. - if self.controller.num_out >= self.controller.max_out && !is_no_slot_occupy { - return Err(self) - } - - if let Some(state) = self.controller.nodes.get_mut(&*self.peer_id) { - debug_assert!( - matches!(state, PeerState::NotConnected), - "State inconsistency: try_outgoing on a connected node" - ); - if !is_no_slot_occupy { - self.controller.num_out += 1; - } - *state = PeerState::Connected(Direction::Outbound); - let _ = self.controller.to_notifications.unbounded_send(Message::Connect { - set_id: self.controller.set_id, - peer_id: *self.peer_id, - }); - } else { - debug_assert!(false, "State inconsistency: try_outgoing on an unknown node"); - } - - Ok(ConnectedPeer { controller: self.controller, peer_id: self.peer_id }) - } - - /// Tries to accept the peer as an incoming connection. - /// - /// If there are enough slots available, switches the node to "connected" and returns `Ok`. If - /// the slots are full, the node stays "not connected" and we return `Err`. - /// - /// Non-slot-occupying nodes don't count towards the number of slots. - fn try_accept_incoming(self, incoming_index: IncomingIndex) -> Result, Self> { - let is_no_slot_occupy = self.controller.reserved_nodes.contains(&*self.peer_id); - - // Note that it is possible for num_in to be strictly superior to the max, in case we were - // connected to reserved node then marked them as not reserved. - if self.controller.num_in >= self.controller.max_in && !is_no_slot_occupy { - return Err(self) - } - - if let Some(state) = self.controller.nodes.get_mut(&*self.peer_id) { - debug_assert!( - matches!(state, PeerState::NotConnected), - "State inconsistency: try_accept_incoming on a connected node" - ); - if !is_no_slot_occupy { - self.controller.num_in += 1; - } - *state = PeerState::Connected(Direction::Inbound); - let _ = - self.controller.to_notifications.unbounded_send(Message::Accept(incoming_index)); - } else { - debug_assert!(false, "State inconsistency: try_accept_incoming on an unknown node"); - } - - Ok(ConnectedPeer { controller: self.controller, peer_id: self.peer_id }) - } - - /// Removes the peer from the list of members of the set. - fn forget_peer(self) -> UnknownPeer<'a> { - if self.controller.nodes.remove(&*self.peer_id).is_none() { - debug_assert!(false, "State inconsistency: forget_peer on an unknown node"); - error!( - target: "peerset", - "State inconsistency with {} when forgetting peer", - self.peer_id - ); - }; - - UnknownPeer { controller: self.controller, peer_id: self.peer_id } - } - - fn is_banned(&self) -> bool { - todo!("request this info from `PeerSet`"); - } -} - -/// A peer that we have never heard of or that isn't part of the set. -pub struct UnknownPeer<'a> { - controller: &'a mut ProtocolController, - peer_id: Cow<'a, PeerId>, -} - -impl<'a> UnknownPeer<'a> { - /// Inserts the peer identity in our list. - /// - /// The node starts with a reputation of 0. You can adjust these default - /// values using the `NotConnectedPeer` that this method returns. - fn discover(self) -> NotConnectedPeer<'a> { - if self.controller.nodes.insert(*self.peer_id, PeerState::NotConnected).is_some() { - debug_assert!(false, "State inconsistency: discovered already known peer"); - } - NotConnectedPeer { controller: self.controller, peer_id: self.peer_id } - } -} From f6b9bca1f4683064841cb62ace28d5c485774f88 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 16 Mar 2023 21:20:53 +0300 Subject: [PATCH 06/98] Apply suggestions from code review Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> --- client/peerset/src/protocol_controller.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index a5f9a81bbddce..022d127199397 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -234,6 +234,7 @@ impl ProtocolController { if !self.reserved_nodes.insert(peer_id) { return } + // Discount occupied slots or connect to the node. match self.nodes.entry(peer_id).or_insert(PeerState::NotConnected) { PeerState::Connected(Direction::Inbound) => self.num_in -= 1, From 6c6fb753f9aac5aa4be8547e3a2e1661312118dc Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 17 Mar 2023 22:09:52 +0200 Subject: [PATCH 07/98] Rework peer management + implement `alloc_slots` --- Cargo.lock | 1 + client/peerset/Cargo.toml | 1 + client/peerset/src/lib.rs | 10 + client/peerset/src/protocol_controller.rs | 347 ++++++++++++++-------- 4 files changed, 243 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9a63e25b4878c..11e10ac1fac78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9053,6 +9053,7 @@ dependencies = [ "rand 0.8.5", "sc-utils", "serde_json", + "sp-arithmetic", "wasm-timer", ] diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index a09508c831189..817f16c810270 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -20,6 +20,7 @@ log = "0.4.17" serde_json = "1.0.85" wasm-timer = "0.2" sc-utils = { version = "4.0.0-dev", path = "../utils" } +sp-arithmetic = { version = "6.0.0", path = "../../primitives/arithmetic" } [dev-dependencies] rand = "0.8.5" diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index eeada09856437..2c71d95b64868 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -187,6 +187,16 @@ impl PeersetHandle { pub fn report_disconnect(&self, peer_id: PeerId) { todo!("will be implemented after `Peerset` is converted into shared struct"); } + + /// Get the candidates for an outgoing connection. + pub fn outgoing_candidates( + &self, + count: usize, + already_connected: HashSet, + ) -> impl Iterator { + todo!("supply `count` peers with highest reputation, but not `already_connected`"); + [].into_iter() + } } /// Message that can be sent by the peer set manager (PSM). diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 022d127199397..a286b795af473 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -16,10 +16,14 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +// TODO: remove this line. +#![allow(unused)] + use futures::{FutureExt, StreamExt}; use libp2p::PeerId; use log::{error, info, trace}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; +use sp_arithmetic::traits::SaturatedConversion; use std::{ collections::{hash_map::Entry, HashMap, HashSet}, time::{Duration, Instant}, @@ -58,7 +62,7 @@ impl ProtocolHandle { let _ = self.to_controller.unbounded_send(Action::AddReservedPeer(peer_id)); } - /// Remove a previously-added reserved peer. + /// Demotes reserved peer to non-reserved. Does not disconnect the peer. /// /// Has no effect if the node was not a reserved peer. pub fn remove_reserved_peer(&self, peer_id: PeerId) { @@ -100,14 +104,14 @@ impl ProtocolHandle { } /// Direction of a connection -#[derive(Debug)] +#[derive(Clone, Copy, Debug)] enum Direction { Inbound, Outbound, } /// Status of a connection with a peer. -#[derive(Debug)] +#[derive(Clone, Debug)] enum PeerState { /// We are connected to the peer. Connected(Direction), @@ -145,12 +149,12 @@ pub struct ProtocolController { max_in: u32, /// Maximum number of slots for outgoing connections. max_out: u32, - /// Reserved nodes. - reserved_nodes: HashSet, - /// Connect only to reserved nodes. - reserved_only: bool, /// Peers and their connection states (including reserved nodes). nodes: HashMap, + /// Reserved nodes. Should be always connected and do not occupy peer slots. + reserved_nodes: HashMap, + /// Connect only to reserved nodes. + reserved_only: bool, /// Next time to allocate slots. This is done once per second. next_periodic_alloc_slots: Instant, /// Outgoing channel for messages to `Notifications`. @@ -170,9 +174,17 @@ impl ProtocolController { ) -> (ProtocolHandle, ProtocolController) { let (to_controller, from_handle) = tracing_unbounded("mpsc_protocol_controller", 10_000); let handle = ProtocolHandle { to_controller }; - let nodes = config + let reserved_nodes = config .reserved_nodes - .union(&config.bootnodes.into_iter().collect()) + .iter() + .map(|p| (*p, PeerState::NotConnected)) + .collect::>(); + // Initialize with bootnodes, but make sure we don't count peers twice. + let nodes = config + .bootnodes + .into_iter() + .collect::>() + .difference(&config.reserved_nodes) .map(|p| (*p, PeerState::NotConnected)) .collect::>(); let controller = ProtocolController { @@ -182,9 +194,9 @@ impl ProtocolController { num_out: 0, max_in: config.in_peers, max_out: config.out_peers, - reserved_nodes: config.reserved_nodes, - reserved_only: config.reserved_only, nodes, + reserved_nodes, + reserved_only: config.reserved_only, next_periodic_alloc_slots: Instant::now(), to_notifications, peerset_handle, @@ -225,18 +237,58 @@ impl ProtocolController { Action::RemovePeer(peer_id) => self.on_remove_peer(peer_id), Action::IncomingConnection(peer_id, index) => self.on_incoming_connection(peer_id, index), - Action::Dropped(peer_id, reason) => self.on_peer_dropped(peer_id, reason), + Action::Dropped(peer_id, reason) => + self.on_peer_dropped(peer_id, reason).unwrap_or_else(|peer_id| { + debug_assert!(false, "Received Action::Dropped for non-connected peer."); + error!( + target: "peerset", + "Received Action::Dropped for non-connected peer {} on {:?}.", + peer_id, self.set_id, + ) + }), } true } + fn accept_connection(&self, incoming_index: IncomingIndex) { + let _ = self.to_notifications.unbounded_send(Message::Accept(incoming_index)); + } + + fn reject_connection(&self, incoming_index: IncomingIndex) { + let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); + } + + fn start_connection(&self, peer_id: PeerId) { + let _ = self + .to_notifications + .unbounded_send(Message::Connect { set_id: self.set_id, peer_id }); + } + + fn drop_connection(&self, peer_id: PeerId) { + let _ = self + .to_notifications + .unbounded_send(Message::Drop { set_id: self.set_id, peer_id }); + } + + fn report_disconnect(&self, peer_id: PeerId) { + self.peerset_handle.report_disconnect(peer_id); + } + + fn is_banned(&self, peer_id: PeerId) -> bool { + self.peerset_handle.is_banned(peer_id) + } + fn on_add_reserved_peer(&mut self, peer_id: PeerId) { - if !self.reserved_nodes.insert(peer_id) { + if self.reserved_nodes.contains_key(&peer_id) { return } + // Get the peer out of non-reserved peers if it's there and add to reserved. + let state = self.nodes.remove(&peer_id).unwrap_or(PeerState::NotConnected); + self.reserved_nodes.insert(peer_id, state.clone()); + // Discount occupied slots or connect to the node. - match self.nodes.entry(peer_id).or_insert(PeerState::NotConnected) { + match state { PeerState::Connected(Direction::Inbound) => self.num_in -= 1, PeerState::Connected(Direction::Outbound) => self.num_out -= 1, PeerState::NotConnected => self.alloc_slots(), @@ -244,53 +296,43 @@ impl ProtocolController { } fn on_remove_reserved_peer(&mut self, peer_id: PeerId) { - if !self.reserved_nodes.remove(&peer_id) { - return - } - - match self.nodes.entry(peer_id) { - Entry::Occupied(mut node) => { - if !node.get().is_connected() { - return - } + let mut state = match self.reserved_nodes.remove(&peer_id) { + Some(state) => state, + None => return, + }; + match state { + PeerState::Connected(d) => { if self.reserved_only { - // If we are in reserved-only mode, the removed reserved node - // should be disconnected. - *node.get_mut() = PeerState::NotConnected; - let _ = self.to_notifications.unbounded_send(Message::Drop { - set_id: self.set_id, - peer_id: *node.key(), - }); + // Disconnect the node. info!( target: "peerset", "Disconnecting previously reserved node {} on {:?}.", - node.key(), self.set_id + peer_id, self.set_id ); + state = PeerState::NotConnected; + self.drop_connection(peer_id); } else { - // Otherwise, just count occupied slots for the node. - match node.get() { - PeerState::Connected(Direction::Inbound) => self.num_in += 1, - PeerState::Connected(Direction::Outbound) => self.num_out += 1, - PeerState::NotConnected => {}, + // Count connections as of regular node. + match d { + Direction::Inbound => self.num_in += 1, + Direction::Outbound => self.num_out += 1, } } }, - Entry::Vacant(_) => { - debug_assert!(false, "Reserved node not in the list of known nodes."); - error!( - target: "peerset", - "Invalid state: reserved node {} not in the list of known nodes.", - peer_id, - ); - }, + PeerState::NotConnected => {}, } + + // Put the node into the list of regular nodes. + let prev = self.nodes.insert(peer_id, state); + assert!(prev.is_none(), "Corrupted state: reserved node was also non-reserved."); } fn on_set_reserved_peers(&mut self, peer_ids: HashSet) { // Determine the difference between the current group and the new list. - let to_insert = peer_ids.difference(&self.reserved_nodes).cloned().collect::>(); - let to_remove = self.reserved_nodes.difference(&peer_ids).cloned().collect::>(); + let current = self.reserved_nodes.keys().cloned().collect(); + let to_insert = peer_ids.difference(¤t).cloned().collect::>(); + let to_remove = current.difference(&peer_ids).cloned().collect::>(); for node in to_insert { self.on_add_reserved_peer(node); @@ -304,31 +346,29 @@ impl ProtocolController { fn on_set_reserved_only(&mut self, reserved_only: bool) { self.reserved_only = reserved_only; - if reserved_only { - // Disconnect all non-reserved peers. - for (peer_id, state) in self.nodes.iter_mut() { - match state { - PeerState::Connected(d) => { - match d { - Direction::Inbound => self.num_in -= 1, - Direction::Outbound => self.num_out -= 1, - } - *state = PeerState::NotConnected; - let _ = self.to_notifications.unbounded_send(Message::Drop { - set_id: self.set_id, - peer_id: *peer_id, - }); - }, - PeerState::NotConnected => {}, - } - } - } else { - // Try connecting to non-reserved peers. - self.alloc_slots(); + if !reserved_only { + return self.alloc_slots() } + + // Disconnect all non-reserved peers. + self.nodes + .iter_mut() + .filter_map(|(peer_id, state)| { + state.is_connected().then_some({ + *state = PeerState::NotConnected; + *peer_id + }) + }) + .collect::>() + .iter() + .for_each(|peer_id| self.drop_connection(*peer_id)); } fn on_add_peer(&mut self, peer_id: PeerId) { + if self.reserved_nodes.contains_key(&peer_id) { + return + } + if self.nodes.insert(peer_id, PeerState::NotConnected).is_none() { self.alloc_slots(); } @@ -336,7 +376,7 @@ impl ProtocolController { fn on_remove_peer(&mut self, peer_id: PeerId) { // Don't do anything if the node is reserved. - if self.reserved_nodes.contains(&peer_id) { + if self.reserved_nodes.contains_key(&peer_id) { return } @@ -347,9 +387,7 @@ impl ProtocolController { Direction::Inbound => self.num_in -= 1, Direction::Outbound => self.num_out -= 1, } - let _ = self - .to_notifications - .unbounded_send(Message::Drop { set_id: self.set_id, peer_id }); + self.drop_connection(peer_id); }, PeerState::NotConnected => {}, }, @@ -364,8 +402,27 @@ impl ProtocolController { } fn on_incoming_connection(&mut self, peer_id: PeerId, incoming_index: IncomingIndex) { - if self.reserved_only && !self.reserved_nodes.contains(&peer_id) { - let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); + if self.reserved_only && !self.reserved_nodes.contains_key(&peer_id) { + self.reject_connection(incoming_index); + return + } + + // Check if the node is reserved first. + if let Some(state) = self.reserved_nodes.get_mut(&peer_id) { + match state { + // If we're already connected, don't answer, as the docs mention. + PeerState::Connected(_) => {}, + PeerState::NotConnected => { + // It's questionable whether we should check a reputation of reserved node. + // FIXME: unable to call `self.is_banned()` because of borrowed `self`. + if self.peerset_handle.is_banned(peer_id) { + self.reject_connection(incoming_index); + } else { + *state = PeerState::Connected(Direction::Inbound); + self.accept_connection(incoming_index); + } + }, + } return } @@ -376,69 +433,127 @@ impl ProtocolController { // If we're already connected, don't answer, as the docs mention. PeerState::Connected(_) => return, PeerState::NotConnected => { + // FIXME: unable to call `self.is_banned()` because of borrowed `self`. if self.peerset_handle.is_banned(peer_id) { - let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); + self.reject_connection(incoming_index); return } - let is_no_slot_occupy = self.reserved_nodes.contains(&peer_id); - // Note that it is possible for num_in to be strictly superior to the max, in case - // we were connected to a reserved node then marked it as not reserved. - if self.num_in >= self.max_in && !is_no_slot_occupy { - let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); + if self.num_in >= self.max_in { + self.reject_connection(incoming_index); return } - if !is_no_slot_occupy { - self.num_in += 1; - } + self.num_in += 1; *state = PeerState::Connected(Direction::Inbound); - let _ = self.to_notifications.unbounded_send(Message::Accept(incoming_index)); + self.accept_connection(incoming_index); }, } } - fn on_peer_dropped(&mut self, peer_id: PeerId, reason: DropReason) { - match self.nodes.entry(peer_id) { - Entry::Occupied(mut node) => match node.get_mut() { - PeerState::Connected(d) => { - if !self.reserved_nodes.contains(&peer_id) { - match d { - Direction::Inbound => self.num_in -= 1, - Direction::Outbound => self.num_out -= 1, - } + /// Returns Err(PeerId) if peer wasn't connected. + fn on_peer_dropped(&mut self, peer_id: PeerId, reason: DropReason) -> Result<(), PeerId> { + let mut is_reserved = true; + let state = self + .reserved_nodes + .get_mut(&peer_id) + .or_else(|| { + is_reserved = false; + self.nodes.get_mut(&peer_id) + }) + .ok_or(peer_id)?; + match state { + PeerState::Connected(d) => { + if is_reserved { + *state = PeerState::NotConnected; + } else { + match d { + Direction::Inbound => self.num_in -= 1, + Direction::Outbound => self.num_out -= 1, } - self.peerset_handle.report_disconnect(peer_id); if let DropReason::Refused = reason { - node.remove(); + self.nodes.remove(&peer_id); } else { - *node.get_mut() = PeerState::NotConnected; + *state = PeerState::NotConnected; } - }, - PeerState::NotConnected => { - debug_assert!(false, "Received on_peer_dropped() for non-connected peer."); - error!( - target: "peerset", - "Received on_peer_dropped() for non-connected peer {} on {:?}.", - peer_id, self.set_id, - ) - }, - }, - Entry::Vacant(_) => { - debug_assert!(false, "Received on_peer_dropped() for unknown peer."); - error!( - target: "peerset", - "Received on_peer_dropped() for unknown peer {} on {:?}.", - peer_id, self.set_id, - ) + } + self.report_disconnect(peer_id); + Ok(()) }, + PeerState::NotConnected => Err(peer_id), } } fn alloc_slots(&mut self) { - todo!( - "Count the number of slots we need to fill and request that number of peers from `Peerset`, - also passing the list of already connected peers to eliminate them from the list." - ); + if self.num_out >= self.max_out { + return + } + + // Try connecting to reserved nodes first. + self.reserved_nodes + .iter_mut() + .filter_map(|(peer_id, state)| { + (matches!(state, PeerState::NotConnected) && + !self.peerset_handle.is_banned(*peer_id)) + .then_some({ + *state = PeerState::Connected(Direction::Outbound); + peer_id + }) + }) + .cloned() + .collect::>() + .into_iter() + .map(|peer_id| { + self.start_connection(peer_id); + }); + + // Nothing more to do if we're in reserved-only mode. + if self.reserved_only { + return + } + + // Fill available slots with non-reserved nodes + let available_slots = (self.max_out - self.num_out).saturated_into(); + // Ignore reserved nodes (connected above) and already connected nodes. + let ignored = self + .reserved_nodes + .keys() + .cloned() + .collect::>() + .union( + &self + .nodes + .iter() + .filter_map(|(peer_id, state)| state.is_connected().then_some(*peer_id)) + .collect::>(), + ) + .cloned() + .collect::>(); + + self.peerset_handle + .outgoing_candidates(available_slots, ignored) + .filter_map(|peer_id| { + let state = self.nodes.entry(*peer_id).or_insert(PeerState::NotConnected); + match state { + PeerState::Connected(_) => { + debug_assert!(false, "`Peerset` returned a node we asked to ignore."); + error!( + target: "peerset", + "`Peerset` returned a node we asked to ignore: {}.", + peer_id + ); + None + }, + PeerState::NotConnected => { + self.num_out += 1; + *state = PeerState::Connected(Direction::Outbound); + Some(peer_id) + }, + } + }) + .cloned() + .collect::>() + .iter() + .map(|peer_id| self.start_connection(*peer_id)); } } From 7d7d1d884d3eddcb84c465fb84d04f8a7a598ade Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 17 Mar 2023 22:25:35 +0200 Subject: [PATCH 08/98] minor: naming --- client/peerset/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 2c71d95b64868..91666251dd42d 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -192,9 +192,9 @@ impl PeersetHandle { pub fn outgoing_candidates( &self, count: usize, - already_connected: HashSet, + ignored: HashSet, ) -> impl Iterator { - todo!("supply `count` peers with highest reputation, but not `already_connected`"); + todo!("supply `count` peers with highest reputation, but not from `ignored`"); [].into_iter() } } From de8e6d2dd04ead381cd2839457344dfac934896d Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 20 Mar 2023 12:13:58 +0300 Subject: [PATCH 09/98] Apply suggestions from code review Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> --- client/peerset/src/protocol_controller.rs | 39 +++++++++++------------ 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index a286b795af473..9a6fdafa3fb2b 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -301,27 +301,24 @@ impl ProtocolController { None => return, }; - match state { - PeerState::Connected(d) => { - if self.reserved_only { - // Disconnect the node. - info!( - target: "peerset", - "Disconnecting previously reserved node {} on {:?}.", - peer_id, self.set_id - ); - state = PeerState::NotConnected; - self.drop_connection(peer_id); - } else { - // Count connections as of regular node. - match d { - Direction::Inbound => self.num_in += 1, - Direction::Outbound => self.num_out += 1, - } - } - }, - PeerState::NotConnected => {}, - } + if let PeerState::Connected(d) = state { + if self.reserved_only { + // Disconnect the node. + info!( + target: "peerset", + "Disconnecting previously reserved node {} on {:?}.", + peer_id, self.set_id + ); + state = PeerState::NotConnected; + self.drop_connection(peer_id); + } else { + // Count connections as of regular node. + match d { + Direction::Inbound => self.num_in += 1, + Direction::Outbound => self.num_out += 1, + } + } + } // Put the node into the list of regular nodes. let prev = self.nodes.insert(peer_id, state); From 4ab043d04e05373e07c795e815984291247b4547 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 20 Mar 2023 11:27:40 +0200 Subject: [PATCH 10/98] Apply review suggestions --- client/peerset/src/protocol_controller.rs | 49 +++++++++++------------ 1 file changed, 23 insertions(+), 26 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 9a6fdafa3fb2b..f4e0c5bb29312 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -174,11 +174,8 @@ impl ProtocolController { ) -> (ProtocolHandle, ProtocolController) { let (to_controller, from_handle) = tracing_unbounded("mpsc_protocol_controller", 10_000); let handle = ProtocolHandle { to_controller }; - let reserved_nodes = config - .reserved_nodes - .iter() - .map(|p| (*p, PeerState::NotConnected)) - .collect::>(); + let reserved_nodes = + config.reserved_nodes.iter().map(|p| (*p, PeerState::NotConnected)).collect(); // Initialize with bootnodes, but make sure we don't count peers twice. let nodes = config .bootnodes @@ -186,7 +183,7 @@ impl ProtocolController { .collect::>() .difference(&config.reserved_nodes) .map(|p| (*p, PeerState::NotConnected)) - .collect::>(); + .collect(); let controller = ProtocolController { set_id, from_handle, @@ -301,24 +298,24 @@ impl ProtocolController { None => return, }; - if let PeerState::Connected(d) = state { - if self.reserved_only { - // Disconnect the node. - info!( - target: "peerset", - "Disconnecting previously reserved node {} on {:?}.", - peer_id, self.set_id - ); - state = PeerState::NotConnected; - self.drop_connection(peer_id); - } else { - // Count connections as of regular node. - match d { - Direction::Inbound => self.num_in += 1, - Direction::Outbound => self.num_out += 1, - } - } - } + if let PeerState::Connected(d) = state { + if self.reserved_only { + // Disconnect the node. + info!( + target: "peerset", + "Disconnecting previously reserved node {} on {:?}.", + peer_id, self.set_id + ); + state = PeerState::NotConnected; + self.drop_connection(peer_id); + } else { + // Count connections as of regular node. + match d { + Direction::Inbound => self.num_in += 1, + Direction::Outbound => self.num_out += 1, + } + } + } // Put the node into the list of regular nodes. let prev = self.nodes.insert(peer_id, state); @@ -500,7 +497,7 @@ impl ProtocolController { .cloned() .collect::>() .into_iter() - .map(|peer_id| { + .for_each(|peer_id| { self.start_connection(peer_id); }); @@ -551,6 +548,6 @@ impl ProtocolController { .cloned() .collect::>() .iter() - .map(|peer_id| self.start_connection(*peer_id)); + .for_each(|peer_id| self.start_connection(*peer_id)); } } From 770a38a03751d8d3cd9774dc419d30fca6897067 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 20 Mar 2023 13:24:35 +0200 Subject: [PATCH 11/98] Refactor `on_peer_dropped()` --- client/peerset/src/protocol_controller.rs | 75 ++++++++++++++--------- 1 file changed, 46 insertions(+), 29 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index f4e0c5bb29312..a580323c77cf0 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -445,36 +445,53 @@ impl ProtocolController { } } - /// Returns Err(PeerId) if peer wasn't connected. + /// Returns `Err(PeerId)` if the peer wasn't connected or not found. fn on_peer_dropped(&mut self, peer_id: PeerId, reason: DropReason) -> Result<(), PeerId> { - let mut is_reserved = true; - let state = self - .reserved_nodes - .get_mut(&peer_id) - .or_else(|| { - is_reserved = false; - self.nodes.get_mut(&peer_id) - }) - .ok_or(peer_id)?; - match state { - PeerState::Connected(d) => { - if is_reserved { - *state = PeerState::NotConnected; - } else { - match d { - Direction::Inbound => self.num_in -= 1, - Direction::Outbound => self.num_out -= 1, - } - if let DropReason::Refused = reason { - self.nodes.remove(&peer_id); - } else { - *state = PeerState::NotConnected; - } - } - self.report_disconnect(peer_id); - Ok(()) - }, - PeerState::NotConnected => Err(peer_id), + if self.drop_reserved_peer(&peer_id)? || self.drop_regular_peer(&peer_id, reason)? { + // The peer found and disconnected. + self.report_disconnect(peer_id); + Ok(()) + } else { + // The peer was not found in neither regular or reserved lists. + Err(peer_id) + } + } + + /// Try dropping as a reserved peer. Return `Ok(true)` if the peer was found and disconnected, + /// `Ok(false)` if it wasn't found, `Err(())`, if the peer found, but not in connected state. + fn drop_reserved_peer(&mut self, peer_id: &PeerId) -> Result { + let Some(state) = self.reserved_nodes.get_mut(peer_id) else { + return Ok(false) + }; + + if let PeerState::Connected(_) = state { + *state = PeerState::NotConnected; + Ok(true) + } else { + Err(peer_id.clone()) + } + } + + /// Try dropping as a regular peer. Return `Ok(true)` if the peer was found and disconnected, + /// `Ok(false)` if it wasn't found, `Err(())`, if the peer found, but not in connected state. + fn drop_regular_peer(&mut self, peer_id: &PeerId, reason: DropReason) -> Result { + let Some(state) = self.nodes.get_mut(peer_id) else { + return Ok(false) + }; + + if let PeerState::Connected(d) = state { + match d { + Direction::Inbound => self.num_in -= 1, + Direction::Outbound => self.num_out -= 1, + } + if let DropReason::Refused = reason { + self.nodes.remove(&peer_id); + } else { + *state = PeerState::NotConnected; + } + Ok(true) + } else { + Err(peer_id.clone()) } } From f4401cf5d6673f619265bf45d0641cbe548bdeb8 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 20 Mar 2023 13:26:24 +0200 Subject: [PATCH 12/98] minor: fix docs --- client/peerset/src/protocol_controller.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index a580323c77cf0..4159b5eef3cba 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -458,7 +458,8 @@ impl ProtocolController { } /// Try dropping as a reserved peer. Return `Ok(true)` if the peer was found and disconnected, - /// `Ok(false)` if it wasn't found, `Err(())`, if the peer found, but not in connected state. + /// `Ok(false)` if it wasn't found, `Err(PeerId)`, if the peer found, but not in connected + /// state. fn drop_reserved_peer(&mut self, peer_id: &PeerId) -> Result { let Some(state) = self.reserved_nodes.get_mut(peer_id) else { return Ok(false) @@ -473,7 +474,8 @@ impl ProtocolController { } /// Try dropping as a regular peer. Return `Ok(true)` if the peer was found and disconnected, - /// `Ok(false)` if it wasn't found, `Err(())`, if the peer found, but not in connected state. + /// `Ok(false)` if it wasn't found, `Err(PeerId)`, if the peer found, but not in connected + /// state. fn drop_regular_peer(&mut self, peer_id: &PeerId, reason: DropReason) -> Result { let Some(state) = self.nodes.get_mut(peer_id) else { return Ok(false) From 91a66ec7acdf42fa8c7254e4e98f2713fe9f7e0d Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 20 Mar 2023 15:36:26 +0200 Subject: [PATCH 13/98] Add more docs --- client/peerset/src/protocol_controller.rs | 46 +++++++++++++++++++---- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 4159b5eef3cba..c39eb4e087231 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -16,6 +16,11 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . +//! Protocol Controller. Generic implementation of peer management for protocols. +//! Responsible for accepting/rejecting incoming connections and initiating outgoing connections, +//! respecting the inbound and outbound peer slot counts. Communicates with `Peerset` to get and +//! update peer reputation values and sends commands to `Notifications`. + // TODO: remove this line. #![allow(unused)] @@ -247,34 +252,42 @@ impl ProtocolController { true } + /// Send "accept" message to `Notifications`. fn accept_connection(&self, incoming_index: IncomingIndex) { let _ = self.to_notifications.unbounded_send(Message::Accept(incoming_index)); } + /// Send "reject" message to `Notifications`. fn reject_connection(&self, incoming_index: IncomingIndex) { let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); } + /// Send "connect" message to `Notifications`. fn start_connection(&self, peer_id: PeerId) { let _ = self .to_notifications .unbounded_send(Message::Connect { set_id: self.set_id, peer_id }); } + /// Send "drop" message to `Notifications`. fn drop_connection(&self, peer_id: PeerId) { let _ = self .to_notifications .unbounded_send(Message::Drop { set_id: self.set_id, peer_id }); } + /// Report peer disconnect event to `Peerset` for it to update peer's reputation accordingly. fn report_disconnect(&self, peer_id: PeerId) { self.peerset_handle.report_disconnect(peer_id); } + /// Ask `Peerset` if the peer has a reputation value not sufficent for connection with it. fn is_banned(&self, peer_id: PeerId) -> bool { self.peerset_handle.is_banned(peer_id) } + /// Add a peer to the set of reserved peers. [`ProtocolCOntroller`] will try to always maintain + /// connections with such peers. fn on_add_reserved_peer(&mut self, peer_id: PeerId) { if self.reserved_nodes.contains_key(&peer_id) { return @@ -292,6 +305,7 @@ impl ProtocolController { } } + /// Remove a peer from the set of reserved peers. The peer is moved to the set of regular nodes. fn on_remove_reserved_peer(&mut self, peer_id: PeerId) { let mut state = match self.reserved_nodes.remove(&peer_id) { Some(state) => state, @@ -322,6 +336,7 @@ impl ProtocolController { assert!(prev.is_none(), "Corrupted state: reserved node was also non-reserved."); } + /// Replace the set of reserved peers. fn on_set_reserved_peers(&mut self, peer_ids: HashSet) { // Determine the difference between the current group and the new list. let current = self.reserved_nodes.keys().cloned().collect(); @@ -337,6 +352,8 @@ impl ProtocolController { } } + /// Change "reserved only" flag. In "reserved only" mode we connect and accept connections to + /// reserved nodes only. fn on_set_reserved_only(&mut self, reserved_only: bool) { self.reserved_only = reserved_only; @@ -358,6 +375,8 @@ impl ProtocolController { .for_each(|peer_id| self.drop_connection(*peer_id)); } + /// Add a peer to the set of known peers. [`ProtocolController`] will try to connect to the + /// peer. fn on_add_peer(&mut self, peer_id: PeerId) { if self.reserved_nodes.contains_key(&peer_id) { return @@ -368,6 +387,7 @@ impl ProtocolController { } } + /// Remove a peer from the set of known peers. No-op if the peer is reserved. fn on_remove_peer(&mut self, peer_id: PeerId) { // Don't do anything if the node is reserved. if self.reserved_nodes.contains_key(&peer_id) { @@ -395,6 +415,15 @@ impl ProtocolController { } } + /// Indicate that we received an incoming connection. Must be answered either with + /// a corresponding `Accept` or `Reject`, except if we were already connected to this peer. + /// + /// Note that this mechanism is orthogonal to `Connect`/`Drop`. Accepting an incoming + /// connection implicitly means `Connect`, but incoming connections aren't cancelled by + /// `dropped`. + // Implementation note: because of concurrency issues, it is possible that we push a `Connect` + // message to the output channel with a `PeerId`, and that `incoming` gets called with the same + // `PeerId` before that message has been read by the user. In this situation we must not answer. fn on_incoming_connection(&mut self, peer_id: PeerId, incoming_index: IncomingIndex) { if self.reserved_only && !self.reserved_nodes.contains_key(&peer_id) { self.reject_connection(incoming_index); @@ -445,7 +474,8 @@ impl ProtocolController { } } - /// Returns `Err(PeerId)` if the peer wasn't connected or not found. + /// Indicate that a connection with the peer was dropped. + /// Returns `Err(PeerId)` if the peer wasn't connected or is not known to us. fn on_peer_dropped(&mut self, peer_id: PeerId, reason: DropReason) -> Result<(), PeerId> { if self.drop_reserved_peer(&peer_id)? || self.drop_regular_peer(&peer_id, reason)? { // The peer found and disconnected. @@ -457,9 +487,9 @@ impl ProtocolController { } } - /// Try dropping as a reserved peer. Return `Ok(true)` if the peer was found and disconnected, - /// `Ok(false)` if it wasn't found, `Err(PeerId)`, if the peer found, but not in connected - /// state. + /// Try dropping a peer as a reserved peer. Return `Ok(true)` if the peer was found and + /// disconnected, `Ok(false)` if it wasn't found, `Err(PeerId)`, if the peer found, but not in + /// connected state. fn drop_reserved_peer(&mut self, peer_id: &PeerId) -> Result { let Some(state) = self.reserved_nodes.get_mut(peer_id) else { return Ok(false) @@ -473,9 +503,9 @@ impl ProtocolController { } } - /// Try dropping as a regular peer. Return `Ok(true)` if the peer was found and disconnected, - /// `Ok(false)` if it wasn't found, `Err(PeerId)`, if the peer found, but not in connected - /// state. + /// Try dropping a peer as a regular peer. Return `Ok(true)` if the peer was found and + /// disconnected, `Ok(false)` if it wasn't found, `Err(PeerId)`, if the peer found, but not in + /// connected state. fn drop_regular_peer(&mut self, peer_id: &PeerId, reason: DropReason) -> Result { let Some(state) = self.nodes.get_mut(peer_id) else { return Ok(false) @@ -497,6 +527,8 @@ impl ProtocolController { } } + /// Initiate outgoing connections trying to connect all reserved nodes and fill in all outgoing + /// slots. fn alloc_slots(&mut self) { if self.num_out >= self.max_out { return From fd439638ea8902e713d362913b0f1c10b6d29256 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 20 Mar 2023 16:57:30 +0200 Subject: [PATCH 14/98] Don't put connected peers into the list of regular nodes --- client/peerset/src/lib.rs | 2 +- client/peerset/src/protocol_controller.rs | 208 ++++++++-------------- 2 files changed, 75 insertions(+), 135 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 91666251dd42d..497f9c30e5468 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -184,7 +184,7 @@ impl PeersetHandle { } /// Report disconnect to adjust peers reputation value. - pub fn report_disconnect(&self, peer_id: PeerId) { + pub fn report_disconnect(&self, peer_id: PeerId, reason: DropReason) { todo!("will be implemented after `Peerset` is converted into shared struct"); } diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index c39eb4e087231..094d7a4df191c 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -43,8 +43,7 @@ enum Action { RemoveReservedPeer(PeerId), SetReservedPeers(HashSet), SetReservedOnly(bool), - AddPeer(PeerId), - RemovePeer(PeerId), + DisconnectPeer(PeerId), IncomingConnection(PeerId, IncomingIndex), Dropped(PeerId, DropReason), } @@ -85,14 +84,10 @@ impl ProtocolHandle { let _ = self.to_controller.unbounded_send(Action::SetReservedOnly(reserved)); } - /// Add a peer to a set of peers we try to connect to. - pub fn add_peer(&self, peer_id: PeerId) { - let _ = self.to_controller.unbounded_send(Action::AddPeer(peer_id)); - } - - /// Remove a peer from a set of peers we try to connect to and disconnect the peer. - pub fn remove_peer(&self, peer_id: PeerId) { - let _ = self.to_controller.unbounded_send(Action::RemovePeer(peer_id)); + /// Disconnect peer. You should remove the `PeerId` from the `Peerset` first + /// to not connect to the peer again during the next slot allocation. + pub fn disconnect_peer(&self, peer_id: PeerId) { + let _ = self.to_controller.unbounded_send(Action::DisconnectPeer(peer_id)); } /// Notify about incoming connection. [`ProtocolController`] will either accept or reject it. @@ -154,8 +149,8 @@ pub struct ProtocolController { max_in: u32, /// Maximum number of slots for outgoing connections. max_out: u32, - /// Peers and their connection states (including reserved nodes). - nodes: HashMap, + /// Connected regular nodes. + nodes: HashMap, /// Reserved nodes. Should be always connected and do not occupy peer slots. reserved_nodes: HashMap, /// Connect only to reserved nodes. @@ -181,14 +176,6 @@ impl ProtocolController { let handle = ProtocolHandle { to_controller }; let reserved_nodes = config.reserved_nodes.iter().map(|p| (*p, PeerState::NotConnected)).collect(); - // Initialize with bootnodes, but make sure we don't count peers twice. - let nodes = config - .bootnodes - .into_iter() - .collect::>() - .difference(&config.reserved_nodes) - .map(|p| (*p, PeerState::NotConnected)) - .collect(); let controller = ProtocolController { set_id, from_handle, @@ -196,7 +183,7 @@ impl ProtocolController { num_out: 0, max_in: config.in_peers, max_out: config.out_peers, - nodes, + nodes: HashMap::new(), reserved_nodes, reserved_only: config.reserved_only, next_periodic_alloc_slots: Instant::now(), @@ -235,8 +222,7 @@ impl ProtocolController { Action::RemoveReservedPeer(peer_id) => self.on_remove_reserved_peer(peer_id), Action::SetReservedPeers(peer_ids) => self.on_set_reserved_peers(peer_ids), Action::SetReservedOnly(reserved_only) => self.on_set_reserved_only(reserved_only), - Action::AddPeer(peer_id) => self.on_add_peer(peer_id), - Action::RemovePeer(peer_id) => self.on_remove_peer(peer_id), + Action::DisconnectPeer(peer_id) => self.on_disconnect_peer(peer_id), Action::IncomingConnection(peer_id, index) => self.on_incoming_connection(peer_id, index), Action::Dropped(peer_id, reason) => @@ -277,8 +263,8 @@ impl ProtocolController { } /// Report peer disconnect event to `Peerset` for it to update peer's reputation accordingly. - fn report_disconnect(&self, peer_id: PeerId) { - self.peerset_handle.report_disconnect(peer_id); + fn report_disconnect(&self, peer_id: PeerId, reason: DropReason) { + self.peerset_handle.report_disconnect(peer_id, reason); } /// Ask `Peerset` if the peer has a reputation value not sufficent for connection with it. @@ -293,8 +279,12 @@ impl ProtocolController { return } - // Get the peer out of non-reserved peers if it's there and add to reserved. - let state = self.nodes.remove(&peer_id).unwrap_or(PeerState::NotConnected); + // Get the peer out of non-reserved peers if it's there. + let state = match self.nodes.remove(&peer_id) { + Some(d) => PeerState::Connected(d), + None => PeerState::NotConnected, + }; + self.reserved_nodes.insert(peer_id, state.clone()); // Discount occupied slots or connect to the node. @@ -312,7 +302,7 @@ impl ProtocolController { None => return, }; - if let PeerState::Connected(d) = state { + if let PeerState::Connected(direction) = state { if self.reserved_only { // Disconnect the node. info!( @@ -320,20 +310,18 @@ impl ProtocolController { "Disconnecting previously reserved node {} on {:?}.", peer_id, self.set_id ); - state = PeerState::NotConnected; self.drop_connection(peer_id); } else { // Count connections as of regular node. - match d { + match direction { Direction::Inbound => self.num_in += 1, Direction::Outbound => self.num_out += 1, } } + // Put the node into the list of regular nodes. + let prev = self.nodes.insert(peer_id, direction); + assert!(prev.is_none(), "Corrupted state: reserved node was also non-reserved."); } - - // Put the node into the list of regular nodes. - let prev = self.nodes.insert(peer_id, state); - assert!(prev.is_none(), "Corrupted state: reserved node was also non-reserved."); } /// Replace the set of reserved peers. @@ -362,48 +350,23 @@ impl ProtocolController { } // Disconnect all non-reserved peers. - self.nodes - .iter_mut() - .filter_map(|(peer_id, state)| { - state.is_connected().then_some({ - *state = PeerState::NotConnected; - *peer_id - }) - }) - .collect::>() - .iter() - .for_each(|peer_id| self.drop_connection(*peer_id)); - } - - /// Add a peer to the set of known peers. [`ProtocolController`] will try to connect to the - /// peer. - fn on_add_peer(&mut self, peer_id: PeerId) { - if self.reserved_nodes.contains_key(&peer_id) { - return - } - - if self.nodes.insert(peer_id, PeerState::NotConnected).is_none() { - self.alloc_slots(); - } + self.nodes.keys().for_each(|peer_id| self.drop_connection(*peer_id)); } - /// Remove a peer from the set of known peers. No-op if the peer is reserved. - fn on_remove_peer(&mut self, peer_id: PeerId) { + /// Disconnect the peer. + fn on_disconnect_peer(&mut self, peer_id: PeerId) { // Don't do anything if the node is reserved. if self.reserved_nodes.contains_key(&peer_id) { return } match self.nodes.remove(&peer_id) { - Some(state) => match state { - PeerState::Connected(d) => { - match d { - Direction::Inbound => self.num_in -= 1, - Direction::Outbound => self.num_out -= 1, - } - self.drop_connection(peer_id); - }, - PeerState::NotConnected => {}, + Some(direction) => { + match direction { + Direction::Inbound => self.num_in -= 1, + Direction::Outbound => self.num_out -= 1, + } + self.drop_connection(peer_id); }, None => { trace!( @@ -449,37 +412,32 @@ impl ProtocolController { return } - // Adding incoming peer to our set of peers even before we decide whether to accept - // it is questionable, but this is how it was implemented in the original `Peerset`. - let state = self.nodes.entry(peer_id).or_insert(PeerState::NotConnected); - match state { - // If we're already connected, don't answer, as the docs mention. - PeerState::Connected(_) => return, - PeerState::NotConnected => { - // FIXME: unable to call `self.is_banned()` because of borrowed `self`. - if self.peerset_handle.is_banned(peer_id) { - self.reject_connection(incoming_index); - return - } + // If we're already connected, don't answer, as the docs mention. + if self.nodes.contains_key(&peer_id) { + return + } - if self.num_in >= self.max_in { - self.reject_connection(incoming_index); - return - } + if self.is_banned(peer_id) { + self.reject_connection(incoming_index); + return + } - self.num_in += 1; - *state = PeerState::Connected(Direction::Inbound); - self.accept_connection(incoming_index); - }, + if self.num_in >= self.max_in { + self.reject_connection(incoming_index); + return } + + self.num_in += 1; + self.nodes.insert(peer_id, Direction::Inbound); + self.accept_connection(incoming_index); } /// Indicate that a connection with the peer was dropped. /// Returns `Err(PeerId)` if the peer wasn't connected or is not known to us. fn on_peer_dropped(&mut self, peer_id: PeerId, reason: DropReason) -> Result<(), PeerId> { - if self.drop_reserved_peer(&peer_id)? || self.drop_regular_peer(&peer_id, reason)? { + if self.drop_reserved_peer(&peer_id)? || self.drop_regular_peer(&peer_id) { // The peer found and disconnected. - self.report_disconnect(peer_id); + self.report_disconnect(peer_id, reason); Ok(()) } else { // The peer was not found in neither regular or reserved lists. @@ -503,28 +461,19 @@ impl ProtocolController { } } - /// Try dropping a peer as a regular peer. Return `Ok(true)` if the peer was found and - /// disconnected, `Ok(false)` if it wasn't found, `Err(PeerId)`, if the peer found, but not in - /// connected state. - fn drop_regular_peer(&mut self, peer_id: &PeerId, reason: DropReason) -> Result { - let Some(state) = self.nodes.get_mut(peer_id) else { - return Ok(false) + /// Try dropping a peer as a regular peer. Return `true` if the peer was found and + /// disconnected, `false` if it wasn't found. + fn drop_regular_peer(&mut self, peer_id: &PeerId) -> bool { + let Some(direction) = self.nodes.remove(peer_id) else { + return false }; - if let PeerState::Connected(d) = state { - match d { - Direction::Inbound => self.num_in -= 1, - Direction::Outbound => self.num_out -= 1, - } - if let DropReason::Refused = reason { - self.nodes.remove(&peer_id); - } else { - *state = PeerState::NotConnected; - } - Ok(true) - } else { - Err(peer_id.clone()) + match direction { + Direction::Inbound => self.num_in -= 1, + Direction::Outbound => self.num_out -= 1, } + + true } /// Initiate outgoing connections trying to connect all reserved nodes and fill in all outgoing @@ -565,36 +514,27 @@ impl ProtocolController { .keys() .cloned() .collect::>() - .union( - &self - .nodes - .iter() - .filter_map(|(peer_id, state)| state.is_connected().then_some(*peer_id)) - .collect::>(), - ) + .union(&self.nodes.keys().cloned().collect::>()) .cloned() .collect::>(); self.peerset_handle .outgoing_candidates(available_slots, ignored) - .filter_map(|peer_id| { - let state = self.nodes.entry(*peer_id).or_insert(PeerState::NotConnected); - match state { - PeerState::Connected(_) => { - debug_assert!(false, "`Peerset` returned a node we asked to ignore."); - error!( - target: "peerset", - "`Peerset` returned a node we asked to ignore: {}.", - peer_id - ); - None - }, - PeerState::NotConnected => { - self.num_out += 1; - *state = PeerState::Connected(Direction::Outbound); - Some(peer_id) - }, - } + .filter_map(|peer_id| match self.nodes.entry(*peer_id) { + Entry::Occupied(_) => { + debug_assert!(false, "`Peerset` returned a node we asked to ignore."); + error!( + target: "peerset", + "`Peerset` returned a node we asked to ignore: {}.", + peer_id + ); + None + }, + Entry::Vacant(entry) => { + self.num_out += 1; + entry.insert(Direction::Outbound); + Some(peer_id) + }, }) .cloned() .collect::>() From 63385ff3e1df413497b3e0c0d458e2839745706a Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 20 Mar 2023 16:59:19 +0200 Subject: [PATCH 15/98] minor: docs --- client/peerset/src/protocol_controller.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 094d7a4df191c..df667dc6b4630 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -272,7 +272,7 @@ impl ProtocolController { self.peerset_handle.is_banned(peer_id) } - /// Add a peer to the set of reserved peers. [`ProtocolCOntroller`] will try to always maintain + /// Add the peer to the set of reserved peers. [`ProtocolCOntroller`] will try to always maintain /// connections with such peers. fn on_add_reserved_peer(&mut self, peer_id: PeerId) { if self.reserved_nodes.contains_key(&peer_id) { @@ -295,7 +295,7 @@ impl ProtocolController { } } - /// Remove a peer from the set of reserved peers. The peer is moved to the set of regular nodes. + /// Remove the peer from the set of reserved peers. The peer is moved to the set of regular nodes. fn on_remove_reserved_peer(&mut self, peer_id: PeerId) { let mut state = match self.reserved_nodes.remove(&peer_id) { Some(state) => state, @@ -445,7 +445,7 @@ impl ProtocolController { } } - /// Try dropping a peer as a reserved peer. Return `Ok(true)` if the peer was found and + /// Try dropping the peer as a reserved peer. Return `Ok(true)` if the peer was found and /// disconnected, `Ok(false)` if it wasn't found, `Err(PeerId)`, if the peer found, but not in /// connected state. fn drop_reserved_peer(&mut self, peer_id: &PeerId) -> Result { @@ -461,7 +461,7 @@ impl ProtocolController { } } - /// Try dropping a peer as a regular peer. Return `true` if the peer was found and + /// Try dropping the peer as a regular peer. Return `true` if the peer was found and /// disconnected, `false` if it wasn't found. fn drop_regular_peer(&mut self, peer_id: &PeerId) -> bool { let Some(direction) = self.nodes.remove(peer_id) else { From 368eaedd4642afbc806b1ae9a42792398fb19074 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 20 Mar 2023 19:02:55 +0200 Subject: [PATCH 16/98] WIP: add tests --- Cargo.lock | 2 + client/peerset/Cargo.toml | 2 + client/peerset/src/lib.rs | 1 + client/peerset/src/peer_store.rs | 32 +++++++ client/peerset/src/protocol_controller.rs | 110 +++++++++++++++++----- 5 files changed, 121 insertions(+), 26 deletions(-) create mode 100644 client/peerset/src/peer_store.rs diff --git a/Cargo.lock b/Cargo.lock index 11e10ac1fac78..436387a3da4c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9050,10 +9050,12 @@ dependencies = [ "futures", "libp2p", "log", + "mockall", "rand 0.8.5", "sc-utils", "serde_json", "sp-arithmetic", + "tokio", "wasm-timer", ] diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index 817f16c810270..e741aebd5a400 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -23,4 +23,6 @@ sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-arithmetic = { version = "6.0.0", path = "../../primitives/arithmetic" } [dev-dependencies] +mockall = "0.11.3" rand = "0.8.5" +tokio = { version = "1.22.0", features = ["macros", "rt"] } diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 497f9c30e5468..9b865fcd0642c 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -32,6 +32,7 @@ //! In addition, for each, set, the peerset also holds a list of reserved nodes towards which it //! will at all time try to maintain a connection with. +mod peer_store; mod peersstate; mod protocol_controller; diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs new file mode 100644 index 0000000000000..7e084f4d4b0ee --- /dev/null +++ b/client/peerset/src/peer_store.rs @@ -0,0 +1,32 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +use crate::DropReason; +use libp2p::PeerId; +use std::collections::HashSet; + +pub trait PeerStore { + /// Check whether the peer is banned. + fn is_banned(&self, peer_id: PeerId) -> bool; + + /// Report the peer disconnect for reputation adjustement. + fn report_disconnect(&self, peer_id: PeerId, reason: DropReason); + + /// Get the candidates for initiating outgoing connections. + fn outgoing_candidates(&self, count: usize, ignored: HashSet) -> Vec; +} diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index df667dc6b4630..f0624401edc31 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -35,7 +35,10 @@ use std::{ }; use wasm_timer::Delay; -use crate::{DropReason, IncomingIndex, Message, PeersetHandle, SetConfig, SetId}; +use crate::{ + peer_store::PeerStore as PeerStoreT, DropReason, IncomingIndex, Message, PeersetHandle, + SetConfig, SetId, +}; #[derive(Debug)] enum Action { @@ -135,7 +138,7 @@ impl Default for PeerState { /// Side of [`ProtocolHandle`] responsible for all the logic. Currently all instances are /// owned by [`crate::Peerset`], but they should eventually be moved to corresponding protocols. #[derive(Debug)] -pub struct ProtocolController { +pub struct ProtocolController { /// Set id to use when sending connect/drop requests to `Notifications`. // Will likely be replaced by `ProtocolName` in the future. set_id: SetId, @@ -161,17 +164,17 @@ pub struct ProtocolController { to_notifications: TracingUnboundedSender, /// Peerset handle for checking peer reputation values and getting connection candidates /// with highest reputation. - peerset_handle: PeersetHandle, + peer_store: PeerStore, } -impl ProtocolController { +impl ProtocolController { /// Construct new [`ProtocolController`]. pub fn new( set_id: SetId, config: SetConfig, to_notifications: TracingUnboundedSender, - peerset_handle: PeersetHandle, - ) -> (ProtocolHandle, ProtocolController) { + peer_store: PeerStore, + ) -> (ProtocolHandle, ProtocolController) { let (to_controller, from_handle) = tracing_unbounded("mpsc_protocol_controller", 10_000); let handle = ProtocolHandle { to_controller }; let reserved_nodes = @@ -188,7 +191,7 @@ impl ProtocolController { reserved_only: config.reserved_only, next_periodic_alloc_slots: Instant::now(), to_notifications, - peerset_handle, + peer_store, }; (handle, controller) } @@ -264,16 +267,16 @@ impl ProtocolController { /// Report peer disconnect event to `Peerset` for it to update peer's reputation accordingly. fn report_disconnect(&self, peer_id: PeerId, reason: DropReason) { - self.peerset_handle.report_disconnect(peer_id, reason); + self.peer_store.report_disconnect(peer_id, reason); } /// Ask `Peerset` if the peer has a reputation value not sufficent for connection with it. fn is_banned(&self, peer_id: PeerId) -> bool { - self.peerset_handle.is_banned(peer_id) + self.peer_store.is_banned(peer_id) } - /// Add the peer to the set of reserved peers. [`ProtocolCOntroller`] will try to always maintain - /// connections with such peers. + /// Add the peer to the set of reserved peers. [`ProtocolCOntroller`] will try to always + /// maintain connections with such peers. fn on_add_reserved_peer(&mut self, peer_id: PeerId) { if self.reserved_nodes.contains_key(&peer_id) { return @@ -295,7 +298,8 @@ impl ProtocolController { } } - /// Remove the peer from the set of reserved peers. The peer is moved to the set of regular nodes. + /// Remove the peer from the set of reserved peers. The peer is moved to the set of regular + /// nodes. fn on_remove_reserved_peer(&mut self, peer_id: PeerId) { let mut state = match self.reserved_nodes.remove(&peer_id) { Some(state) => state, @@ -401,7 +405,7 @@ impl ProtocolController { PeerState::NotConnected => { // It's questionable whether we should check a reputation of reserved node. // FIXME: unable to call `self.is_banned()` because of borrowed `self`. - if self.peerset_handle.is_banned(peer_id) { + if self.peer_store.is_banned(peer_id) { self.reject_connection(incoming_index); } else { *state = PeerState::Connected(Direction::Inbound); @@ -479,20 +483,15 @@ impl ProtocolController { /// Initiate outgoing connections trying to connect all reserved nodes and fill in all outgoing /// slots. fn alloc_slots(&mut self) { - if self.num_out >= self.max_out { - return - } - // Try connecting to reserved nodes first. self.reserved_nodes .iter_mut() .filter_map(|(peer_id, state)| { - (matches!(state, PeerState::NotConnected) && - !self.peerset_handle.is_banned(*peer_id)) - .then_some({ - *state = PeerState::Connected(Direction::Outbound); - peer_id - }) + (matches!(state, PeerState::NotConnected) && !self.peer_store.is_banned(*peer_id)) + .then_some({ + *state = PeerState::Connected(Direction::Outbound); + peer_id + }) }) .cloned() .collect::>() @@ -501,8 +500,8 @@ impl ProtocolController { self.start_connection(peer_id); }); - // Nothing more to do if we're in reserved-only mode. - if self.reserved_only { + // Nothing more to do if we're in reserved-only mode or don't have slots. + if self.reserved_only || self.num_out >= self.max_out { return } @@ -518,8 +517,9 @@ impl ProtocolController { .cloned() .collect::>(); - self.peerset_handle + self.peer_store .outgoing_candidates(available_slots, ignored) + .iter() .filter_map(|peer_id| match self.nodes.entry(*peer_id) { Entry::Occupied(_) => { debug_assert!(false, "`Peerset` returned a node we asked to ignore."); @@ -542,3 +542,61 @@ impl ProtocolController { .for_each(|peer_id| self.start_connection(*peer_id)); } } + +#[cfg(test)] +mod tests { + use super::ProtocolController; + use crate::{peer_store::PeerStore, DropReason, Message, SetConfig, SetId}; + use futures::FutureExt; + use libp2p::PeerId; + use sc_utils::mpsc::tracing_unbounded; + use std::collections::HashSet; + + mockall::mock! { + pub PeerStore {} + + impl PeerStore for PeerStore { + fn is_banned(&self, peer_id: PeerId) -> bool; + fn report_disconnect(&self, peer_id: PeerId, reason: DropReason); + fn outgoing_candidates(&self, count: usize, ignored: HashSet) -> Vec; + } + } + + #[tokio::test] + async fn reserved_nodes_are_connected() { + let reserved1 = PeerId::random(); + let reserved2 = PeerId::random(); + + // Add first reserved node via config. + let config = SetConfig { + in_peers: 0, + out_peers: 0, + bootnodes: Vec::new(), + reserved_nodes: std::iter::once(reserved1).collect(), + reserved_only: true, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().times(2).return_const(false); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Add second reserved node at runtime. + controller.on_add_reserved_peer(reserved2); + + // Poll once + controller.next_action().now_or_never(); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + } + + // TODO +} From 4c9521686ebd963728797131901595739a1b59a3 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 20 Mar 2023 19:12:39 +0200 Subject: [PATCH 17/98] minor: take no more connection candidates than slots available --- client/peerset/src/protocol_controller.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index f0624401edc31..00869701fec08 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -500,7 +500,7 @@ impl ProtocolController { self.start_connection(peer_id); }); - // Nothing more to do if we're in reserved-only mode or don't have slots. + // Nothing more to do if we're in reserved-only mode or don't have slots available. if self.reserved_only || self.num_out >= self.max_out { return } @@ -536,6 +536,7 @@ impl ProtocolController { Some(peer_id) }, }) + .take(available_slots) .cloned() .collect::>() .iter() From d09dc00dd773c988e7154d0ead1a709afcb18e51 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 21 Mar 2023 10:03:52 +0200 Subject: [PATCH 18/98] Test both connect/accept for reserved nodes --- client/peerset/src/protocol_controller.rs | 32 +++++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 00869701fec08..9bd5ba4ecb725 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -547,7 +547,7 @@ impl ProtocolController { #[cfg(test)] mod tests { use super::ProtocolController; - use crate::{peer_store::PeerStore, DropReason, Message, SetConfig, SetId}; + use crate::{peer_store::PeerStore, DropReason, IncomingIndex, Message, SetConfig, SetId}; use futures::FutureExt; use libp2p::PeerId; use sc_utils::mpsc::tracing_unbounded; @@ -563,8 +563,8 @@ mod tests { } } - #[tokio::test] - async fn reserved_nodes_are_connected() { + #[test] + fn reserved_nodes_are_connected_and_accepted() { let reserved1 = PeerId::random(); let reserved2 = PeerId::random(); @@ -579,15 +579,16 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStore::new(); - peer_store.expect_is_banned().times(2).return_const(false); + peer_store.expect_is_banned().times(4).return_const(false); + peer_store.expect_report_disconnect().times(2).return_const(()); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); // Add second reserved node at runtime. controller.on_add_reserved_peer(reserved2); - // Poll once - controller.next_action().now_or_never(); + // Initiate connections. + controller.alloc_slots(); let mut messages = Vec::new(); while let Some(message) = rx.try_recv().ok() { @@ -597,7 +598,22 @@ mod tests { assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); - } - // TODO + // Drop connections to be able to accept reserved nodes. + controller.on_peer_dropped(reserved1, DropReason::Refused); + controller.on_peer_dropped(reserved2, DropReason::Unknown); + + // Incoming connection from `reserved1`. + let incoming1 = IncomingIndex(1); + controller.on_incoming_connection(reserved1, incoming1); + assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming1)); + + // Incoming connection from `reserved2`. + let incoming2 = IncomingIndex(2); + controller.on_incoming_connection(reserved2, incoming2); + assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming2)); + + // No more commands. + assert!(rx.try_recv().is_err()); + } } From 7a25e8a4ca4405ea3e1942d7fca455150a4b2581 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 21 Mar 2023 10:25:53 +0200 Subject: [PATCH 19/98] Test banned reserved nodes, fix `alloc_slots` --- client/peerset/src/protocol_controller.rs | 50 ++++++++++++++++++++++- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 9bd5ba4ecb725..b2cf30b3364d6 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -488,7 +488,8 @@ impl ProtocolController { .iter_mut() .filter_map(|(peer_id, state)| { (matches!(state, PeerState::NotConnected) && !self.peer_store.is_banned(*peer_id)) - .then_some({ + .then(|| { + println!("Connecting"); *state = PeerState::Connected(Direction::Outbound); peer_id }) @@ -584,7 +585,7 @@ mod tests { let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); - // Add second reserved node at runtime. + // Add second reserved node at runtime (this currently calls `alloc_slots` internally). controller.on_add_reserved_peer(reserved2); // Initiate connections. @@ -616,4 +617,49 @@ mod tests { // No more commands. assert!(rx.try_recv().is_err()); } + + #[test] + fn banned_reserved_nodes_are_not_connected_and_not_accepted() { + let reserved1 = PeerId::random(); + let reserved2 = PeerId::random(); + + // Add first reserved node via config. + let config = SetConfig { + in_peers: 0, + out_peers: 0, + bootnodes: Vec::new(), + reserved_nodes: std::iter::once(reserved1).collect(), + reserved_only: true, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().times(6).return_const(true); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Add second reserved node at runtime (this currently calls `alloc_slots` internally). + controller.on_add_reserved_peer(reserved2); + + // Initiate connections. + controller.alloc_slots(); + + // No commands are generated. + assert!(rx.try_recv().is_err()); + + // Incoming connection from `reserved1`. + let incoming1 = IncomingIndex(1); + controller.on_incoming_connection(reserved1, incoming1); + assert_eq!(rx.try_recv().unwrap(), Message::Reject(incoming1)); + + // Incoming connection from `reserved2`. + let incoming2 = IncomingIndex(2); + controller.on_incoming_connection(reserved2, incoming2); + assert_eq!(rx.try_recv().unwrap(), Message::Reject(incoming2)); + + // No more commands. + assert!(rx.try_recv().is_err()); + } + + // TODO } From 9747eafffcf92b85894888cc57c3ad2cc3a6e291 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 21 Mar 2023 16:44:45 +0200 Subject: [PATCH 20/98] Add more tests --- client/peerset/src/protocol_controller.rs | 829 +++++++++++++++++++++- client/utils/src/mpsc.rs | 4 +- 2 files changed, 821 insertions(+), 12 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index b2cf30b3364d6..a422c455d73e0 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -266,6 +266,7 @@ impl ProtocolController { } /// Report peer disconnect event to `Peerset` for it to update peer's reputation accordingly. + /// Should only be called if the remote node disconnected us, not the other way around. fn report_disconnect(&self, peer_id: PeerId, reason: DropReason) { self.peer_store.report_disconnect(peer_id, reason); } @@ -355,6 +356,9 @@ impl ProtocolController { // Disconnect all non-reserved peers. self.nodes.keys().for_each(|peer_id| self.drop_connection(*peer_id)); + self.nodes.clear(); + self.num_in = 0; + self.num_out = 0; } /// Disconnect the peer. @@ -421,12 +425,12 @@ impl ProtocolController { return } - if self.is_banned(peer_id) { + if self.num_in >= self.max_in { self.reject_connection(incoming_index); return } - if self.num_in >= self.max_in { + if self.is_banned(peer_id) { self.reject_connection(incoming_index); return } @@ -489,7 +493,6 @@ impl ProtocolController { .filter_map(|(peer_id, state)| { (matches!(state, PeerState::NotConnected) && !self.peer_store.is_banned(*peer_id)) .then(|| { - println!("Connecting"); *state = PeerState::Connected(Direction::Outbound); peer_id }) @@ -547,11 +550,11 @@ impl ProtocolController { #[cfg(test)] mod tests { - use super::ProtocolController; + use super::{Direction, PeerState, ProtocolController}; use crate::{peer_store::PeerStore, DropReason, IncomingIndex, Message, SetConfig, SetId}; use futures::FutureExt; use libp2p::PeerId; - use sc_utils::mpsc::tracing_unbounded; + use sc_utils::mpsc::{tracing_unbounded, TryRecvError}; use std::collections::HashSet; mockall::mock! { @@ -565,7 +568,7 @@ mod tests { } #[test] - fn reserved_nodes_are_connected_and_accepted() { + fn reserved_nodes_are_connected_dropped_and_accepted() { let reserved1 = PeerId::random(); let reserved2 = PeerId::random(); @@ -595,11 +598,14 @@ mod tests { while let Some(message) = rx.try_recv().ok() { messages.push(message); } - assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + // Reserved peers do not occupy slots. + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + // Drop connections to be able to accept reserved nodes. controller.on_peer_dropped(reserved1, DropReason::Refused); controller.on_peer_dropped(reserved2, DropReason::Unknown); @@ -614,8 +620,12 @@ mod tests { controller.on_incoming_connection(reserved2, incoming2); assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming2)); + // Reserved peers do not occupy slots. + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + // No more commands. - assert!(rx.try_recv().is_err()); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); } #[test] @@ -644,8 +654,12 @@ mod tests { // Initiate connections. controller.alloc_slots(); + // No slots occupied. + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + // No commands are generated. - assert!(rx.try_recv().is_err()); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Incoming connection from `reserved1`. let incoming1 = IncomingIndex(1); @@ -657,9 +671,802 @@ mod tests { controller.on_incoming_connection(reserved2, incoming2); assert_eq!(rx.try_recv().unwrap(), Message::Reject(incoming2)); + // No slots occupied. + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + // No more commands. - assert!(rx.try_recv().is_err()); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + } + + #[test] + fn we_try_to_reconnect_to_dropped_reserved_nodes() { + let reserved1 = PeerId::random(); + let reserved2 = PeerId::random(); + + // Add first reserved node via config. + let config = SetConfig { + in_peers: 0, + out_peers: 0, + bootnodes: Vec::new(), + reserved_nodes: std::iter::once(reserved1).collect(), + reserved_only: true, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().times(4).return_const(false); + peer_store.expect_report_disconnect().times(2).return_const(()); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Add second reserved node at runtime (this calls `alloc_slots` internally). + controller.on_add_reserved_peer(reserved2); + + // Initiate connections (actually redundant, see previous comment). + controller.alloc_slots(); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + + // Drop both reserved nodes. + controller.on_peer_dropped(reserved1, DropReason::Refused); + controller.on_peer_dropped(reserved2, DropReason::Unknown); + + // Initiate connections. + controller.alloc_slots(); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + + // No slots occupied. + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + } + + #[test] + fn nodes_supplied_by_peer_store_are_connected() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + let peer3 = PeerId::random(); + let candidates = vec![peer1, peer2, peer3]; + + let config = SetConfig { + in_peers: 0, + // Less slots than candidates. + out_peers: 2, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_outgoing_candidates().once().return_const(candidates); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Initiate connections. + controller.alloc_slots(); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + + // Only first two peers are connected (we only have 2 slots). + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer2 })); + + // Outgoing slots occupied. + assert_eq!(controller.num_out, 2); + assert_eq!(controller.num_in, 0); + + // No more nodes are connected. + controller.alloc_slots(); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + + // No more slots occupied. + assert_eq!(controller.num_out, 2); + assert_eq!(controller.num_in, 0); + } + + #[test] + fn both_reserved_nodes_and_nodes_supplied_by_peer_store_are_connected() { + let reserved1 = PeerId::random(); + let reserved2 = PeerId::random(); + let regular1 = PeerId::random(); + let regular2 = PeerId::random(); + let outgoing_candidates = vec![regular1, regular2]; + let reserved_nodes = [reserved1, reserved2].iter().cloned().collect(); + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes, + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().times(2).return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Initiate connections. + controller.alloc_slots(); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 4); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: regular1 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: regular2 })); + assert_eq!(controller.num_out, 2); + assert_eq!(controller.num_in, 0); + } + + #[test] + fn if_slots_are_freed_we_try_to_allocate_them_again() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + let peer3 = PeerId::random(); + let candidates1 = vec![peer1.clone(), peer2.clone(), peer3.clone()]; + let candidates2 = vec![peer3.clone()]; + + let config = SetConfig { + in_peers: 0, + // Less slots than candidates. + out_peers: 2, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_outgoing_candidates().once().return_const(candidates1); + peer_store.expect_outgoing_candidates().once().return_const(candidates2); + peer_store.expect_report_disconnect().times(2).return_const(()); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Initiate connections. + controller.alloc_slots(); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + + // Only first two peers are connected (we only have 2 slots). + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer2 })); + + // Outgoing slots occupied. + assert_eq!(controller.num_out, 2); + assert_eq!(controller.num_in, 0); + + // No more nodes are connected. + controller.alloc_slots(); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + + // No more slots occupied. + assert_eq!(controller.num_out, 2); + assert_eq!(controller.num_in, 0); + + // Drop peers. + controller.on_peer_dropped(peer1.clone(), DropReason::Unknown); + controller.on_peer_dropped(peer2.clone(), DropReason::Refused); + + // Slots are freed. + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + + // Initiate connections. + controller.alloc_slots(); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + + // Peers are connected. + assert_eq!(messages.len(), 1); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer3 })); + + // Outgoing slots occupied. + assert_eq!(controller.num_out, 1); + assert_eq!(controller.num_in, 0); + } + + #[test] + fn in_reserved_only_mode_no_peers_are_requested_from_peer_store_and_connected() { + let config = SetConfig { + in_peers: 0, + // Make sure we have slots available. + out_peers: 2, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: true, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Initiate connections. + controller.alloc_slots(); + + // No nodes are connected. + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + } + + #[test] + fn in_reserved_only_mode_no_regular_peers_are_accepted() { + let config = SetConfig { + // Make sure we have slots available. + in_peers: 2, + out_peers: 0, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: true, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + let peer = PeerId::random(); + let incoming_index = IncomingIndex(1); + controller.on_incoming_connection(peer, incoming_index); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + + // Peer is rejected. + assert_eq!(messages.len(), 1); + assert!(messages.contains(&Message::Reject(incoming_index))); + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + } + + #[test] + fn disabling_reserved_only_mode_allows_to_connect_to_peers() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + let candidates = vec![peer1.clone(), peer2.clone()]; + + let config = SetConfig { + in_peers: 0, + // Make sure we have slots available. + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: true, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_outgoing_candidates().once().return_const(candidates); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Initiate connections. + controller.alloc_slots(); + + // No nodes are connected. + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + + // Disable reserved-only mode (this also connects to peers). + controller.on_set_reserved_only(false); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer2 })); + assert_eq!(controller.num_out, 2); + assert_eq!(controller.num_in, 0); + } + + #[test] + fn enabling_reserved_only_mode_disconnects_regular_peers() { + let reserved1 = PeerId::random(); + let reserved2 = PeerId::random(); + let regular1 = PeerId::random(); + let regular2 = PeerId::random(); + let outgoing_candidates = vec![regular1]; + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: [reserved1, reserved2].iter().cloned().collect(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().times(3).return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + + // Connect `regular1` as outbound. + controller.alloc_slots(); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 3); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: regular1 })); + assert_eq!(controller.num_out, 1); + assert_eq!(controller.num_in, 0); + + // Connect `regular2` as inbound. + let incoming_index = IncomingIndex(1); + controller.on_incoming_connection(regular2, incoming_index); + assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming_index)); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.num_out, 1); + assert_eq!(controller.num_in, 1); + + // Switch to reserved-only mode. + controller.on_set_reserved_only(true); + + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Drop { set_id: SetId(0), peer_id: regular1 })); + assert!(messages.contains(&Message::Drop { set_id: SetId(0), peer_id: regular2 })); + assert_eq!(controller.nodes.len(), 0); + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + } + + #[test] + fn removed_disconnected_reserved_node_is_forgotten() { + let reserved1 = PeerId::random(); + let reserved2 = PeerId::random(); + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: [reserved1, reserved2].iter().cloned().collect(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + assert_eq!(controller.reserved_nodes.len(), 2); + assert_eq!(controller.nodes.len(), 0); + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + + controller.on_remove_reserved_peer(reserved1); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.reserved_nodes.len(), 1); + assert!(!controller.reserved_nodes.contains_key(&reserved1)); + assert_eq!(controller.nodes.len(), 0); + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + } + + #[test] + fn removed_connected_reserved_node_is_disconnected_in_reserved_only_mode() { + let reserved1 = PeerId::random(); + let reserved2 = PeerId::random(); + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: [reserved1, reserved2].iter().cloned().collect(), + reserved_only: true, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().times(2).return_const(false); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Initiate connections. + controller.alloc_slots(); + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + + // Remove reserved node + controller.on_remove_reserved_peer(reserved1); + assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: reserved1 }); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); } - // TODO + #[test] + fn removed_connected_reserved_nodes_become_regular_in_non_reserved_mode() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: [peer1, peer2].iter().cloned().collect(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().times(2).return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Connect `peer1` as inbound, `peer2` as outbound. + controller.on_incoming_connection(peer1, IncomingIndex(1)); + controller.alloc_slots(); + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer2 })); + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + + // Remove reserved nodes (and make them regular) + controller.on_remove_reserved_peer(peer1); + controller.on_remove_reserved_peer(peer2); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.nodes.len(), 2); + assert!(matches!(controller.nodes.get(&peer1), Some(Direction::Inbound))); + assert!(matches!(controller.nodes.get(&peer2), Some(Direction::Outbound))); + assert_eq!(controller.num_out, 1); + assert_eq!(controller.num_in, 1); + } + + #[test] + fn regular_nodes_stop_occupying_slots_when_become_reserved() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + let outgoing_candidates = vec![peer1]; + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().once().return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Connect `peer1` as outbound & `peer2` as inbound. + controller.alloc_slots(); + controller.on_incoming_connection(peer2, IncomingIndex(1)); + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); + assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); + assert_eq!(controller.num_in, 1); + assert_eq!(controller.num_out, 1); + + controller.on_add_reserved_peer(peer1); + controller.on_add_reserved_peer(peer2); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.num_in, 0); + assert_eq!(controller.num_out, 0); + } + + #[test] + fn disconnecting_regular_peers_work() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + let outgoing_candidates = vec![peer1]; + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().once().return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Connect `peer1` as outbound & `peer2` as inbound. + controller.alloc_slots(); + controller.on_incoming_connection(peer2, IncomingIndex(1)); + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); + assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); + assert_eq!(controller.nodes.len(), 2); + assert!(matches!(controller.nodes.get(&peer1), Some(Direction::Outbound))); + assert!(matches!(controller.nodes.get(&peer2), Some(Direction::Inbound))); + assert_eq!(controller.num_in, 1); + assert_eq!(controller.num_out, 1); + + controller.on_disconnect_peer(peer1); + assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: peer1 }); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.nodes.len(), 1); + assert!(!controller.nodes.contains_key(&peer1)); + assert_eq!(controller.num_in, 1); + assert_eq!(controller.num_out, 0); + + controller.on_disconnect_peer(peer2); + assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: peer2 }); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.nodes.len(), 0); + assert_eq!(controller.num_in, 0); + assert_eq!(controller.num_out, 0); + } + + #[test] + fn dropping_regular_peers_work() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + let outgoing_candidates = vec![peer1]; + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().once().return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + peer_store.expect_report_disconnect().times(2).return_const(()); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Connect `peer1` as outbound & `peer2` as inbound. + controller.alloc_slots(); + controller.on_incoming_connection(peer2, IncomingIndex(1)); + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); + assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); + assert_eq!(controller.nodes.len(), 2); + assert!(matches!(controller.nodes.get(&peer1), Some(Direction::Outbound))); + assert!(matches!(controller.nodes.get(&peer2), Some(Direction::Inbound))); + assert_eq!(controller.num_in, 1); + assert_eq!(controller.num_out, 1); + + controller.on_peer_dropped(peer1, DropReason::Refused); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.nodes.len(), 1); + assert!(!controller.nodes.contains_key(&peer1)); + assert_eq!(controller.num_in, 1); + assert_eq!(controller.num_out, 0); + + controller.on_peer_dropped(peer2, DropReason::Unknown); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.nodes.len(), 0); + assert_eq!(controller.num_in, 0); + assert_eq!(controller.num_out, 0); + } + + #[test] + fn we_dont_answer_to_incoming_requests_for_already_connected_peers() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + let outgoing_candidates = vec![peer1]; + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().once().return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Connect `peer1` as outbound & `peer2` as inbound. + controller.alloc_slots(); + controller.on_incoming_connection(peer2, IncomingIndex(1)); + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); + assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); + assert_eq!(controller.nodes.len(), 2); + assert!(matches!(controller.nodes.get(&peer1), Some(Direction::Outbound))); + assert!(matches!(controller.nodes.get(&peer2), Some(Direction::Inbound))); + assert_eq!(controller.num_in, 1); + assert_eq!(controller.num_out, 1); + + // Incoming requests for `peer1` & `peer2`. + controller.on_incoming_connection(peer1, IncomingIndex(1)); + controller.on_incoming_connection(peer2, IncomingIndex(2)); + + // No answer. + assert_eq!(controller.num_in, 1); + assert_eq!(controller.num_out, 1); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + } + + #[test] + fn incoming_peers_that_exceed_slots_are_rejected() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + + let config = SetConfig { + in_peers: 1, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().once().return_const(false); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Connect `peer1` as inbound. + controller.on_incoming_connection(peer1, IncomingIndex(1)); + assert_eq!(rx.try_recv().unwrap(), Message::Accept(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + + // Incoming requests for `peer2`. + controller.on_incoming_connection(peer2, IncomingIndex(2)); + assert_eq!(rx.try_recv().unwrap(), Message::Reject(IncomingIndex(2))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + } + + #[test] + fn banned_regular_incoming_node_is_rejected() { + let peer1 = PeerId::random(); + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().once().return_const(true); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Incoming request. + controller.on_incoming_connection(peer1, IncomingIndex(1)); + assert_eq!(rx.try_recv().unwrap(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + } + + #[test] + fn banned_reserved_incoming_node_is_rejected() { + let reserved1 = PeerId::random(); + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: std::iter::once(reserved1).collect(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().once().return_const(true); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + assert!(controller.reserved_nodes.contains_key(&reserved1)); + + // Incoming request. + controller.on_incoming_connection(reserved1, IncomingIndex(1)); + assert_eq!(rx.try_recv().unwrap(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + } + + #[test] + fn we_dont_connect_to_banned_reserved_node() { + let reserved1 = PeerId::random(); + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: std::iter::once(reserved1).collect(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().once().return_const(true); + peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + assert!(matches!(controller.reserved_nodes.get(&reserved1), Some(PeerState::NotConnected))); + + // Initiate connectios + controller.alloc_slots(); + assert!(matches!(controller.reserved_nodes.get(&reserved1), Some(PeerState::NotConnected))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + } } diff --git a/client/utils/src/mpsc.rs b/client/utils/src/mpsc.rs index 3f783b10060bd..f1e714f439087 100644 --- a/client/utils/src/mpsc.rs +++ b/client/utils/src/mpsc.rs @@ -18,8 +18,10 @@ //! Code to meter unbounded channels. +pub use async_channel::{TryRecvError, TrySendError}; + use crate::metrics::UNBOUNDED_CHANNELS_COUNTER; -use async_channel::{Receiver, Sender, TryRecvError, TrySendError}; +use async_channel::{Receiver, Sender}; use futures::{ stream::{FusedStream, Stream}, task::{Context, Poll}, From 0b4a4c24713d4dd08fda779bc3a292030c25a138 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 21 Mar 2023 16:56:02 +0200 Subject: [PATCH 21/98] minor: remove unneded tokio dev dependecy --- Cargo.lock | 1 - client/peerset/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 436387a3da4c3..4adf88b850db2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9055,7 +9055,6 @@ dependencies = [ "sc-utils", "serde_json", "sp-arithmetic", - "tokio", "wasm-timer", ] diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index e741aebd5a400..03ccaf18bf9d3 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -25,4 +25,3 @@ sp-arithmetic = { version = "6.0.0", path = "../../primitives/arithmetic" } [dev-dependencies] mockall = "0.11.3" rand = "0.8.5" -tokio = { version = "1.22.0", features = ["macros", "rt"] } From 29d6e355db5a63b169326ef4448f34a57281acc3 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 22 Mar 2023 11:18:12 +0200 Subject: [PATCH 22/98] Add test for disconnecting reserved peers --- client/peerset/src/protocol_controller.rs | 54 +++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index a422c455d73e0..594e78375b6f6 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -1266,6 +1266,60 @@ mod tests { assert_eq!(controller.num_out, 0); } + #[test] + fn disconnecting_reserved_peers_is_a_noop() { + let reserved1 = PeerId::random(); + let reserved2 = PeerId::random(); + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: [reserved1, reserved2].iter().cloned().collect(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStore::new(); + peer_store.expect_is_banned().times(2).return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); + + let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + + // Connect `reserved1` as inbound & `reserved2` as outbound. + controller.on_incoming_connection(reserved1, IncomingIndex(1)); + controller.alloc_slots(); + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + assert!(matches!( + controller.reserved_nodes.get(&reserved1), + Some(PeerState::Connected(Direction::Inbound)) + )); + assert!(matches!( + controller.reserved_nodes.get(&reserved2), + Some(PeerState::Connected(Direction::Outbound)) + )); + + controller.on_disconnect_peer(reserved1); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!( + controller.reserved_nodes.get(&reserved1), + Some(PeerState::Connected(Direction::Inbound)) + )); + + controller.on_disconnect_peer(reserved2); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!( + controller.reserved_nodes.get(&reserved2), + Some(PeerState::Connected(Direction::Outbound)) + )); + } + #[test] fn dropping_regular_peers_work() { let peer1 = PeerId::random(); From d95562f573ad11ada0300084d6ef562ba88eeb96 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 22 Mar 2023 14:49:09 +0200 Subject: [PATCH 23/98] minor: clean `PeerSet` from `PeerStore` methods --- client/peerset/src/lib.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 9b865fcd0642c..be91f2fd36a21 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -178,26 +178,6 @@ impl PeersetHandle { // The channel can only be closed if the peerset no longer exists. rx.await.map_err(|_| ()) } - - /// Checks whether the peer has a reputation value below [`BANNED_THRESHOLD`]. - pub fn is_banned(&self, peer_id: PeerId) -> bool { - todo!("will be implemented after `Peerset` is converted into shared struct"); - } - - /// Report disconnect to adjust peers reputation value. - pub fn report_disconnect(&self, peer_id: PeerId, reason: DropReason) { - todo!("will be implemented after `Peerset` is converted into shared struct"); - } - - /// Get the candidates for an outgoing connection. - pub fn outgoing_candidates( - &self, - count: usize, - ignored: HashSet, - ) -> impl Iterator { - todo!("supply `count` peers with highest reputation, but not from `ignored`"); - [].into_iter() - } } /// Message that can be sent by the peer set manager (PSM). From 7a58b7e92fa17a6e6733ede3e719d7b6c9a9c4fa Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 22 Mar 2023 15:02:41 +0200 Subject: [PATCH 24/98] Pass ignored nodes as hash set of references to `PeerStore` --- client/peerset/src/peer_store.rs | 2 +- client/peerset/src/protocol_controller.rs | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 7e084f4d4b0ee..db817a512aca3 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -28,5 +28,5 @@ pub trait PeerStore { fn report_disconnect(&self, peer_id: PeerId, reason: DropReason); /// Get the candidates for initiating outgoing connections. - fn outgoing_candidates(&self, count: usize, ignored: HashSet) -> Vec; + fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec; } diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 594e78375b6f6..c96a989887729 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -515,11 +515,10 @@ impl ProtocolController { let ignored = self .reserved_nodes .keys() + .collect::>() + .union(&self.nodes.keys().collect::>()) .cloned() - .collect::>() - .union(&self.nodes.keys().cloned().collect::>()) - .cloned() - .collect::>(); + .collect(); self.peer_store .outgoing_candidates(available_slots, ignored) @@ -563,7 +562,7 @@ mod tests { impl PeerStore for PeerStore { fn is_banned(&self, peer_id: PeerId) -> bool; fn report_disconnect(&self, peer_id: PeerId, reason: DropReason); - fn outgoing_candidates(&self, count: usize, ignored: HashSet) -> Vec; + fn outgoing_candidates<'a>(&self, count: usize, ignored: HashSet<&'a PeerId>) -> Vec; } } From c1bb60159264f15d56a806f690d510d34002b1a6 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 22 Mar 2023 15:11:06 +0200 Subject: [PATCH 25/98] minor test fixes --- client/peerset/src/protocol_controller.rs | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index c96a989887729..b977113889be3 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -613,18 +613,17 @@ mod tests { let incoming1 = IncomingIndex(1); controller.on_incoming_connection(reserved1, incoming1); assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming1)); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Incoming connection from `reserved2`. let incoming2 = IncomingIndex(2); controller.on_incoming_connection(reserved2, incoming2); assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming2)); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Reserved peers do not occupy slots. assert_eq!(controller.num_out, 0); assert_eq!(controller.num_in, 0); - - // No more commands. - assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); } #[test] @@ -664,18 +663,17 @@ mod tests { let incoming1 = IncomingIndex(1); controller.on_incoming_connection(reserved1, incoming1); assert_eq!(rx.try_recv().unwrap(), Message::Reject(incoming1)); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Incoming connection from `reserved2`. let incoming2 = IncomingIndex(2); controller.on_incoming_connection(reserved2, incoming2); assert_eq!(rx.try_recv().unwrap(), Message::Reject(incoming2)); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // No slots occupied. assert_eq!(controller.num_out, 0); assert_eq!(controller.num_in, 0); - - // No more commands. - assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); } #[test] From 013ee1d6561a41a9433e377396abc21350399a11 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 22 Mar 2023 15:26:27 +0200 Subject: [PATCH 26/98] Better validation of `PeerStore` output --- client/peerset/src/protocol_controller.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index b977113889be3..79eefbab6bd61 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -523,8 +523,8 @@ impl ProtocolController { self.peer_store .outgoing_candidates(available_slots, ignored) .iter() - .filter_map(|peer_id| match self.nodes.entry(*peer_id) { - Entry::Occupied(_) => { + .filter_map(|peer_id| { + if self.reserved_nodes.contains_key(peer_id) || self.nodes.contains_key(peer_id) { debug_assert!(false, "`Peerset` returned a node we asked to ignore."); error!( target: "peerset", @@ -532,12 +532,11 @@ impl ProtocolController { peer_id ); None - }, - Entry::Vacant(entry) => { + } else { self.num_out += 1; - entry.insert(Direction::Outbound); + self.nodes.insert(*peer_id, Direction::Outbound); Some(peer_id) - }, + } }) .take(available_slots) .cloned() From 7551cd0814b5024074639a701a42903dc62350e8 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 22 Mar 2023 15:54:52 +0200 Subject: [PATCH 27/98] Rename trait `PeerStore`->`PeerReputationProvider` --- client/peerset/src/peer_store.rs | 4 +- client/peerset/src/protocol_controller.rs | 68 ++++++++++++----------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index db817a512aca3..fddf79552c7ca 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -20,7 +20,7 @@ use crate::DropReason; use libp2p::PeerId; use std::collections::HashSet; -pub trait PeerStore { +pub trait PeerReputationProvider { /// Check whether the peer is banned. fn is_banned(&self, peer_id: PeerId) -> bool; @@ -30,3 +30,5 @@ pub trait PeerStore { /// Get the candidates for initiating outgoing connections. fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec; } + +pub struct PeerStoreHandle {} diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 79eefbab6bd61..2b1a84dc56485 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -36,7 +36,7 @@ use std::{ use wasm_timer::Delay; use crate::{ - peer_store::PeerStore as PeerStoreT, DropReason, IncomingIndex, Message, PeersetHandle, + peer_store::PeerReputationProvider, DropReason, IncomingIndex, Message, PeersetHandle, SetConfig, SetId, }; @@ -138,7 +138,7 @@ impl Default for PeerState { /// Side of [`ProtocolHandle`] responsible for all the logic. Currently all instances are /// owned by [`crate::Peerset`], but they should eventually be moved to corresponding protocols. #[derive(Debug)] -pub struct ProtocolController { +pub struct ProtocolController { /// Set id to use when sending connect/drop requests to `Notifications`. // Will likely be replaced by `ProtocolName` in the future. set_id: SetId, @@ -164,17 +164,17 @@ pub struct ProtocolController { to_notifications: TracingUnboundedSender, /// Peerset handle for checking peer reputation values and getting connection candidates /// with highest reputation. - peer_store: PeerStore, + peer_store: PeerStoreHandle, } -impl ProtocolController { +impl ProtocolController { /// Construct new [`ProtocolController`]. pub fn new( set_id: SetId, config: SetConfig, to_notifications: TracingUnboundedSender, - peer_store: PeerStore, - ) -> (ProtocolHandle, ProtocolController) { + peer_store: PeerStoreHandle, + ) -> (ProtocolHandle, ProtocolController) { let (to_controller, from_handle) = tracing_unbounded("mpsc_protocol_controller", 10_000); let handle = ProtocolHandle { to_controller }; let reserved_nodes = @@ -525,10 +525,10 @@ impl ProtocolController { .iter() .filter_map(|peer_id| { if self.reserved_nodes.contains_key(peer_id) || self.nodes.contains_key(peer_id) { - debug_assert!(false, "`Peerset` returned a node we asked to ignore."); + debug_assert!(false, "`PeerStore` returned a node we asked to ignore."); error!( target: "peerset", - "`Peerset` returned a node we asked to ignore: {}.", + "`PeerStore` returned a node we asked to ignore: {}.", peer_id ); None @@ -549,16 +549,18 @@ impl ProtocolController { #[cfg(test)] mod tests { use super::{Direction, PeerState, ProtocolController}; - use crate::{peer_store::PeerStore, DropReason, IncomingIndex, Message, SetConfig, SetId}; + use crate::{ + peer_store::PeerReputationProvider, DropReason, IncomingIndex, Message, SetConfig, SetId, + }; use futures::FutureExt; use libp2p::PeerId; use sc_utils::mpsc::{tracing_unbounded, TryRecvError}; use std::collections::HashSet; mockall::mock! { - pub PeerStore {} + pub PeerStoreHandle {} - impl PeerStore for PeerStore { + impl PeerReputationProvider for PeerStoreHandle { fn is_banned(&self, peer_id: PeerId) -> bool; fn report_disconnect(&self, peer_id: PeerId, reason: DropReason); fn outgoing_candidates<'a>(&self, count: usize, ignored: HashSet<&'a PeerId>) -> Vec; @@ -580,7 +582,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(4).return_const(false); peer_store.expect_report_disconnect().times(2).return_const(()); @@ -640,7 +642,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(6).return_const(true); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); @@ -690,7 +692,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(4).return_const(false); peer_store.expect_report_disconnect().times(2).return_const(()); @@ -749,7 +751,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_outgoing_candidates().once().return_const(candidates); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); @@ -798,7 +800,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -838,7 +840,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_outgoing_candidates().once().return_const(candidates1); peer_store.expect_outgoing_candidates().once().return_const(candidates2); peer_store.expect_report_disconnect().times(2).return_const(()); @@ -907,7 +909,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); @@ -932,7 +934,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); @@ -968,7 +970,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_outgoing_candidates().once().return_const(candidates); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); @@ -1013,7 +1015,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(3).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1072,7 +1074,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); assert_eq!(controller.reserved_nodes.len(), 2); @@ -1103,7 +1105,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(2).return_const(false); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); @@ -1138,7 +1140,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); @@ -1183,7 +1185,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1224,7 +1226,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1276,7 +1278,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); @@ -1331,7 +1333,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); peer_store.expect_report_disconnect().times(2).return_const(()); @@ -1383,7 +1385,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1429,7 +1431,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(false); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); @@ -1458,7 +1460,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(true); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); @@ -1482,7 +1484,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(true); let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); @@ -1507,7 +1509,7 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStore::new(); + let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(true); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); From 4fb10a10b07e1b3b52e4dcca0f5c68fe20b7bfe7 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 22 Mar 2023 18:32:18 +0200 Subject: [PATCH 28/98] WIP: introduce `PeerStore` reputation storage --- Cargo.lock | 7 ++ client/peerset/Cargo.toml | 1 + client/peerset/src/peer_store.rs | 108 ++++++++++++++++++++++++++++++- 3 files changed, 115 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 4adf88b850db2..50e970ad85088 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7018,6 +7018,12 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "partial_sort" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7924d1d0ad836f665c9065e26d016c673ece3993f30d340068b16f282afc1156" + [[package]] name = "paste" version = "1.0.11" @@ -9051,6 +9057,7 @@ dependencies = [ "libp2p", "log", "mockall", + "partial_sort", "rand 0.8.5", "sc-utils", "serde_json", diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index 03ccaf18bf9d3..cb8e56c0ea5f7 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -17,6 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] futures = "0.3.21" libp2p = "0.50.0" log = "0.4.17" +partial_sort = "0.2.0" serde_json = "1.0.85" wasm-timer = "0.2" sc-utils = { version = "4.0.0-dev", path = "../utils" } diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index fddf79552c7ca..11a206947c581 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -18,7 +18,23 @@ use crate::DropReason; use libp2p::PeerId; -use std::collections::HashSet; +use partial_sort::PartialSort; +use std::{ + cmp::{Ord, Ordering, PartialOrd}, + collections::{HashMap, HashSet}, +}; + +/// We don't accept nodes whose reputation is under this value. +const BANNED_THRESHOLD: i32 = 82 * (i32::MIN / 100); +/// Reputation change for a node when we get disconnected from it. +const DISCONNECT_REPUTATION_CHANGE: i32 = -256; +/// Relative decrement of a reputation that is applied every second. I.e., for inverse decrement of +/// 50 we decrease absolute value of the reputation by 1/50. This corresponds to a factor of `k = +/// 0.98`. It takes `ln(0.5) / ln(k)` seconds to reduce the reputation by half, or 34.3 seconds for +/// the values above. In this setup the maximum allowed absolute value of `i32::MAX` becomes 0 in +/// ~1100 seconds, and this is the maximun time records live in the hash map if they are not +/// updated. +const INVERSE_DECREMENT: i32 = 50; pub trait PeerReputationProvider { /// Check whether the peer is banned. @@ -32,3 +48,93 @@ pub trait PeerReputationProvider { } pub struct PeerStoreHandle {} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Reputation(i32); + +impl Default for Reputation { + fn default() -> Self { + Reputation(0) + } +} + +impl Ord for Reputation { + // We define reverse order for reputation values. + fn cmp(&self, other: &Self) -> Ordering { + self.0.cmp(&other.0).reverse() + } +} + +impl PartialOrd for Reputation { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Reputation { + fn is_banned(&self) -> bool { + self.0 < BANNED_THRESHOLD + } + + fn add_reputation(&mut self, increment: i32) { + self.0 = self.0.saturating_add(increment); + } + + fn decay(&mut self, seconds_passed: u32) { + let reputation = &mut self.0; + + for _ in 0..seconds_passed { + let mut diff = *reputation / INVERSE_DECREMENT; + if diff == 0 && *reputation < 0 { + diff = -1; + } else if diff == 0 && *reputation > 0 { + diff = 1; + } + + *reputation = reputation.saturating_sub(diff); + + if *reputation == 0 { + break + } + } + } +} + +struct PeerStoreInner { + reputations: HashMap, +} + +impl PeerStoreInner { + fn is_banned(&self, peer_id: &PeerId) -> bool { + self.reputations.get(peer_id).cloned().unwrap_or_default().is_banned() + } + + fn report_disconnect(&mut self, peer_id: PeerId, reason: DropReason) { + self.reputations + .entry(peer_id) + .or_default() + .add_reputation(DISCONNECT_REPUTATION_CHANGE); + // FIXME: in the original `PeerSet`, `DropReason` was used to remove peers + // connection attempts to which were rejected. As connection management logic was moved + // to `ProtocolController`, I don't think we need this in `PeerStore`. + } + + fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { + let mut candidates = self + .reputations + .iter() + .filter_map(|(peer_id, reputation)| { + if !reputation.is_banned() && !ignored.contains(peer_id) { + Some((*peer_id, *reputation)) + } else { + None + } + }) + .collect::>(); + candidates.partial_sort(count, |(_, rep1), (_, rep2)| rep1.cmp(rep2)); + candidates.iter().take(count).map(|(peer_id, _)| *peer_id).collect() + + // FIXME: depending on usage patterns, it might be more efficient to always maintain peers + // sorted. + } +} From 67ae9cbb42a530283b5c2bfa978fb1e879ecf2ce Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 24 Mar 2023 13:58:57 +0300 Subject: [PATCH 29/98] Apply suggestions from code review Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> --- client/peerset/src/peer_store.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 11a206947c581..883cf53fbc604 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -124,11 +124,7 @@ impl PeerStoreInner { .reputations .iter() .filter_map(|(peer_id, reputation)| { - if !reputation.is_banned() && !ignored.contains(peer_id) { - Some((*peer_id, *reputation)) - } else { - None - } + (!reputation.is_banned() && !ignored.contains(peer_id)).then_some((*peer_id, *reputation)) }) .collect::>(); candidates.partial_sort(count, |(_, rep1), (_, rep2)| rep1.cmp(rep2)); From 57d7ebdf517053ae24ad1d74a95fed466703596d Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 24 Mar 2023 13:01:21 +0200 Subject: [PATCH 30/98] minor: rustfmt --- client/peerset/src/peer_store.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 883cf53fbc604..a9775a6675cca 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -124,7 +124,8 @@ impl PeerStoreInner { .reputations .iter() .filter_map(|(peer_id, reputation)| { - (!reputation.is_banned() && !ignored.contains(peer_id)).then_some((*peer_id, *reputation)) + (!reputation.is_banned() && !ignored.contains(peer_id)) + .then_some((*peer_id, *reputation)) }) .collect::>(); candidates.partial_sort(count, |(_, rep1), (_, rep2)| rep1.cmp(rep2)); From 7ac4180c2464c46c400d4b26d45f79a8c47cbf64 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 24 Mar 2023 14:43:14 +0200 Subject: [PATCH 31/98] Remove `DropReason` from peer dropped events --- client/peerset/src/peer_store.rs | 8 +--- client/peerset/src/protocol_controller.rs | 51 +++++++++++------------ 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index a9775a6675cca..0d976d6b1f932 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -16,7 +16,6 @@ // You should have received a copy of the GNU General Public License // along with this program. If not, see . -use crate::DropReason; use libp2p::PeerId; use partial_sort::PartialSort; use std::{ @@ -41,7 +40,7 @@ pub trait PeerReputationProvider { fn is_banned(&self, peer_id: PeerId) -> bool; /// Report the peer disconnect for reputation adjustement. - fn report_disconnect(&self, peer_id: PeerId, reason: DropReason); + fn report_disconnect(&self, peer_id: PeerId); /// Get the candidates for initiating outgoing connections. fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec; @@ -109,14 +108,11 @@ impl PeerStoreInner { self.reputations.get(peer_id).cloned().unwrap_or_default().is_banned() } - fn report_disconnect(&mut self, peer_id: PeerId, reason: DropReason) { + fn report_disconnect(&mut self, peer_id: PeerId) { self.reputations .entry(peer_id) .or_default() .add_reputation(DISCONNECT_REPUTATION_CHANGE); - // FIXME: in the original `PeerSet`, `DropReason` was used to remove peers - // connection attempts to which were rejected. As connection management logic was moved - // to `ProtocolController`, I don't think we need this in `PeerStore`. } fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 2b1a84dc56485..cca03124125df 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -48,7 +48,7 @@ enum Action { SetReservedOnly(bool), DisconnectPeer(PeerId), IncomingConnection(PeerId, IncomingIndex), - Dropped(PeerId, DropReason), + Dropped(PeerId), } /// Shared handle to [`ProtocolController`]. Distributed around the code outside of the @@ -101,8 +101,8 @@ impl ProtocolHandle { } /// Notify that connection was dropped (eithere refused or disconnected). - pub fn dropped(&self, peer_id: PeerId, reason: DropReason) { - let _ = self.to_controller.unbounded_send(Action::Dropped(peer_id, reason)); + pub fn dropped(&self, peer_id: PeerId) { + let _ = self.to_controller.unbounded_send(Action::Dropped(peer_id)); } } @@ -228,15 +228,14 @@ impl ProtocolController self.on_disconnect_peer(peer_id), Action::IncomingConnection(peer_id, index) => self.on_incoming_connection(peer_id, index), - Action::Dropped(peer_id, reason) => - self.on_peer_dropped(peer_id, reason).unwrap_or_else(|peer_id| { - debug_assert!(false, "Received Action::Dropped for non-connected peer."); - error!( - target: "peerset", - "Received Action::Dropped for non-connected peer {} on {:?}.", - peer_id, self.set_id, - ) - }), + Action::Dropped(peer_id) => self.on_peer_dropped(peer_id).unwrap_or_else(|peer_id| { + debug_assert!(false, "Received Action::Dropped for non-connected peer."); + error!( + target: "peerset", + "Received Action::Dropped for non-connected peer {} on {:?}.", + peer_id, self.set_id, + ) + }), } true } @@ -265,10 +264,10 @@ impl ProtocolController ProtocolController Result<(), PeerId> { + fn on_peer_dropped(&mut self, peer_id: PeerId) -> Result<(), PeerId> { if self.drop_reserved_peer(&peer_id)? || self.drop_regular_peer(&peer_id) { // The peer found and disconnected. - self.report_disconnect(peer_id, reason); + self.report_disconnect(peer_id); Ok(()) } else { // The peer was not found in neither regular or reserved lists. @@ -562,7 +561,7 @@ mod tests { impl PeerReputationProvider for PeerStoreHandle { fn is_banned(&self, peer_id: PeerId) -> bool; - fn report_disconnect(&self, peer_id: PeerId, reason: DropReason); + fn report_disconnect(&self, peer_id: PeerId); fn outgoing_candidates<'a>(&self, count: usize, ignored: HashSet<&'a PeerId>) -> Vec; } } @@ -607,8 +606,8 @@ mod tests { assert_eq!(controller.num_in, 0); // Drop connections to be able to accept reserved nodes. - controller.on_peer_dropped(reserved1, DropReason::Refused); - controller.on_peer_dropped(reserved2, DropReason::Unknown); + controller.on_peer_dropped(reserved1); + controller.on_peer_dropped(reserved2); // Incoming connection from `reserved1`. let incoming1 = IncomingIndex(1); @@ -714,8 +713,8 @@ mod tests { assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); // Drop both reserved nodes. - controller.on_peer_dropped(reserved1, DropReason::Refused); - controller.on_peer_dropped(reserved2, DropReason::Unknown); + controller.on_peer_dropped(reserved1); + controller.on_peer_dropped(reserved2); // Initiate connections. controller.alloc_slots(); @@ -873,8 +872,8 @@ mod tests { assert_eq!(controller.num_in, 0); // Drop peers. - controller.on_peer_dropped(peer1.clone(), DropReason::Unknown); - controller.on_peer_dropped(peer2.clone(), DropReason::Refused); + controller.on_peer_dropped(peer1.clone()); + controller.on_peer_dropped(peer2.clone()); // Slots are freed. assert_eq!(controller.num_out, 0); @@ -1356,14 +1355,14 @@ mod tests { assert_eq!(controller.num_in, 1); assert_eq!(controller.num_out, 1); - controller.on_peer_dropped(peer1, DropReason::Refused); + controller.on_peer_dropped(peer1); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert_eq!(controller.nodes.len(), 1); assert!(!controller.nodes.contains_key(&peer1)); assert_eq!(controller.num_in, 1); assert_eq!(controller.num_out, 0); - controller.on_peer_dropped(peer2, DropReason::Unknown); + controller.on_peer_dropped(peer2); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert_eq!(controller.nodes.len(), 0); assert_eq!(controller.num_in, 0); From 463653674527ae9d7206150f09dd8d540ab670a5 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 24 Mar 2023 15:27:40 +0200 Subject: [PATCH 32/98] Fix handling of invalid `PeerStore` output in `alloc_slots` --- client/peerset/src/protocol_controller.rs | 56 +++++++++++++---------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index cca03124125df..4aceae402f4c2 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -510,6 +510,7 @@ impl ProtocolController ProtocolController>() - .iter() - .for_each(|peer_id| self.start_connection(*peer_id)); + .collect::>(); + + if candidates.len() > available_slots { + debug_assert!(false, "`PeerStore` returned more nodes than there are slots available."); + error!( + target: "peerset", + "`PeerStore` returned more nodes than there are slots available.", + ) + } + + candidates.into_iter().take(available_slots).for_each(|peer_id| { + self.num_out += 1; + self.nodes.insert(peer_id, Direction::Outbound); + self.start_connection(peer_id); + }) } } @@ -738,7 +748,7 @@ mod tests { let peer1 = PeerId::random(); let peer2 = PeerId::random(); let peer3 = PeerId::random(); - let candidates = vec![peer1, peer2, peer3]; + let candidates = vec![peer1, peer2]; let config = SetConfig { in_peers: 0, @@ -826,8 +836,8 @@ mod tests { let peer1 = PeerId::random(); let peer2 = PeerId::random(); let peer3 = PeerId::random(); - let candidates1 = vec![peer1.clone(), peer2.clone(), peer3.clone()]; - let candidates2 = vec![peer3.clone()]; + let candidates1 = vec![peer1, peer2]; + let candidates2 = vec![peer3]; let config = SetConfig { in_peers: 0, From e865c191807e00564ca2a394413dbedc6c5ada90 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 24 Mar 2023 17:05:59 +0200 Subject: [PATCH 33/98] Implement `PeerStore` --- client/peerset/src/peer_store.rs | 114 +++++++++++++++++++--- client/peerset/src/protocol_controller.rs | 21 ++-- 2 files changed, 114 insertions(+), 21 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 0d976d6b1f932..a5fef42df5f95 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -21,32 +21,67 @@ use partial_sort::PartialSort; use std::{ cmp::{Ord, Ordering, PartialOrd}, collections::{HashMap, HashSet}, + sync::{Arc, Mutex}, + time::{Duration, Instant}, }; +use wasm_timer::Delay; + +use crate::ReputationChange; /// We don't accept nodes whose reputation is under this value. const BANNED_THRESHOLD: i32 = 82 * (i32::MIN / 100); /// Reputation change for a node when we get disconnected from it. const DISCONNECT_REPUTATION_CHANGE: i32 = -256; -/// Relative decrement of a reputation that is applied every second. I.e., for inverse decrement of -/// 50 we decrease absolute value of the reputation by 1/50. This corresponds to a factor of `k = -/// 0.98`. It takes `ln(0.5) / ln(k)` seconds to reduce the reputation by half, or 34.3 seconds for -/// the values above. In this setup the maximum allowed absolute value of `i32::MAX` becomes 0 in -/// ~1100 seconds, and this is the maximun time records live in the hash map if they are not -/// updated. +/// Relative decrement of a reputation value that is applied every second. I.e., for inverse +/// decrement of 50 we decrease absolute value of the reputation by 1/50. This corresponds to a +/// factor of `k = 0.98`. It takes `ln(0.5) / ln(k)` seconds to reduce the reputation by half, or +/// 34.3 seconds for the values above. In this setup the maximum allowed absolute value of +/// `i32::MAX` becomes 0 in ~1100 seconds, and this is the maximun time records live in the hash map +/// if they are not updated. const INVERSE_DECREMENT: i32 = 50; pub trait PeerReputationProvider { /// Check whether the peer is banned. - fn is_banned(&self, peer_id: PeerId) -> bool; + fn is_banned(&self, peer_id: &PeerId) -> bool; /// Report the peer disconnect for reputation adjustement. - fn report_disconnect(&self, peer_id: PeerId); + fn report_disconnect(&mut self, peer_id: PeerId); + + /// Adjust peer reputation value. + fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); + + /// Get peer reputation. + fn peer_reputation(&self, peer_id: &PeerId) -> i32; /// Get the candidates for initiating outgoing connections. fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec; } -pub struct PeerStoreHandle {} +pub struct PeerStoreHandle { + inner: Arc>, +} + +impl PeerReputationProvider for PeerStoreHandle { + fn is_banned(&self, peer_id: &PeerId) -> bool { + self.inner.lock().unwrap().is_banned(peer_id) + } + + fn report_disconnect(&mut self, peer_id: PeerId) { + self.inner.lock().unwrap().report_disconnect(peer_id) + } + + fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange) { + self.inner.lock().unwrap().report_peer(peer_id, change) + } + + fn peer_reputation(&self, peer_id: &PeerId) -> i32 { + self.inner.lock().unwrap().peer_reputation(peer_id) + } + + fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { + self.inner.lock().unwrap().outgoing_candidates(count, ignored) + } +} #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct Reputation(i32); @@ -79,7 +114,11 @@ impl Reputation { self.0 = self.0.saturating_add(increment); } - fn decay(&mut self, seconds_passed: u32) { + fn value(&self) -> i32 { + self.0 + } + + fn decay(&mut self, seconds_passed: u64) { let reputation = &mut self.0; for _ in 0..seconds_passed { @@ -115,6 +154,14 @@ impl PeerStoreInner { .add_reputation(DISCONNECT_REPUTATION_CHANGE); } + fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange) { + self.reputations.entry(peer_id).or_default().add_reputation(change.value); + } + + fn peer_reputation(&self, peer_id: &PeerId) -> i32 { + self.reputations.get(peer_id).cloned().unwrap_or(Reputation::default()).0 + } + fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { let mut candidates = self .reputations @@ -127,7 +174,50 @@ impl PeerStoreInner { candidates.partial_sort(count, |(_, rep1), (_, rep2)| rep1.cmp(rep2)); candidates.iter().take(count).map(|(peer_id, _)| *peer_id).collect() - // FIXME: depending on usage patterns, it might be more efficient to always maintain peers - // sorted. + // FIXME: depending on the usage patterns, it might be more efficient to always maintain + // peers sorted. + } + + fn progress_time(&mut self, seconds_passed: u64) { + if seconds_passed == 0 { + return + } + + self.reputations + .iter_mut() + .for_each(|(_, reputation)| reputation.decay(seconds_passed)); + self.reputations.retain(|_, reputation| reputation.value() != 0); + } +} + +struct PeerStore { + inner: Arc>, +} + +impl PeerStore { + pub fn new() -> Self { + PeerStore { inner: Arc::new(Mutex::new(PeerStoreInner { reputations: HashMap::new() })) } + } + + pub fn handle(&self) -> PeerStoreHandle { + PeerStoreHandle { inner: self.inner.clone() } + } + + pub async fn run(self) { + let started = Instant::now(); + let mut latest_time_update = started; + loop { + let now = Instant::now(); + // We basically do `(now - self.latest_update).as_secs()`, except that by the way we do + // it we know that we're not going to miss seconds because of rounding to integers. + let seconds_passed = { + let elapsed_latest = latest_time_update - started; + let elapsed_now = now - started; + latest_time_update = now; + elapsed_now.as_secs() - elapsed_latest.as_secs() + }; + self.inner.lock().unwrap().progress_time(seconds_passed); + let _ = Delay::new(Duration::from_secs(1)).await; + } } } diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 4aceae402f4c2..418e1b3fdb2ac 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -266,12 +266,12 @@ impl ProtocolController bool { + fn is_banned(&self, peer_id: &PeerId) -> bool { self.peer_store.is_banned(peer_id) } @@ -408,7 +408,7 @@ impl ProtocolController { // It's questionable whether we should check a reputation of reserved node. // FIXME: unable to call `self.is_banned()` because of borrowed `self`. - if self.peer_store.is_banned(peer_id) { + if self.peer_store.is_banned(&peer_id) { self.reject_connection(incoming_index); } else { *state = PeerState::Connected(Direction::Inbound); @@ -429,7 +429,7 @@ impl ProtocolController ProtocolController ProtocolController ProtocolController bool; - fn report_disconnect(&self, peer_id: PeerId); + fn is_banned(&self, peer_id: &PeerId) -> bool; + fn report_disconnect(&mut self, peer_id: PeerId); + fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); + fn peer_reputation(&self, peer_id: &PeerId) -> i32; fn outgoing_candidates<'a>(&self, count: usize, ignored: HashSet<&'a PeerId>) -> Vec; } } From 2587cdc1712e16469338d4d9215af02cb7dfd374 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 24 Mar 2023 17:09:23 +0200 Subject: [PATCH 34/98] minor: docs --- client/peerset/src/peer_store.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index a5fef42df5f95..11beaf524c442 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -50,7 +50,7 @@ pub trait PeerReputationProvider { /// Adjust peer reputation value. fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); - /// Get peer reputation. + /// Get peer reputation value. fn peer_reputation(&self, peer_id: &PeerId) -> i32; /// Get the candidates for initiating outgoing connections. @@ -195,14 +195,17 @@ struct PeerStore { } impl PeerStore { + /// Create new empty peer store. pub fn new() -> Self { PeerStore { inner: Arc::new(Mutex::new(PeerStoreInner { reputations: HashMap::new() })) } } + /// Get `PeerStoreHandle`. pub fn handle(&self) -> PeerStoreHandle { PeerStoreHandle { inner: self.inner.clone() } } + /// Drive the `PeerStore`, decaying reputation values over time and removing expired entries. pub async fn run(self) { let started = Instant::now(); let mut latest_time_update = started; From 06f8fdff927436af505bdfacb988043eaee75043 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 24 Mar 2023 18:25:30 +0200 Subject: [PATCH 35/98] minor: TODO --- client/peerset/src/peer_store.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 11beaf524c442..cf7b925a10161 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -187,6 +187,14 @@ impl PeerStoreInner { .iter_mut() .for_each(|(_, reputation)| reputation.decay(seconds_passed)); self.reputations.retain(|_, reputation| reputation.value() != 0); + + // TODO: we likely need to also account for another condition when removing nodes, like it + // was done in `PeerSet` with `last_connected_or_discovered` and `FORGET_AFTER`. Otherwise, + // bootnodes will be deleted at startup without a chance of being connected. As another + // alternative, we can give bootnodes some initial reputation, enough for them to get + // connected. This alternative is questionable, because we are limited to 1100 seconds of + // reputation decay even for max reputation, which is less than one hour of lifetime in the + // original `PeerSet`. } } From 56d380b2d9e64d2c5846466dab1f7a61832a88f7 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 24 Mar 2023 18:26:26 +0200 Subject: [PATCH 36/98] minor: another TODO --- client/peerset/src/peer_store.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index cf7b925a10161..e3c79722a44a6 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -205,6 +205,7 @@ struct PeerStore { impl PeerStore { /// Create new empty peer store. pub fn new() -> Self { + // TODO: pass bootnodes. PeerStore { inner: Arc::new(Mutex::new(PeerStoreInner { reputations: HashMap::new() })) } } From 9c69f2d7868efb440c79c2afb49c007f157098ef Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 31 Mar 2023 16:21:15 +0300 Subject: [PATCH 37/98] Retain peer info for one hour after it was last updated --- client/peerset/src/peer_store.rs | 105 +++++++++++++++++-------------- 1 file changed, 57 insertions(+), 48 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index e3c79722a44a6..091390fc06ccb 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -20,7 +20,7 @@ use libp2p::PeerId; use partial_sort::PartialSort; use std::{ cmp::{Ord, Ordering, PartialOrd}, - collections::{HashMap, HashSet}, + collections::{hash_map::Entry, HashMap, HashSet}, sync::{Arc, Mutex}, time::{Duration, Instant}, }; @@ -36,9 +36,11 @@ const DISCONNECT_REPUTATION_CHANGE: i32 = -256; /// decrement of 50 we decrease absolute value of the reputation by 1/50. This corresponds to a /// factor of `k = 0.98`. It takes `ln(0.5) / ln(k)` seconds to reduce the reputation by half, or /// 34.3 seconds for the values above. In this setup the maximum allowed absolute value of -/// `i32::MAX` becomes 0 in ~1100 seconds, and this is the maximun time records live in the hash map -/// if they are not updated. +/// `i32::MAX` becomes 0 in ~1100 seconds. const INVERSE_DECREMENT: i32 = 50; +/// Amount of time between the moment we last updated the [`PeerStore`] entry and the moment we +/// remove it, once the reputation value reaches 0. +const FORGET_AFTER: Duration = Duration::from_secs(3600); pub trait PeerReputationProvider { /// Check whether the peer is banned. @@ -83,55 +85,63 @@ impl PeerReputationProvider for PeerStoreHandle { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct Reputation(i32); +#[derive(Debug, Clone, Copy)] +struct PeerInfo { + reputation: i32, + last_updated: Instant, +} -impl Default for Reputation { +impl Default for PeerInfo { fn default() -> Self { - Reputation(0) + Self { reputation: 0, last_updated: Instant::now() } } } -impl Ord for Reputation { - // We define reverse order for reputation values. +impl PartialEq for PeerInfo { + fn eq(&self, other: &Self) -> bool { + self.reputation == other.reputation + } +} + +impl Eq for PeerInfo {} + +impl Ord for PeerInfo { + // We define reverse order by reputation values. fn cmp(&self, other: &Self) -> Ordering { - self.0.cmp(&other.0).reverse() + self.reputation.cmp(&other.reputation).reverse() } } -impl PartialOrd for Reputation { +impl PartialOrd for PeerInfo { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } } -impl Reputation { +impl PeerInfo { fn is_banned(&self) -> bool { - self.0 < BANNED_THRESHOLD + self.reputation < BANNED_THRESHOLD } fn add_reputation(&mut self, increment: i32) { - self.0 = self.0.saturating_add(increment); + self.reputation = self.reputation.saturating_add(increment); + self.last_updated = Instant::now(); } - fn value(&self) -> i32 { - self.0 - } - - fn decay(&mut self, seconds_passed: u64) { - let reputation = &mut self.0; - + fn decay_reputation(&mut self, seconds_passed: u64) { + // Note that decaying a reputation value happens "on its own", + // so we don't bump `last_updated`. for _ in 0..seconds_passed { - let mut diff = *reputation / INVERSE_DECREMENT; - if diff == 0 && *reputation < 0 { + let mut diff = self.reputation / INVERSE_DECREMENT; + if diff == 0 && self.reputation < 0 { diff = -1; - } else if diff == 0 && *reputation > 0 { + } else if diff == 0 && self.reputation > 0 { diff = 1; } - *reputation = reputation.saturating_sub(diff); + self.reputation = self.reputation.saturating_sub(diff); - if *reputation == 0 { + if self.reputation == 0 { break } } @@ -139,39 +149,42 @@ impl Reputation { } struct PeerStoreInner { - reputations: HashMap, + peers: HashMap, } impl PeerStoreInner { fn is_banned(&self, peer_id: &PeerId) -> bool { - self.reputations.get(peer_id).cloned().unwrap_or_default().is_banned() + if let Some(info) = self.peers.get(peer_id) { + info.is_banned() + } else { + false + } } fn report_disconnect(&mut self, peer_id: PeerId) { - self.reputations + self.peers .entry(peer_id) .or_default() .add_reputation(DISCONNECT_REPUTATION_CHANGE); } fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange) { - self.reputations.entry(peer_id).or_default().add_reputation(change.value); + self.peers.entry(peer_id).or_default().add_reputation(change.value); } fn peer_reputation(&self, peer_id: &PeerId) -> i32 { - self.reputations.get(peer_id).cloned().unwrap_or(Reputation::default()).0 + self.peers.get(peer_id).cloned().unwrap_or_default().reputation } fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { let mut candidates = self - .reputations + .peers .iter() - .filter_map(|(peer_id, reputation)| { - (!reputation.is_banned() && !ignored.contains(peer_id)) - .then_some((*peer_id, *reputation)) + .filter_map(|(peer_id, info)| { + (!info.is_banned() && !ignored.contains(peer_id)).then_some((*peer_id, *info)) }) .collect::>(); - candidates.partial_sort(count, |(_, rep1), (_, rep2)| rep1.cmp(rep2)); + candidates.partial_sort(count, |(_, info1), (_, info2)| info1.cmp(info2)); candidates.iter().take(count).map(|(peer_id, _)| *peer_id).collect() // FIXME: depending on the usage patterns, it might be more efficient to always maintain @@ -183,18 +196,14 @@ impl PeerStoreInner { return } - self.reputations + // Drive reputation values towards 0. + self.peers .iter_mut() - .for_each(|(_, reputation)| reputation.decay(seconds_passed)); - self.reputations.retain(|_, reputation| reputation.value() != 0); - - // TODO: we likely need to also account for another condition when removing nodes, like it - // was done in `PeerSet` with `last_connected_or_discovered` and `FORGET_AFTER`. Otherwise, - // bootnodes will be deleted at startup without a chance of being connected. As another - // alternative, we can give bootnodes some initial reputation, enough for them to get - // connected. This alternative is questionable, because we are limited to 1100 seconds of - // reputation decay even for max reputation, which is less than one hour of lifetime in the - // original `PeerSet`. + .for_each(|(_, info)| info.decay_reputation(seconds_passed)); + // Retain only entries with non-zero reputation values or not expired ones. + let now = Instant::now(); + self.peers + .retain(|_, info| info.reputation != 0 || info.last_updated + FORGET_AFTER > now); } } @@ -206,7 +215,7 @@ impl PeerStore { /// Create new empty peer store. pub fn new() -> Self { // TODO: pass bootnodes. - PeerStore { inner: Arc::new(Mutex::new(PeerStoreInner { reputations: HashMap::new() })) } + PeerStore { inner: Arc::new(Mutex::new(PeerStoreInner { peers: HashMap::new() })) } } /// Get `PeerStoreHandle`. From 554dbf872695480930a6ffcabc911f6d84251935 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 31 Mar 2023 16:28:37 +0300 Subject: [PATCH 38/98] Construct `PeerStore` from the list of bootnodes --- client/peerset/src/peer_store.rs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 091390fc06ccb..ff3f99bf34513 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -212,10 +212,16 @@ struct PeerStore { } impl PeerStore { - /// Create new empty peer store. - pub fn new() -> Self { - // TODO: pass bootnodes. - PeerStore { inner: Arc::new(Mutex::new(PeerStoreInner { peers: HashMap::new() })) } + /// Create a new peer store from the list of bootnodes. + pub fn new(bootnodes: Vec) -> Self { + PeerStore { + inner: Arc::new(Mutex::new(PeerStoreInner { + peers: bootnodes + .into_iter() + .map(|peer_id| (peer_id, PeerInfo::default())) + .collect(), + })), + } } /// Get `PeerStoreHandle`. From 72ec71f56bbce595c8db2d79cf2c5925c228549d Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 4 Apr 2023 13:06:00 +0300 Subject: [PATCH 39/98] WIP: plug `PeerStore` and `ProtocolController` into `Peerset` --- client/peerset/src/lib.rs | 590 +++++----------------- client/peerset/src/peer_store.rs | 17 +- client/peerset/src/protocol_controller.rs | 17 +- 3 files changed, 145 insertions(+), 479 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index be91f2fd36a21..aa0f33d528626 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -36,6 +36,9 @@ mod peer_store; mod peersstate; mod protocol_controller; +use peer_store::{PeerReputationProvider, PeerStore, PeerStoreHandle}; +use protocol_controller::{ProtocolController, ProtocolHandle}; + use futures::{channel::oneshot, prelude::*}; use log::{debug, error, trace}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; @@ -50,26 +53,6 @@ use wasm_timer::Delay; pub use libp2p::PeerId; -/// We don't accept nodes whose reputation is under this value. -pub const BANNED_THRESHOLD: i32 = 82 * (i32::MIN / 100); -/// Reputation change for a node when we get disconnected from it. -const DISCONNECT_REPUTATION_CHANGE: i32 = -256; -/// Amount of time between the moment we disconnect from a node and the moment we remove it from -/// the list. -const FORGET_AFTER: Duration = Duration::from_secs(3600); - -#[derive(Debug)] -enum Action { - AddReservedPeer(SetId, PeerId), - RemoveReservedPeer(SetId, PeerId), - SetReservedPeers(SetId, HashSet), - SetReservedOnly(SetId, bool), - ReportPeer(PeerId, ReputationChange), - AddToPeersSet(SetId, PeerId), - RemoveFromPeersSet(SetId, PeerId), - PeerReputation(PeerId, oneshot::Sender), -} - /// Identifier of a set in the peerset. /// /// Can be constructed using the `From` trait implementation based on the index of the set @@ -121,7 +104,10 @@ impl ReputationChange { /// Shared handle to the peer set manager (PSM). Distributed around the code. #[derive(Debug, Clone)] pub struct PeersetHandle { - tx: TracingUnboundedSender, + // Peer store handle. + peer_store_handle: PeerStoreHandle, + // Protocol handles per every `SetId`. The size of this vectore never changes. + protocol_handles: Vec, } impl PeersetHandle { @@ -133,50 +119,46 @@ impl PeersetHandle { /// > **Note**: Keep in mind that the networking has to know an address for this node, /// > otherwise it will not be able to connect to it. pub fn add_reserved_peer(&self, set_id: SetId, peer_id: PeerId) { - let _ = self.tx.unbounded_send(Action::AddReservedPeer(set_id, peer_id)); + self.protocol_handles[set_id.0].add_reserved_peer(peer_id); } /// Remove a previously-added reserved peer. /// /// Has no effect if the node was not a reserved peer. pub fn remove_reserved_peer(&self, set_id: SetId, peer_id: PeerId) { - let _ = self.tx.unbounded_send(Action::RemoveReservedPeer(set_id, peer_id)); + self.protocol_handles[set_id.0].remove_reserved_peer(peer_id); } /// Sets whether or not the peerset only has connections with nodes marked as reserved for /// the given set. pub fn set_reserved_only(&self, set_id: SetId, reserved: bool) { - let _ = self.tx.unbounded_send(Action::SetReservedOnly(set_id, reserved)); + self.protocol_handles[set_id.0].set_reserved_only(reserved); } /// Set reserved peers to the new set. pub fn set_reserved_peers(&self, set_id: SetId, peer_ids: HashSet) { - let _ = self.tx.unbounded_send(Action::SetReservedPeers(set_id, peer_ids)); + self.protocol_handles[set_id.0].set_reserved_peers(peer_ids); } /// Reports an adjustment to the reputation of the given peer. pub fn report_peer(&self, peer_id: PeerId, score_diff: ReputationChange) { - let _ = self.tx.unbounded_send(Action::ReportPeer(peer_id, score_diff)); + self.peer_store_handle.report_peer(peer_id, score_diff); } /// Add a peer to a set. - pub fn add_to_peers_set(&self, set_id: SetId, peer_id: PeerId) { - let _ = self.tx.unbounded_send(Action::AddToPeersSet(set_id, peer_id)); - } + //pub fn add_to_peers_set(&self, set_id: SetId, peer_id: PeerId) { + // let _ = self.tx.unbounded_send(Action::AddToPeersSet(set_id, peer_id)); + //} /// Remove a peer from a set. - pub fn remove_from_peers_set(&self, set_id: SetId, peer_id: PeerId) { - let _ = self.tx.unbounded_send(Action::RemoveFromPeersSet(set_id, peer_id)); - } + //pub fn remove_from_peers_set(&self, set_id: SetId, peer_id: PeerId) { + // let _ = self.tx.unbounded_send(Action::RemoveFromPeersSet(set_id, peer_id)); + //} /// Returns the reputation value of the peer. pub async fn peer_reputation(self, peer_id: PeerId) -> Result { - let (tx, rx) = oneshot::channel(); - - let _ = self.tx.unbounded_send(Action::PeerReputation(peer_id, tx)); - - // The channel can only be closed if the peerset no longer exists. - rx.await.map_err(|_| ()) + Ok(self.peer_store_handle.peer_reputation(&peer_id)) + // TODO: convert into sync function. } } @@ -251,473 +233,131 @@ pub struct SetConfig { /// /// Implements the `Stream` trait and can be polled for messages. The `Stream` never ends and never /// errors. -#[derive(Debug)] pub struct Peerset { - /// Underlying data structure for the nodes's states. - data: peersstate::PeersState, - /// For each set, lists of nodes that don't occupy slots and that we should try to always be - /// connected to, and whether only reserved nodes are accepted. Is kept in sync with the list - /// of non-slot-occupying nodes in [`Peerset::data`]. - reserved_nodes: Vec<(HashSet, bool)>, - /// Receiver for messages from the `PeersetHandle` and from `tx`. - rx: TracingUnboundedReceiver, - /// Sending side of `rx`. - tx: TracingUnboundedSender, - /// Queue of messages to be emitted when the `Peerset` is polled. - message_queue: VecDeque, - /// When the `Peerset` was created. - created: Instant, - /// Last time when we updated the reputations of connected nodes. - latest_time_update: Instant, - /// Next time to do a periodic call to `alloc_slots` with all sets. This is done once per - /// second, to match the period of the reputation updates. - next_periodic_alloc_slots: Delay, + /// Peer reputation store. + peer_store: PeerStore, + /// Protocol handles. + protocol_handles: Vec, + /// Protocol controllers responsible for connections, per `SetId`. + protocol_controllers: Vec>, + /// Commands sent from protocol controllers to `Notifications`. The size of this vector never + /// changes. + rx: TracingUnboundedReceiver, } impl Peerset { /// Builds a new peerset from the given configuration. pub fn from_config(config: PeersetConfig) -> (Self, PeersetHandle) { - let (tx, rx) = tracing_unbounded("mpsc_peerset_messages", 10_000); - - let handle = PeersetHandle { tx: tx.clone() }; - - let mut peerset = { - let now = Instant::now(); - - Self { - data: peersstate::PeersState::new(config.sets.iter().map(|set| { - peersstate::SetConfig { in_peers: set.in_peers, out_peers: set.out_peers } - })), - tx, - rx, - reserved_nodes: config - .sets - .iter() - .map(|set| (set.reserved_nodes.clone(), set.reserved_only)) - .collect(), - message_queue: VecDeque::new(), - created: now, - latest_time_update: now, - next_periodic_alloc_slots: Delay::new(Duration::new(0, 0)), - } + let default_set_config = &config.sets[0]; + + let peer_store = PeerStore::new(default_set_config.bootnodes); + + let (tx, rx) = tracing_unbounded("mpsc_to_notifications", 10_000); + + let controllers = config + .sets + .iter() + .enumerate() + .map(|(set, set_config)| { + ProtocolController::new( + SetId::from(set), + *set_config, + tx.clone(), + peer_store.handle(), + ) + }) + .collect::>(); + + let (protocol_handles, protocol_controllers): (Vec, Vec<_>) = + controllers.into_iter().unzip(); + + let handle = PeersetHandle { + peer_store_handle: peer_store.handle(), + protocol_handles: protocol_handles.clone(), }; - for (set, set_config) in config.sets.into_iter().enumerate() { - for node in set_config.reserved_nodes { - peerset.data.add_no_slot_node(set, node); - } - - for peer_id in set_config.bootnodes { - if let peersstate::Peer::Unknown(entry) = peerset.data.peer(set, &peer_id) { - entry.discover(); - } else { - debug!(target: "peerset", "Duplicate bootnode in config: {:?}", peer_id); - } - } - } - - for set_index in 0..peerset.data.num_sets() { - peerset.alloc_slots(SetId(set_index)); - } + let peerset = Peerset { peer_store, protocol_handles, protocol_controllers, rx }; (peerset, handle) } - fn on_add_reserved_peer(&mut self, set_id: SetId, peer_id: PeerId) { - let newly_inserted = self.reserved_nodes[set_id.0].0.insert(peer_id); - if !newly_inserted { - return - } - - self.data.add_no_slot_node(set_id.0, peer_id); - self.alloc_slots(set_id); - } - - fn on_remove_reserved_peer(&mut self, set_id: SetId, peer_id: PeerId) { - if !self.reserved_nodes[set_id.0].0.remove(&peer_id) { - return - } - - self.data.remove_no_slot_node(set_id.0, &peer_id); - - // Nothing more to do if not in reserved-only mode. - if !self.reserved_nodes[set_id.0].1 { - return - } - - // If, however, the peerset is in reserved-only mode, then the removed node needs to be - // disconnected. - if let peersstate::Peer::Connected(peer) = self.data.peer(set_id.0, &peer_id) { - peer.disconnect(); - self.message_queue.push_back(Message::Drop { set_id, peer_id }); - } - } - - fn on_set_reserved_peers(&mut self, set_id: SetId, peer_ids: HashSet) { - // Determine the difference between the current group and the new list. - let (to_insert, to_remove) = { - let to_insert = peer_ids - .difference(&self.reserved_nodes[set_id.0].0) - .cloned() - .collect::>(); - let to_remove = self.reserved_nodes[set_id.0] - .0 - .difference(&peer_ids) - .cloned() - .collect::>(); - (to_insert, to_remove) - }; - - for node in to_insert { - self.on_add_reserved_peer(set_id, node); - } - - for node in to_remove { - self.on_remove_reserved_peer(set_id, node); - } - } - - fn on_set_reserved_only(&mut self, set_id: SetId, reserved_only: bool) { - self.reserved_nodes[set_id.0].1 = reserved_only; - - if reserved_only { - // Disconnect all the nodes that aren't reserved. - for peer_id in - self.data.connected_peers(set_id.0).cloned().collect::>().into_iter() - { - if self.reserved_nodes[set_id.0].0.contains(&peer_id) { - continue - } - - let peer = self.data.peer(set_id.0, &peer_id).into_connected().expect( - "We are enumerating connected peers, therefore the peer is connected; qed", - ); - peer.disconnect(); - self.message_queue.push_back(Message::Drop { set_id, peer_id }); - } - } else { - self.alloc_slots(set_id); - } - } - /// Returns the list of reserved peers. - pub fn reserved_peers(&self, set_id: SetId) -> impl Iterator { - self.reserved_nodes[set_id.0].0.iter() + pub fn reserved_peers(&self, set_id: SetId, pending_response: oneshot::Sender>) { + self.protocol_handles[set_id.0].reserved_peers(pending_response); } /// Adds a node to the given set. The peerset will, if possible and not already the case, /// try to connect to it. /// /// > **Note**: This has the same effect as [`PeersetHandle::add_to_peers_set`]. - pub fn add_to_peers_set(&mut self, set_id: SetId, peer_id: PeerId) { - if let peersstate::Peer::Unknown(entry) = self.data.peer(set_id.0, &peer_id) { - entry.discover(); - self.alloc_slots(set_id); - } - } - - fn on_remove_from_peers_set(&mut self, set_id: SetId, peer_id: PeerId) { - // Don't do anything if node is reserved. - if self.reserved_nodes[set_id.0].0.contains(&peer_id) { - return - } - - match self.data.peer(set_id.0, &peer_id) { - peersstate::Peer::Connected(peer) => { - self.message_queue.push_back(Message::Drop { set_id, peer_id: *peer.peer_id() }); - peer.disconnect().forget_peer(); - }, - peersstate::Peer::NotConnected(peer) => { - peer.forget_peer(); - }, - peersstate::Peer::Unknown(_) => {}, - } - } - - fn on_report_peer(&mut self, peer_id: PeerId, change: ReputationChange) { - // We want reputations to be up-to-date before adjusting them. - self.update_time(); - - let mut reputation = self.data.peer_reputation(peer_id); - reputation.add_reputation(change.value); - if reputation.reputation() >= BANNED_THRESHOLD { - trace!(target: "peerset", "Report {}: {:+} to {}. Reason: {}", - peer_id, change.value, reputation.reputation(), change.reason - ); - return - } - - debug!(target: "peerset", "Report {}: {:+} to {}. Reason: {}, Disconnecting", - peer_id, change.value, reputation.reputation(), change.reason - ); - - drop(reputation); - - for set_index in 0..self.data.num_sets() { - if let peersstate::Peer::Connected(peer) = self.data.peer(set_index, &peer_id) { - let peer = peer.disconnect(); - self.message_queue.push_back(Message::Drop { - set_id: SetId(set_index), - peer_id: peer.into_peer_id(), - }); - - self.alloc_slots(SetId(set_index)); - } - } - } - - fn on_peer_reputation(&mut self, peer_id: PeerId, pending_response: oneshot::Sender) { - let reputation = self.data.peer_reputation(peer_id); - let _ = pending_response.send(reputation.reputation()); - } - - /// Updates the value of `self.latest_time_update` and performs all the updates that happen - /// over time, such as reputation increases for staying connected. - fn update_time(&mut self) { - let now = Instant::now(); - - // We basically do `(now - self.latest_update).as_secs()`, except that by the way we do it - // we know that we're not going to miss seconds because of rounding to integers. - let secs_diff = { - let elapsed_latest = self.latest_time_update - self.created; - let elapsed_now = now - self.created; - self.latest_time_update = now; - elapsed_now.as_secs() - elapsed_latest.as_secs() - }; - - // For each elapsed second, move the node reputation towards zero. - // If we multiply each second the reputation by `k` (where `k` is between 0 and 1), it - // takes `ln(0.5) / ln(k)` seconds to reduce the reputation by half. Use this formula to - // empirically determine a value of `k` that looks correct. - for _ in 0..secs_diff { - for peer_id in self.data.peers().cloned().collect::>() { - // We use `k = 0.98`, so we divide by `50`. With that value, it takes 34.3 seconds - // to reduce the reputation by half. - fn reput_tick(reput: i32) -> i32 { - let mut diff = reput / 50; - if diff == 0 && reput < 0 { - diff = -1; - } else if diff == 0 && reput > 0 { - diff = 1; - } - reput.saturating_sub(diff) - } - - let mut peer_reputation = self.data.peer_reputation(peer_id); - - let before = peer_reputation.reputation(); - let after = reput_tick(before); - trace!(target: "peerset", "Fleeting {}: {} -> {}", peer_id, before, after); - peer_reputation.set_reputation(after); - - if after != 0 { - continue - } - - drop(peer_reputation); - - // If the peer reaches a reputation of 0, and there is no connection to it, - // forget it. - for set_index in 0..self.data.num_sets() { - match self.data.peer(set_index, &peer_id) { - peersstate::Peer::Connected(_) => {}, - peersstate::Peer::NotConnected(peer) => { - if peer.last_connected_or_discovered() + FORGET_AFTER < now { - peer.forget_peer(); - } - }, - peersstate::Peer::Unknown(_) => { - // Happens if this peer does not belong to this set. - }, - } - } - } - } - } - - /// Try to fill available out slots with nodes for the given set. - fn alloc_slots(&mut self, set_id: SetId) { - self.update_time(); - - // Try to connect to all the reserved nodes that we are not connected to. - for reserved_node in &self.reserved_nodes[set_id.0].0 { - let entry = match self.data.peer(set_id.0, reserved_node) { - peersstate::Peer::Unknown(n) => n.discover(), - peersstate::Peer::NotConnected(n) => n, - peersstate::Peer::Connected(_) => continue, - }; - - // Don't connect to nodes with an abysmal reputation, even if they're reserved. - // This is a rather opinionated behaviour, and it wouldn't be fundamentally wrong to - // remove that check. If necessary, the peerset should be refactored to give more - // control over what happens in that situation. - if entry.reputation() < BANNED_THRESHOLD { - break - } - - match entry.try_outgoing() { - Ok(conn) => self - .message_queue - .push_back(Message::Connect { set_id, peer_id: conn.into_peer_id() }), - Err(_) => { - // An error is returned only if no slot is available. Reserved nodes are - // marked in the state machine with a flag saying "doesn't occupy a slot", - // and as such this should never happen. - debug_assert!(false); - log::error!( - target: "peerset", - "Not enough slots to connect to reserved node" - ); - }, - } - } - - // Now, we try to connect to other nodes. - - // Nothing more to do if we're in reserved mode. - if self.reserved_nodes[set_id.0].1 { - return - } - - // Try to grab the next node to attempt to connect to. - // Since `highest_not_connected_peer` is rather expensive to call, check beforehand - // whether we have an available slot. - while self.data.has_free_outgoing_slot(set_id.0) { - let next = match self.data.highest_not_connected_peer(set_id.0) { - Some(n) => n, - None => break, - }; - - // Don't connect to nodes with an abysmal reputation. - if next.reputation() < BANNED_THRESHOLD { - break - } - - match next.try_outgoing() { - Ok(conn) => self - .message_queue - .push_back(Message::Connect { set_id, peer_id: conn.into_peer_id() }), - Err(_) => { - // This branch can only be entered if there is no free slot, which is - // checked above. - debug_assert!(false); - break - }, - } - } - } - - /// Indicate that we received an incoming connection. Must be answered either with - /// a corresponding `Accept` or `Reject`, except if we were already connected to this peer. - /// - /// Note that this mechanism is orthogonal to `Connect`/`Drop`. Accepting an incoming - /// connection implicitly means `Connect`, but incoming connections aren't cancelled by - /// `dropped`. - // Implementation note: because of concurrency issues, it is possible that we push a `Connect` - // message to the output channel with a `PeerId`, and that `incoming` gets called with the same - // `PeerId` before that message has been read by the user. In this situation we must not answer. - pub fn incoming(&mut self, set_id: SetId, peer_id: PeerId, index: IncomingIndex) { - trace!(target: "peerset", "Incoming {:?}", peer_id); - - self.update_time(); - - if self.reserved_nodes[set_id.0].1 && !self.reserved_nodes[set_id.0].0.contains(&peer_id) { - self.message_queue.push_back(Message::Reject(index)); - return - } - - let not_connected = match self.data.peer(set_id.0, &peer_id) { - // If we're already connected, don't answer, as the docs mention. - peersstate::Peer::Connected(_) => return, - peersstate::Peer::NotConnected(mut entry) => { - entry.bump_last_connected_or_discovered(); - entry - }, - peersstate::Peer::Unknown(entry) => entry.discover(), - }; - - if not_connected.reputation() < BANNED_THRESHOLD { - self.message_queue.push_back(Message::Reject(index)); - return - } - - match not_connected.try_accept_incoming() { - Ok(_) => self.message_queue.push_back(Message::Accept(index)), - Err(_) => self.message_queue.push_back(Message::Reject(index)), - } - } - - /// Indicate that we dropped an active connection with a peer, or that we failed to connect. - /// - /// Must only be called after the PSM has either generated a `Connect` message with this - /// `PeerId`, or accepted an incoming connection with this `PeerId`. - pub fn dropped(&mut self, set_id: SetId, peer_id: PeerId, reason: DropReason) { - // We want reputations to be up-to-date before adjusting them. - self.update_time(); - - match self.data.peer(set_id.0, &peer_id) { - peersstate::Peer::Connected(mut entry) => { - // Decrease the node's reputation so that we don't try it again and again and again. - entry.add_reputation(DISCONNECT_REPUTATION_CHANGE); - trace!(target: "peerset", "Dropping {}: {:+} to {}", - peer_id, DISCONNECT_REPUTATION_CHANGE, entry.reputation()); - entry.disconnect(); - }, - peersstate::Peer::NotConnected(_) | peersstate::Peer::Unknown(_) => { - error!(target: "peerset", "Received dropped() for non-connected node") - }, - } - - if let DropReason::Refused = reason { - self.on_remove_from_peers_set(set_id, peer_id); - } - - self.alloc_slots(set_id); - } + // pub fn add_to_peers_set(&mut self, set_id: SetId, peer_id: PeerId) { + // if let peersstate::Peer::Unknown(entry) = self.data.peer(set_id.0, &peer_id) { + // entry.discover(); + // self.alloc_slots(set_id); + // } + // } + + // fn on_remove_from_peers_set(&mut self, set_id: SetId, peer_id: PeerId) { + // // Don't do anything if node is reserved. + // if self.reserved_nodes[set_id.0].0.contains(&peer_id) { + // return + // } + + // match self.data.peer(set_id.0, &peer_id) { + // peersstate::Peer::Connected(peer) => { + // self.message_queue.push_back(Message::Drop { set_id, peer_id: *peer.peer_id() }); + // peer.disconnect().forget_peer(); + // }, + // peersstate::Peer::NotConnected(peer) => { + // peer.forget_peer(); + // }, + // peersstate::Peer::Unknown(_) => {}, + // } + // } /// Reports an adjustment to the reputation of the given peer. pub fn report_peer(&mut self, peer_id: PeerId, score_diff: ReputationChange) { // We don't immediately perform the adjustments in order to have state consistency. We // don't want the reporting here to take priority over messages sent using the // `PeersetHandle`. - let _ = self.tx.unbounded_send(Action::ReportPeer(peer_id, score_diff)); + let _ = self.peer_store.handle().report_peer(peer_id, score_diff); } /// Produces a JSON object containing the state of the peerset manager, for debugging purposes. - pub fn debug_info(&mut self) -> serde_json::Value { - self.update_time(); - - json!({ - "sets": (0..self.data.num_sets()).map(|set_index| { - json!({ - "nodes": self.data.peers().cloned().collect::>().into_iter().filter_map(|peer_id| { - let state = match self.data.peer(set_index, &peer_id) { - peersstate::Peer::Connected(entry) => json!({ - "connected": true, - "reputation": entry.reputation() - }), - peersstate::Peer::NotConnected(entry) => json!({ - "connected": false, - "reputation": entry.reputation() - }), - peersstate::Peer::Unknown(_) => return None, - }; - - Some((peer_id.to_base58(), state)) - }).collect::>(), - "reserved_nodes": self.reserved_nodes[set_index].0.iter().map(|peer_id| { - peer_id.to_base58() - }).collect::>(), - "reserved_only": self.reserved_nodes[set_index].1, - }) - }).collect::>(), - "message_queue": self.message_queue.len(), - }) - } + // pub fn debug_info(&mut self) -> serde_json::Value { + // self.update_time(); + + // json!({ + // "sets": (0..self.data.num_sets()).map(|set_index| { + // json!({ + // "nodes": self.data.peers().cloned().collect::>().into_iter().filter_map(|peer_id| { + // let state = match self.data.peer(set_index, &peer_id) { + // peersstate::Peer::Connected(entry) => json!({ + // "connected": true, + // "reputation": entry.reputation() + // }), + // peersstate::Peer::NotConnected(entry) => json!({ + // "connected": false, + // "reputation": entry.reputation() + // }), + // peersstate::Peer::Unknown(_) => return None, + // }; + + // Some((peer_id.to_base58(), state)) + // }).collect::>(), + // "reserved_nodes": self.reserved_nodes[set_index].0.iter().map(|peer_id| { + // peer_id.to_base58() + // }).collect::>(), + // "reserved_only": self.reserved_nodes[set_index].1, + // }) + // }).collect::>(), + // "message_queue": self.message_queue.len(), + // }) + // } /// Returns the number of peers that we have discovered. pub fn num_discovered_peers(&self) -> usize { - self.data.peers().len() + self.peer_store.handle().num_known_peers() } } @@ -777,6 +417,7 @@ pub enum DropReason { Refused, } +/* #[cfg(test)] mod tests { use super::{ @@ -1000,3 +641,4 @@ mod tests { futures::executor::block_on(fut); } } +*/ diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index ff3f99bf34513..d7fb32a359647 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -20,7 +20,7 @@ use libp2p::PeerId; use partial_sort::PartialSort; use std::{ cmp::{Ord, Ordering, PartialOrd}, - collections::{hash_map::Entry, HashMap, HashSet}, + collections::{HashMap, HashSet}, sync::{Arc, Mutex}, time::{Duration, Instant}, }; @@ -59,6 +59,7 @@ pub trait PeerReputationProvider { fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec; } +#[derive(Debug, Clone)] pub struct PeerStoreHandle { inner: Arc>, } @@ -85,6 +86,14 @@ impl PeerReputationProvider for PeerStoreHandle { } } +impl PeerStoreHandle { + /// Get the number of known peers. + pub fn num_known_peers(&self) -> usize { + // FIXME: how do we use this info? May be better ask for numbers from protocol controllers? + self.inner.lock().unwrap().peers.len() + } +} + #[derive(Debug, Clone, Copy)] struct PeerInfo { reputation: i32, @@ -129,7 +138,7 @@ impl PeerInfo { } fn decay_reputation(&mut self, seconds_passed: u64) { - // Note that decaying a reputation value happens "on its own", + // Note that decaying the reputation value happens "on its own", // so we don't bump `last_updated`. for _ in 0..seconds_passed { let mut diff = self.reputation / INVERSE_DECREMENT; @@ -148,6 +157,7 @@ impl PeerInfo { } } +#[derive(Debug)] struct PeerStoreInner { peers: HashMap, } @@ -207,7 +217,8 @@ impl PeerStoreInner { } } -struct PeerStore { +#[derive(Debug)] +pub struct PeerStore { inner: Arc>, } diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 418e1b3fdb2ac..6df6d4b43ed5b 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -24,7 +24,7 @@ // TODO: remove this line. #![allow(unused)] -use futures::{FutureExt, StreamExt}; +use futures::{channel::oneshot, FutureExt, StreamExt}; use libp2p::PeerId; use log::{error, info, trace}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; @@ -49,10 +49,12 @@ enum Action { DisconnectPeer(PeerId), IncomingConnection(PeerId, IncomingIndex), Dropped(PeerId), + ReservedPeers(oneshot::Sender>), } /// Shared handle to [`ProtocolController`]. Distributed around the code outside of the /// protocol implementation. +#[derive(Debug, Clone)] pub struct ProtocolHandle { to_controller: TracingUnboundedSender, } @@ -100,10 +102,15 @@ impl ProtocolHandle { .unbounded_send(Action::IncomingConnection(peer_id, incoming_index)); } - /// Notify that connection was dropped (eithere refused or disconnected). + /// Notify that connection was dropped (either refused or disconnected). pub fn dropped(&self, peer_id: PeerId) { let _ = self.to_controller.unbounded_send(Action::Dropped(peer_id)); } + + /// Get the list of reserved peers. + pub fn reserved_peers(&self, pending_response: oneshot::Sender>) { + let _ = self.to_controller.unbounded_send(Action::ReservedPeers(pending_response)); + } } /// Direction of a connection @@ -236,6 +243,7 @@ impl ProtocolController self.on_reserved_peers(pending_response), } true } @@ -360,6 +368,11 @@ impl ProtocolController>) { + pending_response.send(self.reserved_nodes.keys().cloned().collect()); + } + /// Disconnect the peer. fn on_disconnect_peer(&mut self, peer_id: PeerId) { // Don't do anything if the node is reserved. From 086fb5b0fe415f0441a9bb84cf2b60cd4b6aaab9 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 4 Apr 2023 18:00:46 +0300 Subject: [PATCH 40/98] WIP: implement `PeerStore` and `ProtocolController` polling via `Peerset` --- client/network/src/protocol.rs | 4 +- .../src/protocol/notifications/behaviour.rs | 13 +- client/network/src/service.rs | 10 +- client/peerset/src/lib.rs | 131 +++++++----------- client/peerset/src/protocol_controller.rs | 6 +- 5 files changed, 73 insertions(+), 91 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index 06ca02c0ca8d5..b894179620c4e 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -247,8 +247,8 @@ impl Protocol { } /// Returns the list of reserved peers. - pub fn reserved_peers(&self) -> impl Iterator { - self.behaviour.reserved_peers(HARDCODED_PEERSETS_SYNC) + pub fn reserved_peers(&self, pending_response: oneshot::Sender>) { + self.behaviour.reserved_peers(HARDCODED_PEERSETS_SYNC, pending_response); } /// Adds a `PeerId` to the list of reserved peers for syncing purposes. diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 74e27fa17c602..31b1d46ebc757 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -25,7 +25,7 @@ use crate::{ use bytes::BytesMut; use fnv::FnvHashMap; -use futures::prelude::*; +use futures::{channel::oneshot, prelude::*}; use libp2p::{ core::{connection::ConnectionId, Multiaddr, PeerId}, swarm::{ @@ -536,8 +536,12 @@ impl Notifications { } /// Returns the list of reserved peers. - pub fn reserved_peers(&self, set_id: sc_peerset::SetId) -> impl Iterator { - self.peerset.reserved_peers(set_id) + pub fn reserved_peers( + &self, + set_id: sc_peerset::SetId, + pending_response: oneshot::Sender>, + ) { + self.peerset.reserved_peers(set_id, pending_response); } /// Returns the state of the peerset manager, for debugging purposes. @@ -2928,7 +2932,8 @@ mod tests { // check peer information assert_eq!(notif.open_peers().collect::>(), vec![&peer],); - assert_eq!(notif.reserved_peers(set_id).collect::>(), Vec::<&PeerId>::new(),); + // assert_eq!(notif.reserved_peers(set_id).collect::>(), Vec::<&PeerId>::new(),); + todo!("Do we really need to check reserved peers here?"); assert_eq!(notif.num_discovered_peers(), 0usize); // close the other connection and verify that notification replacement event is emitted diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 1abca902aecdb..5a23f0e7d66a2 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -615,8 +615,11 @@ where } /// Returns the list of reserved peers. - fn reserved_peers(&self) -> impl Iterator { - self.network_service.behaviour().user_protocol().reserved_peers() + fn reserved_peers(&self, pending_response: oneshot::Sender>) { + self.network_service + .behaviour() + .user_protocol() + .reserved_peers(pending_response); } } @@ -1344,8 +1347,7 @@ where .user_protocol_mut() .set_notification_handshake(protocol, handshake), ServiceToWorkerMsg::ReservedPeers { pending_response } => { - let _ = - pending_response.send(self.reserved_peers().map(ToOwned::to_owned).collect()); + self.reserved_peers(pending_response); }, } } diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index aa0f33d528626..f412a5c804420 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -39,7 +39,12 @@ mod protocol_controller; use peer_store::{PeerReputationProvider, PeerStore, PeerStoreHandle}; use protocol_controller::{ProtocolController, ProtocolHandle}; -use futures::{channel::oneshot, prelude::*}; +use futures::{ + channel::oneshot, + future::{join_all, Future, JoinAll}, + prelude::*, + stream::Stream, +}; use log::{debug, error, trace}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use serde_json::json; @@ -141,7 +146,7 @@ impl PeersetHandle { } /// Reports an adjustment to the reputation of the given peer. - pub fn report_peer(&self, peer_id: PeerId, score_diff: ReputationChange) { + pub fn report_peer(&mut self, peer_id: PeerId, score_diff: ReputationChange) { self.peer_store_handle.report_peer(peer_id, score_diff); } @@ -234,12 +239,14 @@ pub struct SetConfig { /// Implements the `Stream` trait and can be polled for messages. The `Stream` never ends and never /// errors. pub struct Peerset { + /// Peer reputation store handle. + peer_store_handle: PeerStoreHandle, /// Peer reputation store. - peer_store: PeerStore, + peer_store_future: Pin + Send>>, /// Protocol handles. protocol_handles: Vec, /// Protocol controllers responsible for connections, per `SetId`. - protocol_controllers: Vec>, + protocol_controller_futures: JoinAll + Send>>>, /// Commands sent from protocol controllers to `Notifications`. The size of this vector never /// changes. rx: TracingUnboundedReceiver, @@ -247,21 +254,20 @@ pub struct Peerset { impl Peerset { /// Builds a new peerset from the given configuration. - pub fn from_config(config: PeersetConfig) -> (Self, PeersetHandle) { + pub fn from_config(config: PeersetConfig) -> (Peerset, PeersetHandle) { let default_set_config = &config.sets[0]; + let peer_store = PeerStore::new(default_set_config.bootnodes.clone()); - let peer_store = PeerStore::new(default_set_config.bootnodes); - - let (tx, rx) = tracing_unbounded("mpsc_to_notifications", 10_000); + let (tx, rx) = tracing_unbounded("mpsc_protocol_controllers_to_notifications", 10_000); let controllers = config .sets - .iter() + .into_iter() .enumerate() .map(|(set, set_config)| { ProtocolController::new( SetId::from(set), - *set_config, + set_config, tx.clone(), peer_store.handle(), ) @@ -276,7 +282,16 @@ impl Peerset { protocol_handles: protocol_handles.clone(), }; - let peerset = Peerset { peer_store, protocol_handles, protocol_controllers, rx }; + let protocol_controller_futures = + join_all(protocol_controllers.into_iter().map(|c| c.run().boxed())); + + let peerset = Peerset { + peer_store_handle: peer_store.handle(), + peer_store_future: peer_store.run().boxed(), + protocol_handles, + protocol_controller_futures, + rx, + }; (peerset, handle) } @@ -320,44 +335,12 @@ impl Peerset { // We don't immediately perform the adjustments in order to have state consistency. We // don't want the reporting here to take priority over messages sent using the // `PeersetHandle`. - let _ = self.peer_store.handle().report_peer(peer_id, score_diff); + let _ = self.peer_store_handle.report_peer(peer_id, score_diff); } - /// Produces a JSON object containing the state of the peerset manager, for debugging purposes. - // pub fn debug_info(&mut self) -> serde_json::Value { - // self.update_time(); - - // json!({ - // "sets": (0..self.data.num_sets()).map(|set_index| { - // json!({ - // "nodes": self.data.peers().cloned().collect::>().into_iter().filter_map(|peer_id| { - // let state = match self.data.peer(set_index, &peer_id) { - // peersstate::Peer::Connected(entry) => json!({ - // "connected": true, - // "reputation": entry.reputation() - // }), - // peersstate::Peer::NotConnected(entry) => json!({ - // "connected": false, - // "reputation": entry.reputation() - // }), - // peersstate::Peer::Unknown(_) => return None, - // }; - - // Some((peer_id.to_base58(), state)) - // }).collect::>(), - // "reserved_nodes": self.reserved_nodes[set_index].0.iter().map(|peer_id| { - // peer_id.to_base58() - // }).collect::>(), - // "reserved_only": self.reserved_nodes[set_index].1, - // }) - // }).collect::>(), - // "message_queue": self.message_queue.len(), - // }) - // } - /// Returns the number of peers that we have discovered. pub fn num_discovered_peers(&self) -> usize { - self.peer_store.handle().num_known_peers() + self.peer_store_handle.num_known_peers() } } @@ -365,43 +348,35 @@ impl Stream for Peerset { type Item = Message; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - loop { - if let Some(message) = self.message_queue.pop_front() { - return Poll::Ready(Some(message)) + if let Poll::Ready(msg) = self.rx.poll_next_unpin(cx) { + if let Some(msg) = msg { + return Poll::Ready(Some(msg)) + } else { + debug!( + target: "peerset", + "All `ProtocolController`s have terminated, terminating `Peerset`." + ); + return Poll::Ready(None) } + } - if Future::poll(Pin::new(&mut self.next_periodic_alloc_slots), cx).is_ready() { - self.next_periodic_alloc_slots = Delay::new(Duration::new(1, 0)); - - for set_index in 0..self.data.num_sets() { - self.alloc_slots(SetId(set_index)); - } - } + if let Poll::Ready(()) = self.peer_store_future.poll_unpin(cx) { + debug!( + target: "peerset", + "`PeerStore` has terminated, terminating `PeerSet`." + ); + return Poll::Ready(None) + } - let action = match Stream::poll_next(Pin::new(&mut self.rx), cx) { - Poll::Pending => return Poll::Pending, - Poll::Ready(Some(event)) => event, - Poll::Ready(None) => return Poll::Pending, - }; - - match action { - Action::AddReservedPeer(set_id, peer_id) => - self.on_add_reserved_peer(set_id, peer_id), - Action::RemoveReservedPeer(set_id, peer_id) => - self.on_remove_reserved_peer(set_id, peer_id), - Action::SetReservedPeers(set_id, peer_ids) => - self.on_set_reserved_peers(set_id, peer_ids), - Action::SetReservedOnly(set_id, reserved) => - self.on_set_reserved_only(set_id, reserved), - Action::ReportPeer(peer_id, score_diff) => self.on_report_peer(peer_id, score_diff), - Action::AddToPeersSet(sets_name, peer_id) => - self.add_to_peers_set(sets_name, peer_id), - Action::RemoveFromPeersSet(sets_name, peer_id) => - self.on_remove_from_peers_set(sets_name, peer_id), - Action::PeerReputation(peer_id, pending_response) => - self.on_peer_reputation(peer_id, pending_response), - } + if let Poll::Ready(_) = self.protocol_controller_futures.poll_unpin(cx) { + debug!( + target: "peerset", + "All `ProtocolHandle`s have terminated, terminating `PeerSet`." + ); + return Poll::Ready(None) } + + Poll::Pending } } diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 6df6d4b43ed5b..e08e67c176e7c 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -49,7 +49,7 @@ enum Action { DisconnectPeer(PeerId), IncomingConnection(PeerId, IncomingIndex), Dropped(PeerId), - ReservedPeers(oneshot::Sender>), + ReservedPeers { pending_response: oneshot::Sender> }, } /// Shared handle to [`ProtocolController`]. Distributed around the code outside of the @@ -109,7 +109,7 @@ impl ProtocolHandle { /// Get the list of reserved peers. pub fn reserved_peers(&self, pending_response: oneshot::Sender>) { - let _ = self.to_controller.unbounded_send(Action::ReservedPeers(pending_response)); + let _ = self.to_controller.unbounded_send(Action::ReservedPeers { pending_response }); } } @@ -243,7 +243,7 @@ impl ProtocolController self.on_reserved_peers(pending_response), + Action::ReservedPeers { pending_response } => self.on_reserved_peers(pending_response), } true } From 7e370d96bf0ea0a7e6c361f13b4e7143da4e709d Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 5 Apr 2023 16:30:56 +0300 Subject: [PATCH 41/98] Apply suggestions from code review Co-authored-by: Anton --- client/peerset/src/protocol_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index e08e67c176e7c..9eb5fff2f75fd 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -389,7 +389,7 @@ impl ProtocolController { - trace!( + error!( target: "peerset", "Trying to remove unknown peer {} from {:?}", peer_id, self.set_id, From 64ee662aa264906a4a3086f0adc91ec0beb78f2c Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 5 Apr 2023 17:08:54 +0300 Subject: [PATCH 42/98] minor: improve error reporting --- client/peerset/src/protocol_controller.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 9eb5fff2f75fd..d38126cf7232a 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -26,7 +26,7 @@ use futures::{channel::oneshot, FutureExt, StreamExt}; use libp2p::PeerId; -use log::{error, info, trace}; +use log::{error, info, trace, warn}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_arithmetic::traits::SaturatedConversion; use std::{ @@ -377,6 +377,11 @@ impl ProtocolController ProtocolController { - error!( + warn!( target: "peerset", - "Trying to remove unknown peer {} from {:?}", + "Trying to disconnect unknown peer {} from {:?}", peer_id, self.set_id, ); }, From c9109d76bb288bd51300bc91d480a8d310c179d1 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 5 Apr 2023 17:21:11 +0300 Subject: [PATCH 43/98] Apply review suggestions --- client/peerset/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index f412a5c804420..3aea6d11cdb93 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -41,7 +41,7 @@ use protocol_controller::{ProtocolController, ProtocolHandle}; use futures::{ channel::oneshot, - future::{join_all, Future, JoinAll}, + future::{join_all, JoinAll, BoxFuture}, prelude::*, stream::Stream, }; @@ -242,11 +242,11 @@ pub struct Peerset { /// Peer reputation store handle. peer_store_handle: PeerStoreHandle, /// Peer reputation store. - peer_store_future: Pin + Send>>, + peer_store_future: BoxFuture<'static, ()>, /// Protocol handles. protocol_handles: Vec, /// Protocol controllers responsible for connections, per `SetId`. - protocol_controller_futures: JoinAll + Send>>>, + protocol_controller_futures: JoinAll>, /// Commands sent from protocol controllers to `Notifications`. The size of this vector never /// changes. rx: TracingUnboundedReceiver, From 8677480d6204245718dd73ab1210db1329f0c202 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 7 Apr 2023 18:05:33 +0300 Subject: [PATCH 44/98] WIP: update `PeerSet` use to match `PeerStore` and `ProtocolController` interfaces --- client/network/src/protocol.rs | 37 +------- .../src/protocol/notifications/behaviour.rs | 6 +- client/network/src/service.rs | 94 +++++++++---------- client/network/src/service/traits.rs | 34 +++---- client/peerset/src/lib.rs | 45 +++++---- client/peerset/src/peer_store.rs | 35 ++++++- 6 files changed, 129 insertions(+), 122 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index b894179620c4e..bddf614016517 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -42,6 +42,7 @@ use std::{ iter, task::Poll, }; +use futures::channel::oneshot; use message::{generic::Message as GenericMessage, Message}; use notifications::{Notifications, NotificationsOut}; @@ -300,39 +301,11 @@ impl Protocol { } } - /// Notify the protocol that we have learned about the existence of nodes on the default set. + /// Notify the protocol that we have learned about the existence of some peer. /// - /// Can be called multiple times with the same `PeerId`s. - pub fn add_default_set_discovered_nodes(&mut self, peer_ids: impl Iterator) { - for peer_id in peer_ids { - self.peerset_handle.add_to_peers_set(HARDCODED_PEERSETS_SYNC, peer_id); - } - } - - /// Add a peer to a peers set. - pub fn add_to_peers_set(&self, protocol: ProtocolName, peer: PeerId) { - if let Some(index) = self.notification_protocols.iter().position(|p| *p == protocol) { - self.peerset_handle.add_to_peers_set(sc_peerset::SetId::from(index), peer); - } else { - error!( - target: "sub-libp2p", - "add_to_peers_set with unknown protocol: {}", - protocol - ); - } - } - - /// Remove a peer from a peers set. - pub fn remove_from_peers_set(&self, protocol: ProtocolName, peer: PeerId) { - if let Some(index) = self.notification_protocols.iter().position(|p| *p == protocol) { - self.peerset_handle.remove_from_peers_set(sc_peerset::SetId::from(index), peer); - } else { - error!( - target: "sub-libp2p", - "remove_from_peers_set with unknown protocol: {}", - protocol - ); - } + /// Can be called multiple times with the same `PeerId`. + pub fn add_known_peer(&mut self, peer_id: PeerId) { + self.peerset_handle.add_known_peer(peer_id); } } diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 31b1d46ebc757..097851a2c31e1 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -545,9 +545,9 @@ impl Notifications { } /// Returns the state of the peerset manager, for debugging purposes. - pub fn peerset_debug_info(&mut self) -> serde_json::Value { - self.peerset.debug_info() - } + // pub fn peerset_debug_info(&mut self) -> serde_json::Value { + // self.peerset.debug_info() + // } /// Function that is called when the peerset wants us to connect to a peer. fn peerset_report_connect(&mut self, peer_id: PeerId, set_id: sc_peerset::SetId) { diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 5a23f0e7d66a2..93de714aab2c5 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -880,39 +880,39 @@ where } } - fn add_to_peers_set( - &self, - protocol: ProtocolName, - peers: HashSet, - ) -> Result<(), String> { - let peers = self.split_multiaddr_and_peer_id(peers)?; - - for (peer_id, addr) in peers.into_iter() { - // Make sure the local peer ID is never added to the PSM. - if peer_id == self.local_peer_id { - return Err("Local peer ID cannot be added as a reserved peer.".to_string()) - } - - if !addr.is_empty() { - let _ = self - .to_worker - .unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id, addr)); - } - let _ = self - .to_worker - .unbounded_send(ServiceToWorkerMsg::AddToPeersSet(protocol.clone(), peer_id)); - } - - Ok(()) - } - - fn remove_from_peers_set(&self, protocol: ProtocolName, peers: Vec) { - for peer_id in peers.into_iter() { - let _ = self - .to_worker - .unbounded_send(ServiceToWorkerMsg::RemoveFromPeersSet(protocol.clone(), peer_id)); - } - } + // fn add_to_peers_set( + // &self, + // protocol: ProtocolName, + // peers: HashSet, + // ) -> Result<(), String> { + // let peers = self.split_multiaddr_and_peer_id(peers)?; + + // for (peer_id, addr) in peers.into_iter() { + // // Make sure the local peer ID is never added to the PSM. + // if peer_id == self.local_peer_id { + // return Err("Local peer ID cannot be added as a reserved peer.".to_string()) + // } + + // if !addr.is_empty() { + // let _ = self + // .to_worker + // .unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id, addr)); + // } + // let _ = self + // .to_worker + // .unbounded_send(ServiceToWorkerMsg::AddToPeersSet(protocol.clone(), peer_id)); + // } + + // Ok(()) + // } + + // fn remove_from_peers_set(&self, protocol: ProtocolName, peers: Vec) { + // for peer_id in peers.into_iter() { + // let _ = self + // .to_worker + // .unbounded_send(ServiceToWorkerMsg::RemoveFromPeersSet(protocol.clone(), peer_id)); + // } + // } fn sync_num_connected(&self) -> usize { self.num_connected.load(Ordering::Relaxed) @@ -1126,8 +1126,8 @@ enum ServiceToWorkerMsg { SetPeersetReserved(ProtocolName, HashSet), AddSetReserved(ProtocolName, PeerId), RemoveSetReserved(ProtocolName, PeerId), - AddToPeersSet(ProtocolName, PeerId), - RemoveFromPeersSet(ProtocolName, PeerId), + //AddToPeersSet(ProtocolName, PeerId), + //RemoveFromPeersSet(ProtocolName, PeerId), EventStream(out_events::Sender), Request { target: PeerId, @@ -1304,16 +1304,16 @@ where .remove_set_reserved_peer(protocol, peer_id), ServiceToWorkerMsg::AddKnownAddress(peer_id, addr) => self.network_service.behaviour_mut().add_known_address(peer_id, addr), - ServiceToWorkerMsg::AddToPeersSet(protocol, peer_id) => self - .network_service - .behaviour_mut() - .user_protocol_mut() - .add_to_peers_set(protocol, peer_id), - ServiceToWorkerMsg::RemoveFromPeersSet(protocol, peer_id) => self - .network_service - .behaviour_mut() - .user_protocol_mut() - .remove_from_peers_set(protocol, peer_id), + // ServiceToWorkerMsg::AddToPeersSet(protocol, peer_id) => self + // .network_service + // .behaviour_mut() + // .user_protocol_mut() + // .add_to_peers_set(protocol, peer_id), + // ServiceToWorkerMsg::RemoveFromPeersSet(protocol, peer_id) => self + // .network_service + // .behaviour_mut() + // .user_protocol_mut() + // .remove_from_peers_set(protocol, peer_id), ServiceToWorkerMsg::EventStream(sender) => self.event_streams.push(sender), ServiceToWorkerMsg::Request { target, @@ -1457,13 +1457,13 @@ where self.network_service .behaviour_mut() .user_protocol_mut() - .add_default_set_discovered_nodes(iter::once(peer_id)); + .add_known_peer(peer_id); }, SwarmEvent::Behaviour(BehaviourOut::Discovered(peer_id)) => { self.network_service .behaviour_mut() .user_protocol_mut() - .add_default_set_discovered_nodes(iter::once(peer_id)); + .add_known_peer(peer_id); }, SwarmEvent::Behaviour(BehaviourOut::RandomKademliaStarted) => { if let Some(metrics) = self.metrics.as_ref() { diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 3f9b7e552027b..915235ffb45a6 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -225,16 +225,16 @@ pub trait NetworkPeers { /// /// Returns an `Err` if one of the given addresses is invalid or contains an /// invalid peer ID (which includes the local peer ID). - fn add_to_peers_set( - &self, - protocol: ProtocolName, - peers: HashSet, - ) -> Result<(), String>; + // fn add_to_peers_set( + // &self, + // protocol: ProtocolName, + // peers: HashSet, + // ) -> Result<(), String>; /// Remove peers from a peer set. /// /// If we currently have an open substream with this peer, it will soon be closed. - fn remove_from_peers_set(&self, protocol: ProtocolName, peers: Vec); + //fn remove_from_peers_set(&self, protocol: ProtocolName, peers: Vec); /// Returns the number of peers in the sync peer set we're connected to. fn sync_num_connected(&self) -> usize; @@ -302,17 +302,17 @@ where T::remove_peers_from_reserved_set(self, protocol, peers) } - fn add_to_peers_set( - &self, - protocol: ProtocolName, - peers: HashSet, - ) -> Result<(), String> { - T::add_to_peers_set(self, protocol, peers) - } - - fn remove_from_peers_set(&self, protocol: ProtocolName, peers: Vec) { - T::remove_from_peers_set(self, protocol, peers) - } + // fn add_to_peers_set( + // &self, + // protocol: ProtocolName, + // peers: HashSet, + // ) -> Result<(), String> { + // T::add_to_peers_set(self, protocol, peers) + // } + + // fn remove_from_peers_set(&self, protocol: ProtocolName, peers: Vec) { + // T::remove_from_peers_set(self, protocol, peers) + // } fn sync_num_connected(&self) -> usize { T::sync_num_connected(self) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 3aea6d11cdb93..598f7ea48eff8 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -41,7 +41,7 @@ use protocol_controller::{ProtocolController, ProtocolHandle}; use futures::{ channel::oneshot, - future::{join_all, JoinAll, BoxFuture}, + future::{join_all, BoxFuture, JoinAll}, prelude::*, stream::Stream, }; @@ -58,6 +58,8 @@ use wasm_timer::Delay; pub use libp2p::PeerId; +pub use peer_store::BANNED_THRESHOLD; + /// Identifier of a set in the peerset. /// /// Can be constructed using the `From` trait implementation based on the index of the set @@ -150,15 +152,10 @@ impl PeersetHandle { self.peer_store_handle.report_peer(peer_id, score_diff); } - /// Add a peer to a set. - //pub fn add_to_peers_set(&self, set_id: SetId, peer_id: PeerId) { - // let _ = self.tx.unbounded_send(Action::AddToPeersSet(set_id, peer_id)); - //} - - /// Remove a peer from a set. - //pub fn remove_from_peers_set(&self, set_id: SetId, peer_id: PeerId) { - // let _ = self.tx.unbounded_send(Action::RemoveFromPeersSet(set_id, peer_id)); - //} + /// Add a peer to the list of known peers. + pub fn add_known_peer(&mut self, peer_id: PeerId) { + self.peer_store_handle.add_known_peer(peer_id); + } /// Returns the reputation value of the peer. pub async fn peer_reputation(self, peer_id: PeerId) -> Result { @@ -301,16 +298,26 @@ impl Peerset { self.protocol_handles[set_id.0].reserved_peers(pending_response); } - /// Adds a node to the given set. The peerset will, if possible and not already the case, - /// try to connect to it. + /// Indicate that we received an incoming connection. Must be answered either with + /// a corresponding `Accept` or `Reject`, except if we were already connected to this peer. /// - /// > **Note**: This has the same effect as [`PeersetHandle::add_to_peers_set`]. - // pub fn add_to_peers_set(&mut self, set_id: SetId, peer_id: PeerId) { - // if let peersstate::Peer::Unknown(entry) = self.data.peer(set_id.0, &peer_id) { - // entry.discover(); - // self.alloc_slots(set_id); - // } - // } + /// Note that this mechanism is orthogonal to `Connect`/`Drop`. Accepting an incoming + /// connection implicitly means `Connect`, but incoming connections aren't cancelled by + /// `dropped`. + // Implementation note: because of concurrency issues, it is possible that we push a `Connect` + // message to the output channel with a `PeerId`, and that `incoming` gets called with the same + // `PeerId` before that message has been read by the user. In this situation we must not answer. + pub fn incoming(&mut self, set_id: SetId, peer_id: PeerId, index: IncomingIndex) { + self.protocol_handles[set_id.0].incoming_connection(peer_id, index); + } + + /// Indicate that we dropped an active connection with a peer, or that we failed to connect. + /// + /// Must only be called after the PSM has either generated a `Connect` message with this + /// `PeerId`, or accepted an incoming connection with this `PeerId`. + pub fn dropped(&mut self, set_id: SetId, peer_id: PeerId, reason: DropReason) { + self.protocol_handles[set_id.0].dropped(peer_id); + } // fn on_remove_from_peers_set(&mut self, set_id: SetId, peer_id: PeerId) { // // Don't do anything if node is reserved. diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index d7fb32a359647..b48b6445ebe1d 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -20,16 +20,17 @@ use libp2p::PeerId; use partial_sort::PartialSort; use std::{ cmp::{Ord, Ordering, PartialOrd}, - collections::{HashMap, HashSet}, + collections::{hash_map::Entry, HashMap, HashSet}, sync::{Arc, Mutex}, time::{Duration, Instant}, }; use wasm_timer::Delay; +use log::trace; use crate::ReputationChange; /// We don't accept nodes whose reputation is under this value. -const BANNED_THRESHOLD: i32 = 82 * (i32::MIN / 100); +pub const BANNED_THRESHOLD: i32 = 82 * (i32::MIN / 100); /// Reputation change for a node when we get disconnected from it. const DISCONNECT_REPUTATION_CHANGE: i32 = -256; /// Relative decrement of a reputation value that is applied every second. I.e., for inverse @@ -92,6 +93,11 @@ impl PeerStoreHandle { // FIXME: how do we use this info? May be better ask for numbers from protocol controllers? self.inner.lock().unwrap().peers.len() } + + /// Add known peer. + pub fn add_known_peer(&mut self, peer_id: PeerId) { + self.inner.lock().unwrap().add_known_peer(peer_id); + } } #[derive(Debug, Clone, Copy)] @@ -134,12 +140,12 @@ impl PeerInfo { fn add_reputation(&mut self, increment: i32) { self.reputation = self.reputation.saturating_add(increment); - self.last_updated = Instant::now(); + self.bump_last_updated(); } fn decay_reputation(&mut self, seconds_passed: u64) { // Note that decaying the reputation value happens "on its own", - // so we don't bump `last_updated`. + // so we don't do `bump_last_updated()`. for _ in 0..seconds_passed { let mut diff = self.reputation / INVERSE_DECREMENT; if diff == 0 && self.reputation < 0 { @@ -155,6 +161,10 @@ impl PeerInfo { } } } + + fn bump_last_updated(&mut self) { + self.last_updated = Instant::now(); + } } #[derive(Debug)] @@ -215,6 +225,23 @@ impl PeerStoreInner { self.peers .retain(|_, info| info.reputation != 0 || info.last_updated + FORGET_AFTER > now); } + + fn add_known_peer(&mut self, peer_id: PeerId) { + match self.peers.entry(peer_id) { + Entry::Occupied(mut e) => { + trace!( + target: "peerset", + "Trying to add an already known peer {}, bumping `last_updated`.", + peer_id, + ); + e.get_mut().bump_last_updated(); + }, + Entry::Vacant(e) => { + trace!(target: "peerset", "Adding a new known peer {}.", peer_id); + e.insert(PeerInfo::default()); + }, + } + } } #[derive(Debug)] From ff43c19a04e681d8c530d86cd215b4969d851271 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 7 Apr 2023 18:24:43 +0300 Subject: [PATCH 45/98] WIP: try to make `sc-network` compile --- client/network/src/protocol/notifications/behaviour.rs | 6 +++--- client/network/src/service/traits.rs | 1 + client/peerset/src/lib.rs | 6 ++++++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 097851a2c31e1..31b1d46ebc757 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -545,9 +545,9 @@ impl Notifications { } /// Returns the state of the peerset manager, for debugging purposes. - // pub fn peerset_debug_info(&mut self) -> serde_json::Value { - // self.peerset.debug_info() - // } + pub fn peerset_debug_info(&mut self) -> serde_json::Value { + self.peerset.debug_info() + } /// Function that is called when the peerset wants us to connect to a peer. fn peerset_report_connect(&mut self, peer_id: PeerId, set_id: sc_peerset::SetId) { diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 915235ffb45a6..b6ab1e557cbc9 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -259,6 +259,7 @@ where } fn report_peer(&self, who: PeerId, cost_benefit: ReputationChange) { + // TODO: make async interface to `PeerStore`? T::report_peer(self, who, cost_benefit) } diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 598f7ea48eff8..136732cda05a3 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -345,6 +345,12 @@ impl Peerset { let _ = self.peer_store_handle.report_peer(peer_id, score_diff); } + /// Produces a JSON object containing the state of the peerset manager, for debugging purposes. + pub fn debug_info(&mut self) -> serde_json::Value { + // TODO: check what info we can include here. + json!("unimplemented") + } + /// Returns the number of peers that we have discovered. pub fn num_discovered_peers(&self) -> usize { self.peer_store_handle.num_known_peers() From f53e1a1f1adbbd4628f3610e738d4507d7064f5c Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 10 Apr 2023 15:54:49 +0300 Subject: [PATCH 46/98] Restore the original API of `Peerset` --- client/network/src/protocol.rs | 2 +- client/network/src/service.rs | 10 +--- client/peerset/src/lib.rs | 96 +++++++++++++++++++++++--------- client/peerset/src/peer_store.rs | 5 +- client/peerset/tests/fuzz.rs | 2 +- 5 files changed, 78 insertions(+), 37 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index bddf614016517..61e4b6367bc0f 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -37,12 +37,12 @@ use log::{debug, error, warn}; use sc_network_common::{role::Roles, sync::message::BlockAnnouncesHandshake}; use sp_runtime::traits::Block as BlockT; +use futures::channel::oneshot; use std::{ collections::{HashMap, HashSet, VecDeque}, iter, task::Poll, }; -use futures::channel::oneshot; use message::{generic::Message as GenericMessage, Message}; use notifications::{Notifications, NotificationsOut}; diff --git a/client/network/src/service.rs b/client/network/src/service.rs index 93de714aab2c5..a9588b0f21dc1 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -1454,16 +1454,10 @@ where .behaviour_mut() .add_self_reported_address_to_dht(&peer_id, &protocols, addr); } - self.network_service - .behaviour_mut() - .user_protocol_mut() - .add_known_peer(peer_id); + self.network_service.behaviour_mut().user_protocol_mut().add_known_peer(peer_id); }, SwarmEvent::Behaviour(BehaviourOut::Discovered(peer_id)) => { - self.network_service - .behaviour_mut() - .user_protocol_mut() - .add_known_peer(peer_id); + self.network_service.behaviour_mut().user_protocol_mut().add_known_peer(peer_id); }, SwarmEvent::Behaviour(BehaviourOut::RandomKademliaStarted) => { if let Some(metrics) = self.metrics.as_ref() { diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 136732cda05a3..8bbc0d6798c76 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -60,6 +60,17 @@ pub use libp2p::PeerId; pub use peer_store::BANNED_THRESHOLD; +#[derive(Debug)] +enum Action { + AddReservedPeer(SetId, PeerId), + RemoveReservedPeer(SetId, PeerId), + SetReservedPeers(SetId, HashSet), + SetReservedOnly(SetId, bool), + ReportPeer(PeerId, ReputationChange), + AddKnownPeer(PeerId), + PeerReputation(PeerId, oneshot::Sender), +} + /// Identifier of a set in the peerset. /// /// Can be constructed using the `From` trait implementation based on the index of the set @@ -111,10 +122,7 @@ impl ReputationChange { /// Shared handle to the peer set manager (PSM). Distributed around the code. #[derive(Debug, Clone)] pub struct PeersetHandle { - // Peer store handle. - peer_store_handle: PeerStoreHandle, - // Protocol handles per every `SetId`. The size of this vectore never changes. - protocol_handles: Vec, + tx: TracingUnboundedSender, } impl PeersetHandle { @@ -126,41 +134,45 @@ impl PeersetHandle { /// > **Note**: Keep in mind that the networking has to know an address for this node, /// > otherwise it will not be able to connect to it. pub fn add_reserved_peer(&self, set_id: SetId, peer_id: PeerId) { - self.protocol_handles[set_id.0].add_reserved_peer(peer_id); + let _ = self.tx.unbounded_send(Action::AddReservedPeer(set_id, peer_id)); } /// Remove a previously-added reserved peer. /// /// Has no effect if the node was not a reserved peer. pub fn remove_reserved_peer(&self, set_id: SetId, peer_id: PeerId) { - self.protocol_handles[set_id.0].remove_reserved_peer(peer_id); + let _ = self.tx.unbounded_send(Action::RemoveReservedPeer(set_id, peer_id)); } /// Sets whether or not the peerset only has connections with nodes marked as reserved for /// the given set. pub fn set_reserved_only(&self, set_id: SetId, reserved: bool) { - self.protocol_handles[set_id.0].set_reserved_only(reserved); + let _ = self.tx.unbounded_send(Action::SetReservedOnly(set_id, reserved)); } /// Set reserved peers to the new set. pub fn set_reserved_peers(&self, set_id: SetId, peer_ids: HashSet) { - self.protocol_handles[set_id.0].set_reserved_peers(peer_ids); + let _ = self.tx.unbounded_send(Action::SetReservedPeers(set_id, peer_ids)); } /// Reports an adjustment to the reputation of the given peer. - pub fn report_peer(&mut self, peer_id: PeerId, score_diff: ReputationChange) { - self.peer_store_handle.report_peer(peer_id, score_diff); + pub fn report_peer(&self, peer_id: PeerId, score_diff: ReputationChange) { + let _ = self.tx.unbounded_send(Action::ReportPeer(peer_id, score_diff)); } /// Add a peer to the list of known peers. - pub fn add_known_peer(&mut self, peer_id: PeerId) { - self.peer_store_handle.add_known_peer(peer_id); + pub fn add_known_peer(&self, peer_id: PeerId) { + let _ = self.tx.unbounded_send(Action::AddKnownPeer(peer_id)); } /// Returns the reputation value of the peer. pub async fn peer_reputation(self, peer_id: PeerId) -> Result { - Ok(self.peer_store_handle.peer_reputation(&peer_id)) - // TODO: convert into sync function. + let (tx, rx) = oneshot::channel(); + + let _ = self.tx.unbounded_send(Action::PeerReputation(peer_id, tx)); + + // The channel can only be closed if the peerset no longer exists. + rx.await.map_err(|_| ()) } } @@ -246,7 +258,11 @@ pub struct Peerset { protocol_controller_futures: JoinAll>, /// Commands sent from protocol controllers to `Notifications`. The size of this vector never /// changes. - rx: TracingUnboundedReceiver, + from_controllers: TracingUnboundedReceiver, + /// Receiver for messages from the `PeersetHandle` and from `to_self`. + from_handle: TracingUnboundedReceiver, + /// Sending side of `from_handle`. + to_self: TracingUnboundedSender, } impl Peerset { @@ -255,7 +271,8 @@ impl Peerset { let default_set_config = &config.sets[0]; let peer_store = PeerStore::new(default_set_config.bootnodes.clone()); - let (tx, rx) = tracing_unbounded("mpsc_protocol_controllers_to_notifications", 10_000); + let (to_notifications, from_controllers) = + tracing_unbounded("mpsc_protocol_controllers_to_notifications", 10_000); let controllers = config .sets @@ -265,7 +282,7 @@ impl Peerset { ProtocolController::new( SetId::from(set), set_config, - tx.clone(), + to_notifications.clone(), peer_store.handle(), ) }) @@ -274,10 +291,9 @@ impl Peerset { let (protocol_handles, protocol_controllers): (Vec, Vec<_>) = controllers.into_iter().unzip(); - let handle = PeersetHandle { - peer_store_handle: peer_store.handle(), - protocol_handles: protocol_handles.clone(), - }; + let (to_self, from_handle) = tracing_unbounded("mpsc_peerset_messages", 10_000); + + let handle = PeersetHandle { tx: to_self.clone() }; let protocol_controller_futures = join_all(protocol_controllers.into_iter().map(|c| c.run().boxed())); @@ -287,7 +303,9 @@ impl Peerset { peer_store_future: peer_store.run().boxed(), protocol_handles, protocol_controller_futures, - rx, + from_controllers, + from_handle, + to_self, }; (peerset, handle) @@ -315,7 +333,7 @@ impl Peerset { /// /// Must only be called after the PSM has either generated a `Connect` message with this /// `PeerId`, or accepted an incoming connection with this `PeerId`. - pub fn dropped(&mut self, set_id: SetId, peer_id: PeerId, reason: DropReason) { + pub fn dropped(&mut self, set_id: SetId, peer_id: PeerId, _reason: DropReason) { self.protocol_handles[set_id.0].dropped(peer_id); } @@ -342,7 +360,7 @@ impl Peerset { // We don't immediately perform the adjustments in order to have state consistency. We // don't want the reporting here to take priority over messages sent using the // `PeersetHandle`. - let _ = self.peer_store_handle.report_peer(peer_id, score_diff); + let _ = self.to_self.unbounded_send(Action::ReportPeer(peer_id, score_diff)); } /// Produces a JSON object containing the state of the peerset manager, for debugging purposes. @@ -361,7 +379,35 @@ impl Stream for Peerset { type Item = Message; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - if let Poll::Ready(msg) = self.rx.poll_next_unpin(cx) { + while let Poll::Ready(action) = self.from_handle.poll_next_unpin(cx) { + if let Some(action) = action { + match action { + Action::AddReservedPeer(set_id, peer_id) => + self.protocol_handles[set_id.0].add_reserved_peer(peer_id), + Action::RemoveReservedPeer(set_id, peer_id) => + self.protocol_handles[set_id.0].remove_reserved_peer(peer_id), + Action::SetReservedPeers(set_id, peer_ids) => + self.protocol_handles[set_id.0].set_reserved_peers(peer_ids), + Action::SetReservedOnly(set_id, reserved_only) => + self.protocol_handles[set_id.0].set_reserved_only(reserved_only), + Action::ReportPeer(peer_id, score_diff) => + self.peer_store_handle.report_peer(peer_id, score_diff), + Action::AddKnownPeer(peer_id) => self.peer_store_handle.add_known_peer(peer_id), + Action::PeerReputation(peer_id, pending_response) => { + let _ = + pending_response.send(self.peer_store_handle.peer_reputation(&peer_id)); + }, + } + } else { + debug!( + target: "peerset", + "`PeersetHandle` was dropped, terminating `Peerset`." + ); + return Poll::Ready(None) + } + } + + if let Poll::Ready(msg) = self.from_controllers.poll_next_unpin(cx) { if let Some(msg) = msg { return Poll::Ready(Some(msg)) } else { diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index b48b6445ebe1d..9b849d2213462 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -17,6 +17,7 @@ // along with this program. If not, see . use libp2p::PeerId; +use log::trace; use partial_sort::PartialSort; use std::{ cmp::{Ord, Ordering, PartialOrd}, @@ -25,7 +26,6 @@ use std::{ time::{Duration, Instant}, }; use wasm_timer::Delay; -use log::trace; use crate::ReputationChange; @@ -204,6 +204,7 @@ impl PeerStoreInner { (!info.is_banned() && !ignored.contains(peer_id)).then_some((*peer_id, *info)) }) .collect::>(); + let count = std::cmp::min(count, candidates.len()); candidates.partial_sort(count, |(_, info1), (_, info2)| info1.cmp(info2)); candidates.iter().take(count).map(|(peer_id, _)| *peer_id).collect() @@ -228,7 +229,7 @@ impl PeerStoreInner { fn add_known_peer(&mut self, peer_id: PeerId) { match self.peers.entry(peer_id) { - Entry::Occupied(mut e) => { + Entry::Occupied(mut e) => { trace!( target: "peerset", "Trying to add an already known peer {}, bumping `last_updated`.", diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index 1f4bd053b553a..c0f986ec6af9e 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -115,7 +115,7 @@ fn test_once() { 1 => { let new_id = PeerId::random(); known_nodes.insert(new_id); - peerset.add_to_peers_set(SetId::from(0), new_id); + peerset_handle.add_known_peer(new_id); }, // If we generate 2, adjust a random reputation. From 4e1b7734ef280b10a61bdd4ff2d3c74be8b643b3 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 10 Apr 2023 15:57:46 +0300 Subject: [PATCH 47/98] docs: apply review suggestions --- client/peerset/src/protocol_controller.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index d38126cf7232a..59463b1374fd0 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -151,13 +151,13 @@ pub struct ProtocolController { set_id: SetId, /// Receiver for messages from [`ProtocolHandle`]. from_handle: TracingUnboundedReceiver, - /// Number of occupied slots for incoming connections. + /// Number of occupied slots for incoming connections (not counting reserved nodes). num_in: u32, - /// Number of occupied slots for outgoing connections. + /// Number of occupied slots for outgoing connections (not counting reserved nodes). num_out: u32, - /// Maximum number of slots for incoming connections. + /// Maximum number of slots for incoming connections (not counting reserved nodes). max_in: u32, - /// Maximum number of slots for outgoing connections. + /// Maximum number of slots for outgoing connections (not counting reserved nodes). max_out: u32, /// Connected regular nodes. nodes: HashMap, From 23bf89b50414820e43f1e32ff5cd613a305f97db Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 10 Apr 2023 16:58:34 +0300 Subject: [PATCH 48/98] Make substrate compile --- .../grandpa/src/communication/tests.rs | 12 ----- client/network-gossip/src/bridge.rs | 12 ----- client/network-gossip/src/state_machine.rs | 12 ----- client/network/sync/src/service/mock.rs | 6 --- client/offchain/src/api.rs | 12 ----- client/offchain/src/lib.rs | 12 ----- client/peerset/tests/fuzz.rs | 44 ++++++++++--------- 7 files changed, 24 insertions(+), 86 deletions(-) diff --git a/client/consensus/grandpa/src/communication/tests.rs b/client/consensus/grandpa/src/communication/tests.rs index f97b1f1e88181..eb88382989175 100644 --- a/client/consensus/grandpa/src/communication/tests.rs +++ b/client/consensus/grandpa/src/communication/tests.rs @@ -116,18 +116,6 @@ impl NetworkPeers for TestNetwork { fn remove_peers_from_reserved_set(&self, _protocol: ProtocolName, _peers: Vec) {} - fn add_to_peers_set( - &self, - _protocol: ProtocolName, - _peers: HashSet, - ) -> Result<(), String> { - unimplemented!(); - } - - fn remove_from_peers_set(&self, _protocol: ProtocolName, _peers: Vec) { - unimplemented!(); - } - fn sync_num_connected(&self) -> usize { unimplemented!(); } diff --git a/client/network-gossip/src/bridge.rs b/client/network-gossip/src/bridge.rs index 4793d7822ddbe..baf11c9e8649b 100644 --- a/client/network-gossip/src/bridge.rs +++ b/client/network-gossip/src/bridge.rs @@ -416,18 +416,6 @@ mod tests { fn remove_peers_from_reserved_set(&self, _protocol: ProtocolName, _peers: Vec) {} - fn add_to_peers_set( - &self, - _protocol: ProtocolName, - _peers: HashSet, - ) -> Result<(), String> { - unimplemented!(); - } - - fn remove_from_peers_set(&self, _protocol: ProtocolName, _peers: Vec) { - unimplemented!(); - } - fn sync_num_connected(&self) -> usize { unimplemented!(); } diff --git a/client/network-gossip/src/state_machine.rs b/client/network-gossip/src/state_machine.rs index e6d2b0e2ae4c8..ee65bd890e15b 100644 --- a/client/network-gossip/src/state_machine.rs +++ b/client/network-gossip/src/state_machine.rs @@ -646,18 +646,6 @@ mod tests { fn remove_peers_from_reserved_set(&self, _protocol: ProtocolName, _peers: Vec) {} - fn add_to_peers_set( - &self, - _protocol: ProtocolName, - _peers: HashSet, - ) -> Result<(), String> { - unimplemented!(); - } - - fn remove_from_peers_set(&self, _protocol: ProtocolName, _peers: Vec) { - unimplemented!(); - } - fn sync_num_connected(&self) -> usize { unimplemented!(); } diff --git a/client/network/sync/src/service/mock.rs b/client/network/sync/src/service/mock.rs index c882633993c8b..9bce9f91b1d24 100644 --- a/client/network/sync/src/service/mock.rs +++ b/client/network/sync/src/service/mock.rs @@ -101,12 +101,6 @@ mockall::mock! { peers: HashSet, ) -> Result<(), String>; fn remove_peers_from_reserved_set(&self, protocol: ProtocolName, peers: Vec); - fn add_to_peers_set( - &self, - protocol: ProtocolName, - peers: HashSet, - ) -> Result<(), String>; - fn remove_from_peers_set(&self, protocol: ProtocolName, peers: Vec); fn sync_num_connected(&self) -> usize; } diff --git a/client/offchain/src/api.rs b/client/offchain/src/api.rs index a15f03bab6f84..f6cfdcd536bb5 100644 --- a/client/offchain/src/api.rs +++ b/client/offchain/src/api.rs @@ -392,18 +392,6 @@ mod tests { unimplemented!(); } - fn add_to_peers_set( - &self, - _protocol: ProtocolName, - _peers: HashSet, - ) -> Result<(), String> { - unimplemented!(); - } - - fn remove_from_peers_set(&self, _protocol: ProtocolName, _peers: Vec) { - unimplemented!(); - } - fn sync_num_connected(&self) -> usize { unimplemented!(); } diff --git a/client/offchain/src/lib.rs b/client/offchain/src/lib.rs index 677d89267e3a6..3b35a961e5858 100644 --- a/client/offchain/src/lib.rs +++ b/client/offchain/src/lib.rs @@ -331,18 +331,6 @@ mod tests { unimplemented!(); } - fn add_to_peers_set( - &self, - _protocol: ProtocolName, - _peers: HashSet, - ) -> Result<(), String> { - unimplemented!(); - } - - fn remove_from_peers_set(&self, _protocol: ProtocolName, _peers: Vec) { - unimplemented!(); - } - fn sync_num_connected(&self) -> usize { unimplemented!(); } diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index c0f986ec6af9e..6f7d131127bc9 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -89,26 +89,30 @@ fn test_once() { let action_weights = [150, 90, 90, 30, 30, 1, 1, 4, 4]; match WeightedIndex::new(&action_weights).unwrap().sample(&mut rng) { // If we generate 0, poll the peerset. - 0 => match Stream::poll_next(Pin::new(&mut peerset), cx) { - Poll::Ready(Some(Message::Connect { peer_id, .. })) => { - if let Some(id) = - incoming_nodes.iter().find(|(_, v)| **v == peer_id).map(|(&id, _)| id) - { - incoming_nodes.remove(&id); - } - assert!(connected_nodes.insert(peer_id)); - }, - Poll::Ready(Some(Message::Drop { peer_id, .. })) => { - connected_nodes.remove(&peer_id); - }, - Poll::Ready(Some(Message::Accept(n))) => { - assert!(connected_nodes.insert(incoming_nodes.remove(&n).unwrap())) - }, - Poll::Ready(Some(Message::Reject(n))) => { - assert!(!connected_nodes.contains(&incoming_nodes.remove(&n).unwrap())) - }, - Poll::Ready(None) => panic!(), - Poll::Pending => {}, + 0 => loop { + match Stream::poll_next(Pin::new(&mut peerset), cx) { + Poll::Ready(Some(Message::Connect { peer_id, .. })) => { + if let Some(id) = incoming_nodes + .iter() + .find(|(_, v)| **v == peer_id) + .map(|(&id, _)| id) + { + incoming_nodes.remove(&id); + } + assert!(connected_nodes.insert(peer_id)); + }, + Poll::Ready(Some(Message::Drop { peer_id, .. })) => { + connected_nodes.remove(&peer_id); + }, + Poll::Ready(Some(Message::Accept(n))) => { + assert!(connected_nodes.insert(incoming_nodes.remove(&n).unwrap())) + }, + Poll::Ready(Some(Message::Reject(n))) => { + assert!(!connected_nodes.contains(&incoming_nodes.remove(&n).unwrap())) + }, + Poll::Ready(None) => panic!(), + Poll::Pending => break, + } }, // If we generate 1, discover a new node. From d74f8b97f43676c318e8fdeb59d862aa88faf659 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 10 Apr 2023 17:02:26 +0300 Subject: [PATCH 49/98] Get rid of `PeersState` --- client/peerset/src/lib.rs | 1 - client/peerset/src/peersstate.rs | 737 ------------------------------- 2 files changed, 738 deletions(-) delete mode 100644 client/peerset/src/peersstate.rs diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 8bbc0d6798c76..924bbf8efd7c4 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -33,7 +33,6 @@ //! will at all time try to maintain a connection with. mod peer_store; -mod peersstate; mod protocol_controller; use peer_store::{PeerReputationProvider, PeerStore, PeerStoreHandle}; diff --git a/client/peerset/src/peersstate.rs b/client/peerset/src/peersstate.rs deleted file mode 100644 index 84907ac914b45..0000000000000 --- a/client/peerset/src/peersstate.rs +++ /dev/null @@ -1,737 +0,0 @@ -// This file is part of Substrate. - -// Copyright (C) Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: GPL-3.0-or-later WITH Classpath-exception-2.0 - -// This program is free software: you can redistribute it and/or modify -// it under the terms of the GNU General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. - -// This program is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU General Public License for more details. - -// You should have received a copy of the GNU General Public License -// along with this program. If not, see . - -//! Reputation and slots allocation system behind the peerset. -//! -//! The [`PeersState`] state machine is responsible for managing the reputation and allocating -//! slots. It holds a list of nodes, each associated with a reputation value, a list of sets the -//! node belongs to, and for each set whether we are connected or not to this node. Thanks to this -//! list, it knows how many slots are occupied. It also holds a list of nodes which don't occupy -//! slots. -//! -//! > Note: This module is purely dedicated to managing slots and reputations. Features such as -//! > for example connecting to some nodes in priority should be added outside of this -//! > module, rather than inside. - -use libp2p::PeerId; -use log::error; -use std::{ - borrow::Cow, - collections::{ - hash_map::{Entry, OccupiedEntry}, - HashMap, HashSet, - }, - time::Instant, -}; - -/// State storage behind the peerset. -/// -/// # Usage -/// -/// This struct is nothing more but a data structure containing a list of nodes, where each node -/// has a reputation and is either connected to us or not. -#[derive(Debug, Clone)] -pub struct PeersState { - /// List of nodes that we know about. - /// - /// > **Note**: This list should really be ordered by decreasing reputation, so that we can - /// > easily select the best node to connect to. As a first draft, however, we don't sort, to - /// > make the logic easier. - nodes: HashMap, - - /// Configuration of each set. The size of this `Vec` is never modified. - sets: Vec, -} - -/// Configuration of a single set. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct SetConfig { - /// Maximum allowed number of slot-occupying nodes for ingoing connections. - pub in_peers: u32, - - /// Maximum allowed number of slot-occupying nodes for outgoing connections. - pub out_peers: u32, -} - -/// State of a single set. -#[derive(Debug, Clone, PartialEq, Eq)] -struct SetInfo { - /// Number of slot-occupying nodes for which the `MembershipState` is `In`. - num_in: u32, - - /// Number of slot-occupying nodes for which the `MembershipState` is `In`. - num_out: u32, - - /// Maximum allowed number of slot-occupying nodes for which the `MembershipState` is `In`. - max_in: u32, - - /// Maximum allowed number of slot-occupying nodes for which the `MembershipState` is `Out`. - max_out: u32, - - /// List of node identities (discovered or not) that don't occupy slots. - /// - /// Note for future readers: this module is purely dedicated to managing slots. If you are - /// considering adding more features, please consider doing so outside of this module rather - /// than inside. - no_slot_nodes: HashSet, -} - -/// State of a single node that we know about. -#[derive(Debug, Clone, PartialEq, Eq)] -struct Node { - /// List of sets the node belongs to. - /// Always has a fixed size equal to the one of [`PeersState::set`]. The various possible sets - /// are indices into this `Vec`. - sets: Vec, - - /// Reputation value of the node, between `i32::MIN` (we hate that node) and - /// `i32::MAX` (we love that node). - reputation: i32, -} - -impl Node { - fn new(num_sets: usize) -> Self { - Self { sets: (0..num_sets).map(|_| MembershipState::NotMember).collect(), reputation: 0 } - } -} - -/// Whether we are connected to a node in the context of a specific set. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -enum MembershipState { - /// Node isn't part of that set. - NotMember, - /// We are connected through an ingoing connection. - In, - /// We are connected through an outgoing connection. - Out, - /// Node is part of that set, but we are not connected to it. - NotConnected { - /// When we were last connected to the node, or if we were never connected when we - /// discovered it. - last_connected: Instant, - }, -} - -impl MembershipState { - /// Returns `true` for [`MembershipState::In`] and [`MembershipState::Out`]. - fn is_connected(self) -> bool { - match self { - Self::In | Self::Out => true, - Self::NotMember | Self::NotConnected { .. } => false, - } - } - - /// Returns `true` for [`MembershipState::NotConnected`]. - fn is_not_connected(self) -> bool { - matches!(self, Self::NotConnected { .. }) - } -} - -impl PeersState { - /// Builds a new empty [`PeersState`]. - pub fn new(sets: impl IntoIterator) -> Self { - Self { - nodes: HashMap::new(), - sets: sets - .into_iter() - .map(|config| SetInfo { - num_in: 0, - num_out: 0, - max_in: config.in_peers, - max_out: config.out_peers, - no_slot_nodes: HashSet::new(), - }) - .collect(), - } - } - - /// Returns the number of sets. - /// - /// Corresponds to the number of elements passed to [`PeersState::new`]. - pub fn num_sets(&self) -> usize { - self.sets.len() - } - - /// Returns an object that grants access to the reputation value of a peer. - pub fn peer_reputation(&mut self, peer_id: PeerId) -> Reputation { - self.nodes.entry(peer_id).or_insert_with(|| Node::new(self.sets.len())); - - let entry = match self.nodes.entry(peer_id) { - Entry::Vacant(_) => unreachable!("guaranteed to be inserted above; qed"), - Entry::Occupied(e) => e, - }; - - Reputation { node: Some(entry) } - } - - /// Returns an object that grants access to the state of a peer in the context of a specific - /// set. - /// - /// # Panic - /// - /// `set` must be within range of the sets passed to [`PeersState::new`]. - pub fn peer<'a>(&'a mut self, set: usize, peer_id: &'a PeerId) -> Peer<'a> { - // The code below will panic anyway if this happens to be false, but this earlier assert - // makes it explicit what is wrong. - assert!(set < self.sets.len()); - - match self.nodes.get_mut(peer_id).map(|p| &p.sets[set]) { - None | Some(MembershipState::NotMember) => - Peer::Unknown(UnknownPeer { parent: self, set, peer_id: Cow::Borrowed(peer_id) }), - Some(MembershipState::In) | Some(MembershipState::Out) => - Peer::Connected(ConnectedPeer { state: self, set, peer_id: Cow::Borrowed(peer_id) }), - Some(MembershipState::NotConnected { .. }) => Peer::NotConnected(NotConnectedPeer { - state: self, - set, - peer_id: Cow::Borrowed(peer_id), - }), - } - } - - /// Returns the list of all the peers we know of. - // Note: this method could theoretically return a `Peer`, but implementing that - // isn't simple. - pub fn peers(&self) -> impl ExactSizeIterator { - self.nodes.keys() - } - - /// Returns the list of peers we are connected to in the context of a specific set. - /// - /// # Panic - /// - /// `set` must be within range of the sets passed to [`PeersState::new`]. - // Note: this method could theoretically return a `ConnectedPeer`, but implementing that - // isn't simple. - pub fn connected_peers(&self, set: usize) -> impl Iterator { - // The code below will panic anyway if this happens to be false, but this earlier assert - // makes it explicit what is wrong. - assert!(set < self.sets.len()); - - self.nodes - .iter() - .filter(move |(_, p)| p.sets[set].is_connected()) - .map(|(p, _)| p) - } - - /// Returns the peer with the highest reputation and that we are not connected to. - /// - /// If multiple nodes have the same reputation, which one is returned is unspecified. - /// - /// # Panic - /// - /// `set` must be within range of the sets passed to [`PeersState::new`]. - pub fn highest_not_connected_peer(&mut self, set: usize) -> Option { - // The code below will panic anyway if this happens to be false, but this earlier assert - // makes it explicit what is wrong. - assert!(set < self.sets.len()); - - let outcome = self - .nodes - .iter_mut() - .filter(|(_, Node { sets, .. })| sets[set].is_not_connected()) - .fold(None::<(&PeerId, &mut Node)>, |mut cur_node, to_try| { - if let Some(cur_node) = cur_node.take() { - if cur_node.1.reputation >= to_try.1.reputation { - return Some(cur_node) - } - } - Some(to_try) - }) - .map(|(peer_id, _)| *peer_id); - - outcome.map(move |peer_id| NotConnectedPeer { - state: self, - set, - peer_id: Cow::Owned(peer_id), - }) - } - - /// Returns `true` if there is a free outgoing slot available related to this set. - pub fn has_free_outgoing_slot(&self, set: usize) -> bool { - self.sets[set].num_out < self.sets[set].max_out - } - - /// Add a node to the list of nodes that don't occupy slots. - /// - /// Has no effect if the node was already in the group. - pub fn add_no_slot_node(&mut self, set: usize, peer_id: PeerId) { - // Reminder: `HashSet::insert` returns false if the node was already in the set - if !self.sets[set].no_slot_nodes.insert(peer_id) { - return - } - - if let Some(peer) = self.nodes.get_mut(&peer_id) { - match peer.sets[set] { - MembershipState::In => self.sets[set].num_in -= 1, - MembershipState::Out => self.sets[set].num_out -= 1, - MembershipState::NotConnected { .. } | MembershipState::NotMember => {}, - } - } - } - - /// Removes a node from the list of nodes that don't occupy slots. - /// - /// Has no effect if the node was not in the group. - pub fn remove_no_slot_node(&mut self, set: usize, peer_id: &PeerId) { - // Reminder: `HashSet::remove` returns false if the node was already not in the set - if !self.sets[set].no_slot_nodes.remove(peer_id) { - return - } - - if let Some(peer) = self.nodes.get_mut(peer_id) { - match peer.sets[set] { - MembershipState::In => self.sets[set].num_in += 1, - MembershipState::Out => self.sets[set].num_out += 1, - MembershipState::NotConnected { .. } | MembershipState::NotMember => {}, - } - } - } -} - -/// Grants access to the state of a peer in the [`PeersState`] in the context of a specific set. -pub enum Peer<'a> { - /// We are connected to this node. - Connected(ConnectedPeer<'a>), - /// We are not connected to this node. - NotConnected(NotConnectedPeer<'a>), - /// We have never heard of this node, or it is not part of the set. - Unknown(UnknownPeer<'a>), -} - -impl<'a> Peer<'a> { - /// If we are the `Connected` variant, returns the inner [`ConnectedPeer`]. Returns `None` - /// otherwise. - pub fn into_connected(self) -> Option> { - match self { - Self::Connected(peer) => Some(peer), - Self::NotConnected(..) | Self::Unknown(..) => None, - } - } - - /// If we are the `NotConnected` variant, returns the inner [`NotConnectedPeer`]. Returns `None` - /// otherwise. - #[cfg(test)] // Feel free to remove this if this function is needed outside of tests - pub fn into_not_connected(self) -> Option> { - match self { - Self::NotConnected(peer) => Some(peer), - Self::Connected(..) | Self::Unknown(..) => None, - } - } - - /// If we are the `Unknown` variant, returns the inner [`UnknownPeer`]. Returns `None` - /// otherwise. - #[cfg(test)] // Feel free to remove this if this function is needed outside of tests - pub fn into_unknown(self) -> Option> { - match self { - Self::Unknown(peer) => Some(peer), - Self::Connected(..) | Self::NotConnected(..) => None, - } - } -} - -/// A peer that is connected to us. -pub struct ConnectedPeer<'a> { - state: &'a mut PeersState, - set: usize, - peer_id: Cow<'a, PeerId>, -} - -impl<'a> ConnectedPeer<'a> { - /// Get the `PeerId` associated to this `ConnectedPeer`. - pub fn peer_id(&self) -> &PeerId { - &self.peer_id - } - - /// Destroys this `ConnectedPeer` and returns the `PeerId` inside of it. - pub fn into_peer_id(self) -> PeerId { - self.peer_id.into_owned() - } - - /// Switches the peer to "not connected". - pub fn disconnect(self) -> NotConnectedPeer<'a> { - let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id); - if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) { - if !is_no_slot_occupy { - match node.sets[self.set] { - MembershipState::In => self.state.sets[self.set].num_in -= 1, - MembershipState::Out => self.state.sets[self.set].num_out -= 1, - MembershipState::NotMember | MembershipState::NotConnected { .. } => { - debug_assert!( - false, - "State inconsistency: disconnecting a disconnected node" - ) - }, - } - } - node.sets[self.set] = MembershipState::NotConnected { last_connected: Instant::now() }; - } else { - debug_assert!(false, "State inconsistency: disconnecting a disconnected node"); - } - - NotConnectedPeer { state: self.state, set: self.set, peer_id: self.peer_id } - } - - /// Performs an arithmetic addition on the reputation score of that peer. - /// - /// In case of overflow, the value will be capped. - /// - /// > **Note**: Reputation values aren't specific to a set but are global per peer. - pub fn add_reputation(&mut self, modifier: i32) { - if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) { - node.reputation = node.reputation.saturating_add(modifier); - } else { - debug_assert!(false, "State inconsistency: add_reputation on an unknown node"); - } - } - - /// Returns the reputation value of the node. - /// - /// > **Note**: Reputation values aren't specific to a set but are global per peer. - pub fn reputation(&self) -> i32 { - self.state.nodes.get(&*self.peer_id).map_or(0, |p| p.reputation) - } -} - -/// A peer that is not connected to us. -#[derive(Debug)] -pub struct NotConnectedPeer<'a> { - state: &'a mut PeersState, - set: usize, - peer_id: Cow<'a, PeerId>, -} - -impl<'a> NotConnectedPeer<'a> { - /// Destroys this `NotConnectedPeer` and returns the `PeerId` inside of it. - pub fn into_peer_id(self) -> PeerId { - self.peer_id.into_owned() - } - - /// Bumps the value that `last_connected_or_discovered` would return to now, even if we - /// didn't connect or disconnect. - pub fn bump_last_connected_or_discovered(&mut self) { - let state = match self.state.nodes.get_mut(&*self.peer_id) { - Some(s) => s, - None => return, - }; - - if let MembershipState::NotConnected { last_connected } = &mut state.sets[self.set] { - *last_connected = Instant::now(); - } - } - - /// Returns when we were last connected to this peer, or when we discovered it if we were - /// never connected. - /// - /// Guaranteed to be earlier than calling `Instant::now()` after the function returns. - pub fn last_connected_or_discovered(&self) -> Instant { - let state = match self.state.nodes.get(&*self.peer_id) { - Some(s) => s, - None => { - error!( - target: "peerset", - "State inconsistency with {}; not connected after borrow", - self.peer_id - ); - return Instant::now() - }, - }; - - match state.sets[self.set] { - MembershipState::NotConnected { last_connected } => last_connected, - _ => { - error!(target: "peerset", "State inconsistency with {}", self.peer_id); - Instant::now() - }, - } - } - - /// Tries to set the peer as connected as an outgoing connection. - /// - /// If there are enough slots available, switches the node to "connected" and returns `Ok`. If - /// the slots are full, the node stays "not connected" and we return `Err`. - /// - /// Non-slot-occupying nodes don't count towards the number of slots. - pub fn try_outgoing(self) -> Result, Self> { - let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id); - - // Note that it is possible for num_out to be strictly superior to the max, in case we were - // connected to reserved node then marked them as not reserved. - if !self.state.has_free_outgoing_slot(self.set) && !is_no_slot_occupy { - return Err(self) - } - - if let Some(peer) = self.state.nodes.get_mut(&*self.peer_id) { - peer.sets[self.set] = MembershipState::Out; - if !is_no_slot_occupy { - self.state.sets[self.set].num_out += 1; - } - } else { - debug_assert!(false, "State inconsistency: try_outgoing on an unknown node"); - } - - Ok(ConnectedPeer { state: self.state, set: self.set, peer_id: self.peer_id }) - } - - /// Tries to accept the peer as an incoming connection. - /// - /// If there are enough slots available, switches the node to "connected" and returns `Ok`. If - /// the slots are full, the node stays "not connected" and we return `Err`. - /// - /// Non-slot-occupying nodes don't count towards the number of slots. - pub fn try_accept_incoming(self) -> Result, Self> { - let is_no_slot_occupy = self.state.sets[self.set].no_slot_nodes.contains(&*self.peer_id); - - // Note that it is possible for num_in to be strictly superior to the max, in case we were - // connected to reserved node then marked them as not reserved. - if self.state.sets[self.set].num_in >= self.state.sets[self.set].max_in && - !is_no_slot_occupy - { - return Err(self) - } - - if let Some(peer) = self.state.nodes.get_mut(&*self.peer_id) { - peer.sets[self.set] = MembershipState::In; - if !is_no_slot_occupy { - self.state.sets[self.set].num_in += 1; - } - } else { - debug_assert!(false, "State inconsistency: try_accept_incoming on an unknown node"); - } - - Ok(ConnectedPeer { state: self.state, set: self.set, peer_id: self.peer_id }) - } - - /// Returns the reputation value of the node. - /// - /// > **Note**: Reputation values aren't specific to a set but are global per peer. - pub fn reputation(&self) -> i32 { - self.state.nodes.get(&*self.peer_id).map_or(0, |p| p.reputation) - } - - /// Sets the reputation of the peer. - /// - /// > **Note**: Reputation values aren't specific to a set but are global per peer. - #[cfg(test)] // Feel free to remove this if this function is needed outside of tests - pub fn set_reputation(&mut self, value: i32) { - if let Some(node) = self.state.nodes.get_mut(&*self.peer_id) { - node.reputation = value; - } else { - debug_assert!(false, "State inconsistency: set_reputation on an unknown node"); - } - } - - /// Removes the peer from the list of members of the set. - pub fn forget_peer(self) -> UnknownPeer<'a> { - if let Some(peer) = self.state.nodes.get_mut(&*self.peer_id) { - debug_assert!(!matches!(peer.sets[self.set], MembershipState::NotMember)); - peer.sets[self.set] = MembershipState::NotMember; - - // Remove the peer from `self.state.nodes` entirely if it isn't a member of any set. - if peer.reputation == 0 && - peer.sets.iter().all(|set| matches!(set, MembershipState::NotMember)) - { - self.state.nodes.remove(&*self.peer_id); - } - } else { - debug_assert!(false, "State inconsistency: forget_peer on an unknown node"); - error!( - target: "peerset", - "State inconsistency with {} when forgetting peer", - self.peer_id - ); - }; - - UnknownPeer { parent: self.state, set: self.set, peer_id: self.peer_id } - } -} - -/// A peer that we have never heard of or that isn't part of the set. -pub struct UnknownPeer<'a> { - parent: &'a mut PeersState, - set: usize, - peer_id: Cow<'a, PeerId>, -} - -impl<'a> UnknownPeer<'a> { - /// Inserts the peer identity in our list. - /// - /// The node starts with a reputation of 0. You can adjust these default - /// values using the `NotConnectedPeer` that this method returns. - pub fn discover(self) -> NotConnectedPeer<'a> { - let num_sets = self.parent.sets.len(); - - self.parent - .nodes - .entry(self.peer_id.clone().into_owned()) - .or_insert_with(|| Node::new(num_sets)) - .sets[self.set] = MembershipState::NotConnected { last_connected: Instant::now() }; - - NotConnectedPeer { state: self.parent, set: self.set, peer_id: self.peer_id } - } -} - -/// Access to the reputation of a peer. -pub struct Reputation<'a> { - /// Node entry in [`PeersState::nodes`]. Always `Some` except right before dropping. - node: Option>, -} - -impl<'a> Reputation<'a> { - /// Returns the reputation value of the node. - pub fn reputation(&self) -> i32 { - self.node.as_ref().unwrap().get().reputation - } - - /// Sets the reputation of the peer. - pub fn set_reputation(&mut self, value: i32) { - self.node.as_mut().unwrap().get_mut().reputation = value; - } - - /// Performs an arithmetic addition on the reputation score of that peer. - /// - /// In case of overflow, the value will be capped. - pub fn add_reputation(&mut self, modifier: i32) { - let reputation = &mut self.node.as_mut().unwrap().get_mut().reputation; - *reputation = reputation.saturating_add(modifier); - } -} - -impl<'a> Drop for Reputation<'a> { - fn drop(&mut self) { - if let Some(node) = self.node.take() { - if node.get().reputation == 0 && - node.get().sets.iter().all(|set| matches!(set, MembershipState::NotMember)) - { - node.remove(); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::{Peer, PeersState, SetConfig}; - use libp2p::PeerId; - use std::iter; - - #[test] - fn full_slots_in() { - let mut peers_state = PeersState::new(iter::once(SetConfig { in_peers: 1, out_peers: 1 })); - let id1 = PeerId::random(); - let id2 = PeerId::random(); - - if let Peer::Unknown(e) = peers_state.peer(0, &id1) { - assert!(e.discover().try_accept_incoming().is_ok()); - } - - if let Peer::Unknown(e) = peers_state.peer(0, &id2) { - assert!(e.discover().try_accept_incoming().is_err()); - } - } - - #[test] - fn no_slot_node_doesnt_use_slot() { - let mut peers_state = PeersState::new(iter::once(SetConfig { in_peers: 1, out_peers: 1 })); - let id1 = PeerId::random(); - let id2 = PeerId::random(); - - peers_state.add_no_slot_node(0, id1); - if let Peer::Unknown(p) = peers_state.peer(0, &id1) { - assert!(p.discover().try_accept_incoming().is_ok()); - } else { - panic!() - } - - if let Peer::Unknown(e) = peers_state.peer(0, &id2) { - assert!(e.discover().try_accept_incoming().is_ok()); - } else { - panic!() - } - } - - #[test] - fn disconnecting_frees_slot() { - let mut peers_state = PeersState::new(iter::once(SetConfig { in_peers: 1, out_peers: 1 })); - let id1 = PeerId::random(); - let id2 = PeerId::random(); - - assert!(peers_state - .peer(0, &id1) - .into_unknown() - .unwrap() - .discover() - .try_accept_incoming() - .is_ok()); - assert!(peers_state - .peer(0, &id2) - .into_unknown() - .unwrap() - .discover() - .try_accept_incoming() - .is_err()); - peers_state.peer(0, &id1).into_connected().unwrap().disconnect(); - assert!(peers_state - .peer(0, &id2) - .into_not_connected() - .unwrap() - .try_accept_incoming() - .is_ok()); - } - - #[test] - fn highest_not_connected_peer() { - let mut peers_state = - PeersState::new(iter::once(SetConfig { in_peers: 25, out_peers: 25 })); - let id1 = PeerId::random(); - let id2 = PeerId::random(); - - assert!(peers_state.highest_not_connected_peer(0).is_none()); - peers_state.peer(0, &id1).into_unknown().unwrap().discover().set_reputation(50); - peers_state.peer(0, &id2).into_unknown().unwrap().discover().set_reputation(25); - assert_eq!(peers_state.highest_not_connected_peer(0).map(|p| p.into_peer_id()), Some(id1)); - peers_state.peer(0, &id2).into_not_connected().unwrap().set_reputation(75); - assert_eq!(peers_state.highest_not_connected_peer(0).map(|p| p.into_peer_id()), Some(id2)); - peers_state - .peer(0, &id2) - .into_not_connected() - .unwrap() - .try_accept_incoming() - .unwrap(); - assert_eq!(peers_state.highest_not_connected_peer(0).map(|p| p.into_peer_id()), Some(id1)); - peers_state.peer(0, &id1).into_not_connected().unwrap().set_reputation(100); - peers_state.peer(0, &id2).into_connected().unwrap().disconnect(); - assert_eq!(peers_state.highest_not_connected_peer(0).map(|p| p.into_peer_id()), Some(id1)); - peers_state.peer(0, &id1).into_not_connected().unwrap().set_reputation(-100); - assert_eq!(peers_state.highest_not_connected_peer(0).map(|p| p.into_peer_id()), Some(id2)); - } - - #[test] - fn disconnect_no_slot_doesnt_panic() { - let mut peers_state = PeersState::new(iter::once(SetConfig { in_peers: 1, out_peers: 1 })); - let id = PeerId::random(); - peers_state.add_no_slot_node(0, id); - let peer = peers_state - .peer(0, &id) - .into_unknown() - .unwrap() - .discover() - .try_outgoing() - .unwrap(); - peer.disconnect(); - } -} From bfb9c7f0ab8e97a20a215995fb67f2112bfc255f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 11 Apr 2023 14:49:24 +0300 Subject: [PATCH 50/98] Fix bug with removing reserved node in `ProtocolController`, improve logging --- client/peerset/src/protocol_controller.rs | 82 ++++++++++++++++++----- 1 file changed, 67 insertions(+), 15 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 59463b1374fd0..fb6df4c7536ca 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -89,7 +89,7 @@ impl ProtocolHandle { let _ = self.to_controller.unbounded_send(Action::SetReservedOnly(reserved)); } - /// Disconnect peer. You should remove the `PeerId` from the `Peerset` first + /// Disconnect peer. You should remove the `PeerId` from the `PeerStore` first /// to not connect to the peer again during the next slot allocation. pub fn disconnect_peer(&self, peer_id: PeerId) { let _ = self.to_controller.unbounded_send(Action::DisconnectPeer(peer_id)); @@ -236,10 +236,12 @@ impl ProtocolController self.on_incoming_connection(peer_id, index), Action::Dropped(peer_id) => self.on_peer_dropped(peer_id).unwrap_or_else(|peer_id| { - debug_assert!(false, "Received Action::Dropped for non-connected peer."); - error!( + // We do not assert here, because due to asynchronous nature of communication + // between `ProtocolController` and `Notifications` we can receive `Action::Dropped` + // for a peer we already disconnected ourself. + warn!( target: "peerset", - "Received Action::Dropped for non-connected peer {} on {:?}.", + "Received `Action::Dropped` for not connected peer {} on {:?}.", peer_id, self.set_id, ) }), @@ -287,13 +289,29 @@ impl ProtocolController PeerState::Connected(d), - None => PeerState::NotConnected, + Some(direction) => { + trace!( + target: "peerset", + "Marking previously connected node {} ({:?}) as reserved.", + peer_id, direction, + ); + PeerState::Connected(direction) + }, + None => { + trace!(target: "peerset", "Adding reserved node {}.", peer_id); + PeerState::NotConnected + }, }; self.reserved_nodes.insert(peer_id, state.clone()); @@ -311,7 +329,14 @@ impl ProtocolController state, - None => return, + None => { + warn!( + target: "peerset", + "Trying to remove unknown reserved node: {}.", + peer_id, + ); + return + }, }; if let PeerState::Connected(direction) = state { @@ -319,20 +344,27 @@ impl ProtocolController self.num_in += 1, Direction::Outbound => self.num_out += 1, } + // Put the node into the list of regular nodes. + let prev = self.nodes.insert(peer_id, direction); + assert!(prev.is_none(), "Corrupted state: reserved node was also non-reserved."); } - // Put the node into the list of regular nodes. - let prev = self.nodes.insert(peer_id, direction); - assert!(prev.is_none(), "Corrupted state: reserved node was also non-reserved."); + } else { + trace!(target: "peerset", "Removed disconnected reserved node {}.", peer_id); } } @@ -355,6 +387,8 @@ impl ProtocolController ProtocolController { + trace!(target: "peerset", "Disconnecting peer {} ({:?}).", peer_id, direction); match direction { Direction::Inbound => self.num_in -= 1, Direction::Outbound => self.num_out -= 1, @@ -396,7 +432,7 @@ impl ProtocolController { warn!( target: "peerset", - "Trying to disconnect unknown peer {} from {:?}", + "Trying to disconnect unknown peer {} from {:?}.", peer_id, self.set_id, ); }, @@ -413,6 +449,12 @@ impl ProtocolController ProtocolController ProtocolController self.num_in -= 1, Direction::Outbound => self.num_out -= 1, @@ -1149,11 +1194,18 @@ mod tests { assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + assert_eq!(controller.reserved_nodes.len(), 2); + assert!(controller.reserved_nodes.contains_key(&reserved1)); + assert!(controller.reserved_nodes.contains_key(&reserved2)); + assert!(controller.nodes.is_empty()); // Remove reserved node controller.on_remove_reserved_peer(reserved1); assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: reserved1 }); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.reserved_nodes.len(), 1); + assert!(controller.reserved_nodes.contains_key(&reserved2)); + assert!(controller.nodes.is_empty()); } #[test] From c7aaf9fe4940fd0f19d012f9971fa9f36f8da833 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 11 Apr 2023 17:15:22 +0300 Subject: [PATCH 51/98] minor: comment --- client/peerset/src/protocol_controller.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index fb6df4c7536ca..cf8e74780b758 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -666,7 +666,8 @@ mod tests { // Add second reserved node at runtime (this currently calls `alloc_slots` internally). controller.on_add_reserved_peer(reserved2); - // Initiate connections. + // Initiate connections (currently, `alloc_slots` is also called internally in + // `on_add_reserved_peer` above). controller.alloc_slots(); let mut messages = Vec::new(); From 82c3351d9fed02eaad817e3aff2652e376631ea7 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 13 Apr 2023 13:11:00 +0300 Subject: [PATCH 52/98] Apply suggestions from code review Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> --- .../src/protocol/notifications/behaviour.rs | 2 -- client/peerset/src/peer_store.rs | 15 ++++++--------- client/peerset/src/protocol_controller.rs | 6 ++++-- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 68f52dc17753e..e60090bc5709f 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -2927,8 +2927,6 @@ mod tests { // check peer information assert_eq!(notif.open_peers().collect::>(), vec![&peer],); - // assert_eq!(notif.reserved_peers(set_id).collect::>(), Vec::<&PeerId>::new(),); - todo!("Do we really need to check reserved peers here?"); assert_eq!(notif.num_discovered_peers(), 0usize); // close the other connection and verify that notification replacement event is emitted diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 9b849d2213462..070bed59fb339 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -174,11 +174,7 @@ struct PeerStoreInner { impl PeerStoreInner { fn is_banned(&self, peer_id: &PeerId) -> bool { - if let Some(info) = self.peers.get(peer_id) { - info.is_banned() - } else { - false - } + self.peers.get(peer_id).map_or(false, |info| info.is_banned()) } fn report_disconnect(&mut self, peer_id: PeerId) { @@ -193,7 +189,7 @@ impl PeerStoreInner { } fn peer_reputation(&self, peer_id: &PeerId) -> i32 { - self.peers.get(peer_id).cloned().unwrap_or_default().reputation + self.peers.get(peer_id).map_or(0, |info| info.reputation) } fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { @@ -232,13 +228,12 @@ impl PeerStoreInner { Entry::Occupied(mut e) => { trace!( target: "peerset", - "Trying to add an already known peer {}, bumping `last_updated`.", - peer_id, + "Trying to add an already known peer {peer_id}, bumping `last_updated`.", ); e.get_mut().bump_last_updated(); }, Entry::Vacant(e) => { - trace!(target: "peerset", "Adding a new known peer {}.", peer_id); + trace!(target: "peerset", "Adding a new known peer {peer_id}."); e.insert(PeerInfo::default()); }, } @@ -272,6 +267,7 @@ impl PeerStore { pub async fn run(self) { let started = Instant::now(); let mut latest_time_update = started; + loop { let now = Instant::now(); // We basically do `(now - self.latest_update).as_secs()`, except that by the way we do @@ -282,6 +278,7 @@ impl PeerStore { latest_time_update = now; elapsed_now.as_secs() - elapsed_latest.as_secs() }; + self.inner.lock().unwrap().progress_time(seconds_passed); let _ = Delay::new(Duration::from_secs(1)).await; } diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index cf8e74780b758..9f44e22e10e51 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -285,7 +285,7 @@ impl ProtocolController ProtocolController self.num_in += 1, Direction::Outbound => self.num_out += 1, } + // Put the node into the list of regular nodes. let prev = self.nodes.insert(peer_id, direction); assert!(prev.is_none(), "Corrupted state: reserved node was also non-reserved."); @@ -553,7 +555,7 @@ impl ProtocolController Date: Thu, 13 Apr 2023 15:33:26 +0300 Subject: [PATCH 53/98] Apply more code review suggestions --- client/network/src/protocol.rs | 2 + client/network/src/service.rs | 46 ------------ client/network/src/service/traits.rs | 36 +-------- client/peerset/src/lib.rs | 34 ++------- client/peerset/src/peer_store.rs | 11 ++- client/peerset/src/protocol_controller.rs | 90 ++++++++++------------- 6 files changed, 54 insertions(+), 165 deletions(-) diff --git a/client/network/src/protocol.rs b/client/network/src/protocol.rs index df84de5403d9f..2cef251c69ce8 100644 --- a/client/network/src/protocol.rs +++ b/client/network/src/protocol.rs @@ -316,6 +316,8 @@ impl Protocol { /// /// Can be called multiple times with the same `PeerId`. pub fn add_known_peer(&mut self, peer_id: PeerId) { + // TODO: get rid of this function and call `Peerset`/`PeerStore` directly + // from `NetworkWorker`. self.peerset_handle.add_known_peer(peer_id); } } diff --git a/client/network/src/service.rs b/client/network/src/service.rs index a1066648ccb0e..1dc8bcfb84c37 100644 --- a/client/network/src/service.rs +++ b/client/network/src/service.rs @@ -882,40 +882,6 @@ where } } - // fn add_to_peers_set( - // &self, - // protocol: ProtocolName, - // peers: HashSet, - // ) -> Result<(), String> { - // let peers = self.split_multiaddr_and_peer_id(peers)?; - - // for (peer_id, addr) in peers.into_iter() { - // // Make sure the local peer ID is never added to the PSM. - // if peer_id == self.local_peer_id { - // return Err("Local peer ID cannot be added as a reserved peer.".to_string()) - // } - - // if !addr.is_empty() { - // let _ = self - // .to_worker - // .unbounded_send(ServiceToWorkerMsg::AddKnownAddress(peer_id, addr)); - // } - // let _ = self - // .to_worker - // .unbounded_send(ServiceToWorkerMsg::AddToPeersSet(protocol.clone(), peer_id)); - // } - - // Ok(()) - // } - - // fn remove_from_peers_set(&self, protocol: ProtocolName, peers: Vec) { - // for peer_id in peers.into_iter() { - // let _ = self - // .to_worker - // .unbounded_send(ServiceToWorkerMsg::RemoveFromPeersSet(protocol.clone(), peer_id)); - // } - // } - fn sync_num_connected(&self) -> usize { self.num_connected.load(Ordering::Relaxed) } @@ -1128,8 +1094,6 @@ enum ServiceToWorkerMsg { SetPeersetReserved(ProtocolName, HashSet), AddSetReserved(ProtocolName, PeerId), RemoveSetReserved(ProtocolName, PeerId), - //AddToPeersSet(ProtocolName, PeerId), - //RemoveFromPeersSet(ProtocolName, PeerId), EventStream(out_events::Sender), Request { target: PeerId, @@ -1306,16 +1270,6 @@ where .remove_set_reserved_peer(protocol, peer_id), ServiceToWorkerMsg::AddKnownAddress(peer_id, addr) => self.network_service.behaviour_mut().add_known_address(peer_id, addr), - // ServiceToWorkerMsg::AddToPeersSet(protocol, peer_id) => self - // .network_service - // .behaviour_mut() - // .user_protocol_mut() - // .add_to_peers_set(protocol, peer_id), - // ServiceToWorkerMsg::RemoveFromPeersSet(protocol, peer_id) => self - // .network_service - // .behaviour_mut() - // .user_protocol_mut() - // .remove_from_peers_set(protocol, peer_id), ServiceToWorkerMsg::EventStream(sender) => self.event_streams.push(sender), ServiceToWorkerMsg::Request { target, diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index b6ab1e557cbc9..1da890c0bae2a 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -216,26 +216,6 @@ pub trait NetworkPeers { /// Remove peers from a peer set. fn remove_peers_from_reserved_set(&self, protocol: ProtocolName, peers: Vec); - /// Add a peer to a set of peers. - /// - /// If the set has slots available, it will try to open a substream with this peer. - /// - /// Each `Multiaddr` must end with a `/p2p/` component containing the `PeerId`. It can also - /// consist of only `/p2p/`. - /// - /// Returns an `Err` if one of the given addresses is invalid or contains an - /// invalid peer ID (which includes the local peer ID). - // fn add_to_peers_set( - // &self, - // protocol: ProtocolName, - // peers: HashSet, - // ) -> Result<(), String>; - - /// Remove peers from a peer set. - /// - /// If we currently have an open substream with this peer, it will soon be closed. - //fn remove_from_peers_set(&self, protocol: ProtocolName, peers: Vec); - /// Returns the number of peers in the sync peer set we're connected to. fn sync_num_connected(&self) -> usize; } @@ -259,7 +239,9 @@ where } fn report_peer(&self, who: PeerId, cost_benefit: ReputationChange) { - // TODO: make async interface to `PeerStore`? + // TODO: when we get rid of `Peerset`, we'll likely need to add some kind of async + // interface to `PeerStore`, otherwise we'll have trouble calling functions accepting + // `&mut self` via `Arc`. T::report_peer(self, who, cost_benefit) } @@ -303,18 +285,6 @@ where T::remove_peers_from_reserved_set(self, protocol, peers) } - // fn add_to_peers_set( - // &self, - // protocol: ProtocolName, - // peers: HashSet, - // ) -> Result<(), String> { - // T::add_to_peers_set(self, protocol, peers) - // } - - // fn remove_from_peers_set(&self, protocol: ProtocolName, peers: Vec) { - // T::remove_from_peers_set(self, protocol, peers) - // } - fn sync_num_connected(&self) -> usize { T::sync_num_connected(self) } diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 924bbf8efd7c4..5dcfd6e321489 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -59,6 +59,8 @@ pub use libp2p::PeerId; pub use peer_store::BANNED_THRESHOLD; +pub const LOG_TARGET: &str = "peerset"; + #[derive(Debug)] enum Action { AddReservedPeer(SetId, PeerId), @@ -336,24 +338,6 @@ impl Peerset { self.protocol_handles[set_id.0].dropped(peer_id); } - // fn on_remove_from_peers_set(&mut self, set_id: SetId, peer_id: PeerId) { - // // Don't do anything if node is reserved. - // if self.reserved_nodes[set_id.0].0.contains(&peer_id) { - // return - // } - - // match self.data.peer(set_id.0, &peer_id) { - // peersstate::Peer::Connected(peer) => { - // self.message_queue.push_back(Message::Drop { set_id, peer_id: *peer.peer_id() }); - // peer.disconnect().forget_peer(); - // }, - // peersstate::Peer::NotConnected(peer) => { - // peer.forget_peer(); - // }, - // peersstate::Peer::Unknown(_) => {}, - // } - // } - /// Reports an adjustment to the reputation of the given peer. pub fn report_peer(&mut self, peer_id: PeerId, score_diff: ReputationChange) { // We don't immediately perform the adjustments in order to have state consistency. We @@ -398,10 +382,7 @@ impl Stream for Peerset { }, } } else { - debug!( - target: "peerset", - "`PeersetHandle` was dropped, terminating `Peerset`." - ); + debug!(target: LOG_TARGET, "`PeersetHandle` was dropped, terminating `Peerset`."); return Poll::Ready(None) } } @@ -411,7 +392,7 @@ impl Stream for Peerset { return Poll::Ready(Some(msg)) } else { debug!( - target: "peerset", + target: LOG_TARGET, "All `ProtocolController`s have terminated, terminating `Peerset`." ); return Poll::Ready(None) @@ -419,16 +400,13 @@ impl Stream for Peerset { } if let Poll::Ready(()) = self.peer_store_future.poll_unpin(cx) { - debug!( - target: "peerset", - "`PeerStore` has terminated, terminating `PeerSet`." - ); + debug!(target: LOG_TARGET, "`PeerStore` has terminated, terminating `PeerSet`."); return Poll::Ready(None) } if let Poll::Ready(_) = self.protocol_controller_futures.poll_unpin(cx) { debug!( - target: "peerset", + target: LOG_TARGET, "All `ProtocolHandle`s have terminated, terminating `PeerSet`." ); return Poll::Ready(None) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 070bed59fb339..805ec70f23b34 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -27,7 +27,7 @@ use std::{ }; use wasm_timer::Delay; -use crate::ReputationChange; +use crate::{ReputationChange, LOG_TARGET}; /// We don't accept nodes whose reputation is under this value. pub const BANNED_THRESHOLD: i32 = 82 * (i32::MIN / 100); @@ -174,7 +174,7 @@ struct PeerStoreInner { impl PeerStoreInner { fn is_banned(&self, peer_id: &PeerId) -> bool { - self.peers.get(peer_id).map_or(false, |info| info.is_banned()) + self.peers.get(peer_id).map_or(false, |info| info.is_banned()) } fn report_disconnect(&mut self, peer_id: PeerId) { @@ -204,8 +204,7 @@ impl PeerStoreInner { candidates.partial_sort(count, |(_, info1), (_, info2)| info1.cmp(info2)); candidates.iter().take(count).map(|(peer_id, _)| *peer_id).collect() - // FIXME: depending on the usage patterns, it might be more efficient to always maintain - // peers sorted. + // TODO: keep the peers sorted (in a "bi-multi-map"?) to not repeat sorting every time. } fn progress_time(&mut self, seconds_passed: u64) { @@ -227,13 +226,13 @@ impl PeerStoreInner { match self.peers.entry(peer_id) { Entry::Occupied(mut e) => { trace!( - target: "peerset", + target: LOG_TARGET, "Trying to add an already known peer {peer_id}, bumping `last_updated`.", ); e.get_mut().bump_last_updated(); }, Entry::Vacant(e) => { - trace!(target: "peerset", "Adding a new known peer {peer_id}."); + trace!(target: LOG_TARGET, "Adding a new known peer {peer_id}."); e.insert(PeerInfo::default()); }, } diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 9f44e22e10e51..e5ee6304605f9 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -37,7 +37,7 @@ use wasm_timer::Delay; use crate::{ peer_store::PeerReputationProvider, DropReason, IncomingIndex, Message, PeersetHandle, - SetConfig, SetId, + SetConfig, SetId, LOG_TARGET, }; #[derive(Debug)] @@ -49,7 +49,7 @@ enum Action { DisconnectPeer(PeerId), IncomingConnection(PeerId, IncomingIndex), Dropped(PeerId), - ReservedPeers { pending_response: oneshot::Sender> }, + GetReservedPeers(oneshot::Sender>), } /// Shared handle to [`ProtocolController`]. Distributed around the code outside of the @@ -109,7 +109,7 @@ impl ProtocolHandle { /// Get the list of reserved peers. pub fn reserved_peers(&self, pending_response: oneshot::Sender>) { - let _ = self.to_controller.unbounded_send(Action::ReservedPeers { pending_response }); + let _ = self.to_controller.unbounded_send(Action::GetReservedPeers(pending_response)); } } @@ -240,12 +240,14 @@ impl ProtocolController self.on_reserved_peers(pending_response), + Action::GetReservedPeers(pending_response) => + self.on_get_reserved_peers(pending_response), } true } @@ -290,26 +292,23 @@ impl ProtocolController { trace!( - target: "peerset", - "Marking previously connected node {} ({:?}) as reserved.", - peer_id, direction, + target: LOG_TARGET, + "Marking previously connected node {peer_id} ({direction:?}) as reserved.", ); PeerState::Connected(direction) }, None => { - trace!(target: "peerset", "Adding reserved node {}.", peer_id); + trace!(target: LOG_TARGET, "Adding reserved node {peer_id}."); PeerState::NotConnected }, }; @@ -330,11 +329,7 @@ impl ProtocolController state, None => { - warn!( - target: "peerset", - "Trying to remove unknown reserved node: {}.", - peer_id, - ); + warn!(target: LOG_TARGET, "Trying to remove unknown reserved node: {peer_id}."); return }, }; @@ -343,17 +338,18 @@ impl ProtocolController ProtocolController ProtocolController ProtocolController>) { + fn on_get_reserved_peers(&self, pending_response: oneshot::Sender>) { pending_response.send(self.reserved_nodes.keys().cloned().collect()); } @@ -414,17 +410,15 @@ impl ProtocolController { - trace!(target: "peerset", "Disconnecting peer {} ({:?}).", peer_id, direction); + trace!(target: LOG_TARGET, "Disconnecting peer {peer_id} ({direction:?})."); match direction { Direction::Inbound => self.num_in -= 1, Direction::Outbound => self.num_out -= 1, @@ -433,9 +427,8 @@ impl ProtocolController { warn!( - target: "peerset", - "Trying to disconnect unknown peer {} from {:?}.", - peer_id, self.set_id, + target: LOG_TARGET, + "Trying to disconnect unknown peer {} from {:?}.", peer_id, self.set_id, ); }, } @@ -451,11 +444,7 @@ impl ProtocolController ProtocolController {}, PeerState::NotConnected => { - // It's questionable whether we should check a reputation of reserved node. // FIXME: unable to call `self.is_banned()` because of borrowed `self`. if self.peer_store.is_banned(&peer_id) { self.reject_connection(incoming_index); @@ -523,7 +511,7 @@ impl ProtocolController ProtocolController self.num_in -= 1, @@ -555,11 +543,10 @@ impl ProtocolController>() @@ -595,9 +582,8 @@ impl ProtocolController ProtocolController available_slots { debug_assert!(false, "`PeerStore` returned more nodes than there are slots available."); error!( - target: "peerset", + target: LOG_TARGET, "`PeerStore` returned more nodes than there are slots available.", ) } From d839590737c05be18d0c8453983dcf7c40249c66 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 14 Apr 2023 11:19:07 +0300 Subject: [PATCH 54/98] minor: use boxing instead of generics for `PeerStoreHandle` reference in `ProtocolController` --- client/peerset/src/lib.rs | 2 +- client/peerset/src/peer_store.rs | 3 +- client/peerset/src/protocol_controller.rs | 79 +++++++++++++++-------- 3 files changed, 54 insertions(+), 30 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 5dcfd6e321489..c677c647060d9 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -284,7 +284,7 @@ impl Peerset { SetId::from(set), set_config, to_notifications.clone(), - peer_store.handle(), + Box::new(peer_store.handle()), ) }) .collect::>(); diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 805ec70f23b34..d9c8f4ae34914 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -22,6 +22,7 @@ use partial_sort::PartialSort; use std::{ cmp::{Ord, Ordering, PartialOrd}, collections::{hash_map::Entry, HashMap, HashSet}, + fmt::Debug, sync::{Arc, Mutex}, time::{Duration, Instant}, }; @@ -43,7 +44,7 @@ const INVERSE_DECREMENT: i32 = 50; /// remove it, once the reputation value reaches 0. const FORGET_AFTER: Duration = Duration::from_secs(3600); -pub trait PeerReputationProvider { +pub trait PeerReputationProvider: Debug + Send { /// Check whether the peer is banned. fn is_banned(&self, peer_id: &PeerId) -> bool; diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index e5ee6304605f9..aab31214953f0 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -145,7 +145,7 @@ impl Default for PeerState { /// Side of [`ProtocolHandle`] responsible for all the logic. Currently all instances are /// owned by [`crate::Peerset`], but they should eventually be moved to corresponding protocols. #[derive(Debug)] -pub struct ProtocolController { +pub struct ProtocolController { /// Set id to use when sending connect/drop requests to `Notifications`. // Will likely be replaced by `ProtocolName` in the future. set_id: SetId, @@ -169,19 +169,19 @@ pub struct ProtocolController { next_periodic_alloc_slots: Instant, /// Outgoing channel for messages to `Notifications`. to_notifications: TracingUnboundedSender, - /// Peerset handle for checking peer reputation values and getting connection candidates + /// `PeerStore` handle for checking peer reputation values and getting connection candidates /// with highest reputation. - peer_store: PeerStoreHandle, + peer_store: Box, } -impl ProtocolController { +impl ProtocolController { /// Construct new [`ProtocolController`]. pub fn new( set_id: SetId, config: SetConfig, to_notifications: TracingUnboundedSender, - peer_store: PeerStoreHandle, - ) -> (ProtocolHandle, ProtocolController) { + peer_store: Box, + ) -> (ProtocolHandle, ProtocolController) { let (to_controller, from_handle) = tracing_unbounded("mpsc_protocol_controller", 10_000); let handle = ProtocolHandle { to_controller }; let reserved_nodes = @@ -619,6 +619,7 @@ mod tests { use std::collections::HashSet; mockall::mock! { + #[derive(Debug)] pub PeerStoreHandle {} impl PeerReputationProvider for PeerStoreHandle { @@ -649,7 +650,8 @@ mod tests { peer_store.expect_is_banned().times(4).return_const(false); peer_store.expect_report_disconnect().times(2).return_const(()); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Add second reserved node at runtime (this currently calls `alloc_slots` internally). controller.on_add_reserved_peer(reserved2); @@ -709,7 +711,8 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(6).return_const(true); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Add second reserved node at runtime (this currently calls `alloc_slots` internally). controller.on_add_reserved_peer(reserved2); @@ -760,7 +763,8 @@ mod tests { peer_store.expect_is_banned().times(4).return_const(false); peer_store.expect_report_disconnect().times(2).return_const(()); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Add second reserved node at runtime (this calls `alloc_slots` internally). controller.on_add_reserved_peer(reserved2); @@ -818,7 +822,8 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_outgoing_candidates().once().return_const(candidates); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. controller.alloc_slots(); @@ -868,7 +873,8 @@ mod tests { peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. controller.alloc_slots(); @@ -909,7 +915,8 @@ mod tests { peer_store.expect_outgoing_candidates().once().return_const(candidates2); peer_store.expect_report_disconnect().times(2).return_const(()); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. controller.alloc_slots(); @@ -975,7 +982,8 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. controller.alloc_slots(); @@ -1000,7 +1008,8 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); let peer = PeerId::random(); let incoming_index = IncomingIndex(1); @@ -1037,7 +1046,8 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_outgoing_candidates().once().return_const(candidates); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. controller.alloc_slots(); @@ -1083,7 +1093,8 @@ mod tests { peer_store.expect_is_banned().times(3).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); assert_eq!(controller.num_out, 0); assert_eq!(controller.num_in, 0); @@ -1140,7 +1151,8 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); assert_eq!(controller.reserved_nodes.len(), 2); assert_eq!(controller.nodes.len(), 0); assert_eq!(controller.num_out, 0); @@ -1172,7 +1184,8 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(2).return_const(false); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. controller.alloc_slots(); @@ -1215,7 +1228,8 @@ mod tests { peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as inbound, `peer2` as outbound. controller.on_incoming_connection(peer1, IncomingIndex(1)); @@ -1260,7 +1274,8 @@ mod tests { peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as outbound & `peer2` as inbound. controller.alloc_slots(); @@ -1301,7 +1316,8 @@ mod tests { peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as outbound & `peer2` as inbound. controller.alloc_slots(); @@ -1353,7 +1369,8 @@ mod tests { peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `reserved1` as inbound & `reserved2` as outbound. controller.on_incoming_connection(reserved1, IncomingIndex(1)); @@ -1409,7 +1426,8 @@ mod tests { peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); peer_store.expect_report_disconnect().times(2).return_const(()); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as outbound & `peer2` as inbound. controller.alloc_slots(); @@ -1460,7 +1478,8 @@ mod tests { peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as outbound & `peer2` as inbound. controller.alloc_slots(); @@ -1505,7 +1524,8 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(false); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as inbound. controller.on_incoming_connection(peer1, IncomingIndex(1)); @@ -1534,7 +1554,8 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(true); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Incoming request. controller.on_incoming_connection(peer1, IncomingIndex(1)); @@ -1558,7 +1579,8 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(true); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); assert!(controller.reserved_nodes.contains_key(&reserved1)); // Incoming request. @@ -1584,7 +1606,8 @@ mod tests { peer_store.expect_is_banned().once().return_const(true); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); - let (handle, mut controller) = ProtocolController::new(SetId(0), config, tx, peer_store); + let (handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); assert!(matches!(controller.reserved_nodes.get(&reserved1), Some(PeerState::NotConnected))); // Initiate connectios From 7c8ac45e20361d073b55a2c2a9d53b765b190650 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 17 Apr 2023 17:33:41 +0300 Subject: [PATCH 55/98] Test `PeerInfo::decay_reputation` --- client/peerset/src/peer_store.rs | 75 +++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index d9c8f4ae34914..6615611e42f89 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -36,8 +36,8 @@ pub const BANNED_THRESHOLD: i32 = 82 * (i32::MIN / 100); const DISCONNECT_REPUTATION_CHANGE: i32 = -256; /// Relative decrement of a reputation value that is applied every second. I.e., for inverse /// decrement of 50 we decrease absolute value of the reputation by 1/50. This corresponds to a -/// factor of `k = 0.98`. It takes `ln(0.5) / ln(k)` seconds to reduce the reputation by half, or -/// 34.3 seconds for the values above. In this setup the maximum allowed absolute value of +/// factor of `k = 0.98`. It takes ~ `ln(0.5) / ln(k)` seconds to reduce the reputation by half, +/// or 34.3 seconds for the values above. In this setup the maximum allowed absolute value of /// `i32::MAX` becomes 0 in ~1100 seconds. const INVERSE_DECREMENT: i32 = 50; /// Amount of time between the moment we last updated the [`PeerStore`] entry and the moment we @@ -284,3 +284,74 @@ impl PeerStore { } } } + +#[cfg(test)] +mod tests { + use super::PeerInfo; + + #[test] + fn decaying_zero_reputation_yields_zero() { + let mut peer_info = PeerInfo::default(); + assert_eq!(peer_info.reputation, 0); + + peer_info.decay_reputation(1); + assert_eq!(peer_info.reputation, 0); + + peer_info.decay_reputation(100_000); + assert_eq!(peer_info.reputation, 0); + } + + #[test] + fn decaying_positive_reputation_decreases_it() { + const INITIAL_REPUTATION: i32 = 100; + + let mut peer_info = PeerInfo::default(); + peer_info.reputation = INITIAL_REPUTATION; + + peer_info.decay_reputation(1); + assert!(peer_info.reputation >= 0); + assert!(peer_info.reputation < INITIAL_REPUTATION); + } + + #[test] + fn decaying_negative_reputation_increases_it() { + const INITIAL_REPUTATION: i32 = -100; + + let mut peer_info = PeerInfo::default(); + peer_info.reputation = INITIAL_REPUTATION; + + peer_info.decay_reputation(1); + assert!(peer_info.reputation <= 0); + assert!(peer_info.reputation > INITIAL_REPUTATION); + } + + #[test] + fn decaying_max_reputation_finally_yields_zero() { + const INITIAL_REPUTATION: i32 = i32::MAX; + const SECONDS: u64 = 1000; + + let mut peer_info = PeerInfo::default(); + peer_info.reputation = INITIAL_REPUTATION; + + peer_info.decay_reputation(SECONDS / 2); + assert!(peer_info.reputation > 0); + + peer_info.decay_reputation(SECONDS / 2); + assert_eq!(peer_info.reputation, 0); + } + + #[test] + fn decaying_min_reputation_finally_yields_zero() { + const INITIAL_REPUTATION: i32 = i32::MIN; + const SECONDS: u64 = 1000; + + let mut peer_info = PeerInfo::default(); + peer_info.reputation = INITIAL_REPUTATION; + + peer_info.decay_reputation(SECONDS / 2); + assert!(peer_info.reputation < 0); + + peer_info.decay_reputation(SECONDS / 2); + assert_eq!(peer_info.reputation, 0); + } +} From 06cb192b06fd9ffbf6c2f3fc74e194a770c01c70 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 20 Apr 2023 16:44:26 +0300 Subject: [PATCH 56/98] Make concurrency issues slightly milder --- Cargo.lock | 1 + client/peerset/Cargo.toml | 1 + client/peerset/src/protocol_controller.rs | 121 +++++++++---- client/peerset/tests/fuzz.rs | 211 +++++++++++++++++----- 4 files changed, 248 insertions(+), 86 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8429d0157bcc3..caa640038dd04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9291,6 +9291,7 @@ dependencies = [ "sc-utils", "serde_json", "sp-arithmetic", + "sp-tracing", "wasm-timer", ] diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index cb8e56c0ea5f7..00a05601ee879 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -24,5 +24,6 @@ sc-utils = { version = "4.0.0-dev", path = "../utils" } sp-arithmetic = { version = "6.0.0", path = "../../primitives/arithmetic" } [dev-dependencies] +sp-tracing = { version = "6.0.0", path = "../../primitives/tracing" } mockall = "0.11.3" rand = "0.8.5" diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index aab31214953f0..4975028ff4e78 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -24,7 +24,7 @@ // TODO: remove this line. #![allow(unused)] -use futures::{channel::oneshot, FutureExt, StreamExt}; +use futures::{channel::oneshot, future::Either, FutureExt, StreamExt}; use libp2p::PeerId; use log::{error, info, trace, warn}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; @@ -47,16 +47,23 @@ enum Action { SetReservedPeers(HashSet), SetReservedOnly(bool), DisconnectPeer(PeerId), + GetReservedPeers(oneshot::Sender>), +} + +#[derive(Debug)] +enum Event { IncomingConnection(PeerId, IncomingIndex), Dropped(PeerId), - GetReservedPeers(oneshot::Sender>), } /// Shared handle to [`ProtocolController`]. Distributed around the code outside of the /// protocol implementation. #[derive(Debug, Clone)] pub struct ProtocolHandle { - to_controller: TracingUnboundedSender, + /// Actions from outer API. + actions_tx: TracingUnboundedSender, + /// Connection events from `Notifications`. We prioritize them over actions. + events_tx: TracingUnboundedSender, } impl ProtocolHandle { @@ -68,48 +75,48 @@ impl ProtocolHandle { /// > **Note**: Keep in mind that the networking has to know an address for this node, /// > otherwise it will not be able to connect to it. pub fn add_reserved_peer(&self, peer_id: PeerId) { - let _ = self.to_controller.unbounded_send(Action::AddReservedPeer(peer_id)); + let _ = self.actions_tx.unbounded_send(Action::AddReservedPeer(peer_id)); } /// Demotes reserved peer to non-reserved. Does not disconnect the peer. /// /// Has no effect if the node was not a reserved peer. pub fn remove_reserved_peer(&self, peer_id: PeerId) { - let _ = self.to_controller.unbounded_send(Action::RemoveReservedPeer(peer_id)); + let _ = self.actions_tx.unbounded_send(Action::RemoveReservedPeer(peer_id)); } /// Set reserved peers to the new set. pub fn set_reserved_peers(&self, peer_ids: HashSet) { - let _ = self.to_controller.unbounded_send(Action::SetReservedPeers(peer_ids)); + let _ = self.actions_tx.unbounded_send(Action::SetReservedPeers(peer_ids)); } /// Sets whether or not [`ProtocolController`] only has connections with nodes marked /// as reserved for the given set. pub fn set_reserved_only(&self, reserved: bool) { - let _ = self.to_controller.unbounded_send(Action::SetReservedOnly(reserved)); + let _ = self.actions_tx.unbounded_send(Action::SetReservedOnly(reserved)); } /// Disconnect peer. You should remove the `PeerId` from the `PeerStore` first /// to not connect to the peer again during the next slot allocation. pub fn disconnect_peer(&self, peer_id: PeerId) { - let _ = self.to_controller.unbounded_send(Action::DisconnectPeer(peer_id)); + let _ = self.actions_tx.unbounded_send(Action::DisconnectPeer(peer_id)); + } + + /// Get the list of reserved peers. + pub fn reserved_peers(&self, pending_response: oneshot::Sender>) { + let _ = self.actions_tx.unbounded_send(Action::GetReservedPeers(pending_response)); } /// Notify about incoming connection. [`ProtocolController`] will either accept or reject it. pub fn incoming_connection(&self, peer_id: PeerId, incoming_index: IncomingIndex) { let _ = self - .to_controller - .unbounded_send(Action::IncomingConnection(peer_id, incoming_index)); + .events_tx + .unbounded_send(Event::IncomingConnection(peer_id, incoming_index)); } /// Notify that connection was dropped (either refused or disconnected). pub fn dropped(&self, peer_id: PeerId) { - let _ = self.to_controller.unbounded_send(Action::Dropped(peer_id)); - } - - /// Get the list of reserved peers. - pub fn reserved_peers(&self, pending_response: oneshot::Sender>) { - let _ = self.to_controller.unbounded_send(Action::GetReservedPeers(pending_response)); + let _ = self.events_tx.unbounded_send(Event::Dropped(peer_id)); } } @@ -149,8 +156,10 @@ pub struct ProtocolController { /// Set id to use when sending connect/drop requests to `Notifications`. // Will likely be replaced by `ProtocolName` in the future. set_id: SetId, - /// Receiver for messages from [`ProtocolHandle`]. - from_handle: TracingUnboundedReceiver, + /// Receiver for outer API messages from [`ProtocolHandle`]. + actions_rx: TracingUnboundedReceiver, + /// Receiver for connection events from `Notifications` sent via [`ProtocolHandle`]. + events_rx: TracingUnboundedReceiver, /// Number of occupied slots for incoming connections (not counting reserved nodes). num_in: u32, /// Number of occupied slots for outgoing connections (not counting reserved nodes). @@ -182,13 +191,15 @@ impl ProtocolController { to_notifications: TracingUnboundedSender, peer_store: Box, ) -> (ProtocolHandle, ProtocolController) { - let (to_controller, from_handle) = tracing_unbounded("mpsc_protocol_controller", 10_000); - let handle = ProtocolHandle { to_controller }; + let (actions_tx, actions_rx) = tracing_unbounded("mpsc_api_protocol", 10_000); + let (events_tx, events_rx) = tracing_unbounded("mpsc_notifications_protocol", 10_000); + let handle = ProtocolHandle { actions_tx, events_tx }; let reserved_nodes = config.reserved_nodes.iter().map(|p| (*p, PeerState::NotConnected)).collect(); let controller = ProtocolController { set_id, - from_handle, + actions_rx, + events_rx, num_in: 0, num_out: 0, max_in: config.in_peers, @@ -213,11 +224,23 @@ impl ProtocolController { /// /// Intended for tests only. Use `run` for driving [`ProtocolController`]. pub async fn next_action(&mut self) -> bool { - let action = loop { + // Perform tasks prioritizing connection events processing. + let either = loop { + if let Some(event) = self.events_rx.next().now_or_never() { + match event { + None => return false, + Some(event) => break Either::Left(event), + } + } + let mut next_alloc_slots = Delay::new_at(self.next_periodic_alloc_slots).fuse(); futures::select! { - action = self.from_handle.next() => match action { - Some(action) => break action, + event = self.events_rx.next() => match event { + Some(event) => break Either::Left(event), + None => return false, + }, + action = self.actions_rx.next() => match action { + Some(action) => break Either::Right(action), None => return false, }, _ = next_alloc_slots => { @@ -227,29 +250,34 @@ impl ProtocolController { } }; + match either { + Either::Left(event) => self.process_event(event), + Either::Right(action) => self.process_action(action), + } + + true + } + + /// Process connection event. + fn process_event(&mut self, event: Event) { + match event { + Event::IncomingConnection(peer_id, index) => + self.on_incoming_connection(peer_id, index), + Event::Dropped(peer_id) => self.on_peer_dropped(peer_id), + } + } + + /// Process action command. + fn process_action(&mut self, action: Action) { match action { Action::AddReservedPeer(peer_id) => self.on_add_reserved_peer(peer_id), Action::RemoveReservedPeer(peer_id) => self.on_remove_reserved_peer(peer_id), Action::SetReservedPeers(peer_ids) => self.on_set_reserved_peers(peer_ids), Action::SetReservedOnly(reserved_only) => self.on_set_reserved_only(reserved_only), Action::DisconnectPeer(peer_id) => self.on_disconnect_peer(peer_id), - Action::IncomingConnection(peer_id, index) => - self.on_incoming_connection(peer_id, index), - Action::Dropped(peer_id) => self.on_peer_dropped(peer_id).unwrap_or_else(|peer_id| { - // We do not assert here, because due to asynchronous nature of communication - // between `ProtocolController` and `Notifications` we can receive `Action::Dropped` - // for a peer we already disconnected ourself. - warn!( - target: LOG_TARGET, - "Received `Action::Dropped` for not connected peer {} on {:?}.", - peer_id, - self.set_id, - ) - }), Action::GetReservedPeers(pending_response) => self.on_get_reserved_peers(pending_response), } - true } /// Send "accept" message to `Notifications`. @@ -489,9 +517,24 @@ impl ProtocolController { self.accept_connection(incoming_index); } + /// Indicate that a connection with the peer was dropped. + fn on_peer_dropped(&mut self, peer_id: PeerId) { + self.on_peer_dropped_inner(peer_id).unwrap_or_else(|peer_id| { + // We do not assert here, because due to asynchronous nature of communication + // between `ProtocolController` and `Notifications` we can receive `Action::Dropped` + // for a peer we already disconnected ourself. + warn!( + target: LOG_TARGET, + "Received `Action::Dropped` for not connected peer {} on {:?}.", + peer_id, + self.set_id, + ) + }); + } + /// Indicate that a connection with the peer was dropped. /// Returns `Err(PeerId)` if the peer wasn't connected or is not known to us. - fn on_peer_dropped(&mut self, peer_id: PeerId) -> Result<(), PeerId> { + fn on_peer_dropped_inner(&mut self, peer_id: PeerId) -> Result<(), PeerId> { if self.drop_reserved_peer(&peer_id)? || self.drop_regular_peer(&peer_id) { // The peer found and disconnected. self.report_disconnect(peer_id); diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index 6f7d131127bc9..83da87a3cba33 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -31,14 +31,72 @@ use std::{ task::Poll, }; +/// Peer events as observed by `Notifications` (fuzz test). +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +enum Event { + /// Either API requested to disconnect from the peer, or the peer dropped. + Disconnected, + /// Incoming request. + Incoming, + /// Answer from PSM: accept. + PsmAccept, + /// Answer from PSM: reject. + PsmReject, + /// Command from PSM: connect. + PsmConnect, + /// Command from PSM: drop connection. + PsmDrop, +} + #[test] fn run() { + sp_tracing::try_init_simple(); + for _ in 0..50 { test_once(); } } fn test_once() { + // Allowed bigrams of peer events. Events marked by /* */ are invalid, but can be safely + // dropped. + let allowed_events: HashMap> = [ + ( + Event::Disconnected, + [Event::Incoming, Event::PsmConnect, Event::PsmDrop /* synonymous */].into_iter().collect::>(), + ), + ( + Event::Incoming, + [Event::PsmAccept, Event::PsmReject, Event::PsmConnect, Event::PsmDrop /* must be ignored, will be obsoleted by the answer to `Incoming` */] + .into_iter() + .collect::>(), + ), + ( + Event::PsmAccept, + [Event::Disconnected, Event::PsmDrop].into_iter().collect::>(), + ), + ( + Event::PsmReject, + [Event::Disconnected, Event::Incoming, Event::PsmConnect] + .into_iter() + .collect::>(), + ), + ( + Event::PsmConnect, + [Event::Disconnected, Event::PsmAccept /* direction switch */, Event::PsmDrop] + .into_iter() + .collect::>(), + ), + ( + Event::PsmDrop, + [Event::Incoming, Event::PsmReject /* synonymous */, Event::PsmConnect] + .into_iter() + .collect::>(), + ), + ] + .into_iter() + .collect(); + // PRNG to use. let mut rng = rand::thread_rng(); @@ -49,29 +107,35 @@ fn test_once() { let (mut peerset, peerset_handle) = Peerset::from_config(PeersetConfig { sets: vec![SetConfig { - bootnodes: (0..Uniform::new_inclusive(0, 4).sample(&mut rng)) - .map(|_| { - let id = PeerId::random(); - known_nodes.insert(id); - id - }) - .collect(), - reserved_nodes: { - (0..Uniform::new_inclusive(0, 2).sample(&mut rng)) - .map(|_| { - let id = PeerId::random(); - known_nodes.insert(id); - reserved_nodes.insert(id); - id - }) - .collect() - }, + // bootnodes: (0..Uniform::new_inclusive(0, 4).sample(&mut rng)) + // .map(|_| { + // let id = PeerId::random(); + // known_nodes.insert(id); + // id + // }) + // .collect(), + // reserved_nodes: { + // (0..Uniform::new_inclusive(0, 2).sample(&mut rng)) + // .map(|_| { + // let id = PeerId::random(); + // known_nodes.insert(id); + // reserved_nodes.insert(id); + // id + // }) + // .collect() + // }, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), in_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), out_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), reserved_only: Uniform::new_inclusive(0, 10).sample(&mut rng) == 0, }], }); + let new_id = PeerId::random(); + known_nodes.insert(new_id); + peerset_handle.add_known_peer(new_id); + futures::executor::block_on(futures::future::poll_fn(move |cx| { // List of nodes the user of `peerset` assumes it's connected to. Always a subset of // `known_nodes`. @@ -81,45 +145,71 @@ fn test_once() { let mut incoming_nodes = HashMap::::new(); // Next id for incoming connections. let mut next_incoming_id = IncomingIndex(0); + // Last event with the peer for bigram validation. + let mut events = HashMap::::new(); // Perform a certain number of actions while checking that the state is consistent. If we // reach the end of the loop, the run has succeeded. - for _ in 0..2500 { + for _ in 0..25000 { + // Peer we are working with. + let mut current_peer = None; + // Current event for event bigrams validation. + let mut current_event = None; + // Each of these weights corresponds to an action that we may perform. let action_weights = [150, 90, 90, 30, 30, 1, 1, 4, 4]; match WeightedIndex::new(&action_weights).unwrap().sample(&mut rng) { // If we generate 0, poll the peerset. - 0 => loop { - match Stream::poll_next(Pin::new(&mut peerset), cx) { - Poll::Ready(Some(Message::Connect { peer_id, .. })) => { - if let Some(id) = incoming_nodes - .iter() - .find(|(_, v)| **v == peer_id) - .map(|(&id, _)| id) - { - incoming_nodes.remove(&id); - } - assert!(connected_nodes.insert(peer_id)); - }, - Poll::Ready(Some(Message::Drop { peer_id, .. })) => { - connected_nodes.remove(&peer_id); - }, - Poll::Ready(Some(Message::Accept(n))) => { - assert!(connected_nodes.insert(incoming_nodes.remove(&n).unwrap())) - }, - Poll::Ready(Some(Message::Reject(n))) => { - assert!(!connected_nodes.contains(&incoming_nodes.remove(&n).unwrap())) - }, - Poll::Ready(None) => panic!(), - Poll::Pending => break, - } + 0 => match Stream::poll_next(Pin::new(&mut peerset), cx) { + Poll::Ready(Some(Message::Connect { peer_id, .. })) => { + log::info!("PSM: connecting to peer {}", peer_id); + if let Some(id) = incoming_nodes + .iter() + .find(|(_, v)| **v == peer_id) + .map(|(&id, _)| id) + { + // incoming_nodes.remove(&id); + // Do not remove the incoming index for subsequent bigram validation. + } + assert!(connected_nodes.insert(peer_id)); + + current_peer = Some(peer_id); + current_event = Some(Event::PsmConnect); + }, + Poll::Ready(Some(Message::Drop { peer_id, .. })) => { + log::info!("PSM: dropping peer {}", peer_id); + connected_nodes.remove(&peer_id); + + current_peer = Some(peer_id); + current_event = Some(Event::PsmDrop); + }, + Poll::Ready(Some(Message::Accept(n))) => { + log::info!("PSM: accepting index {}", n.0); + + let peer_id = incoming_nodes.remove(&n).unwrap(); + assert!(connected_nodes.insert(peer_id)); + + current_peer = Some(peer_id); + current_event = Some(Event::PsmAccept); + }, + Poll::Ready(Some(Message::Reject(n))) => { + log::info!("PSM: rejecting index {}", n.0); + + let peer_id = incoming_nodes.remove(&n).unwrap(); + assert!(!connected_nodes.contains(&peer_id)); + + current_peer = Some(peer_id); + current_event = Some(Event::PsmReject); + }, + Poll::Ready(None) => panic!(), + Poll::Pending => {}, }, // If we generate 1, discover a new node. 1 => { - let new_id = PeerId::random(); - known_nodes.insert(new_id); - peerset_handle.add_known_peer(new_id); + // let new_id = PeerId::random(); + // known_nodes.insert(new_id); + // peerset_handle.add_known_peer(new_id); }, // If we generate 2, adjust a random reputation. @@ -132,8 +222,12 @@ fn test_once() { // If we generate 3, disconnect from a random node. 3 => if let Some(id) = connected_nodes.iter().choose(&mut rng).cloned() { + log::info!("Disconnected from {}", id); connected_nodes.remove(&id); peerset.dropped(SetId::from(0), id, DropReason::Unknown); + + current_peer = Some(id); + current_event = Some(Event::Disconnected); }, // If we generate 4, connect to a random node. @@ -146,33 +240,56 @@ fn test_once() { }) .choose(&mut rng) { + log::info!("Incoming connection from {}, index {}", id, next_incoming_id.0); peerset.incoming(SetId::from(0), *id, next_incoming_id); incoming_nodes.insert(next_incoming_id, *id); next_incoming_id.0 += 1; + + current_peer = Some(*id); + current_event = Some(Event::Incoming); } }, // 5 and 6 are the reserved-only mode. - 5 => peerset_handle.set_reserved_only(SetId::from(0), true), - 6 => peerset_handle.set_reserved_only(SetId::from(0), false), + 5 => { + log::info!("Set reserved only"); + peerset_handle.set_reserved_only(SetId::from(0), true); + }, + 6 => { + log::info!("Unset reserved only"); + peerset_handle.set_reserved_only(SetId::from(0), false); + }, // 7 and 8 are about switching a random node in or out of reserved mode. 7 => { if let Some(id) = known_nodes.iter().filter(|n| !reserved_nodes.contains(*n)).choose(&mut rng) { + log::info!("Add reserved: {}", id); peerset_handle.add_reserved_peer(SetId::from(0), *id); reserved_nodes.insert(*id); } }, 8 => if let Some(id) = reserved_nodes.iter().choose(&mut rng).cloned() { + log::info!("Remove reserved: {}", id); reserved_nodes.remove(&id); peerset_handle.remove_reserved_peer(SetId::from(0), id); }, _ => unreachable!(), } + + // Validate event bigrams. + if let Some(peer_id) = current_peer { + let event = current_event.unwrap(); + if let Some(last_event) = events.insert(peer_id, event) { + if !allowed_events.get(&last_event).unwrap().contains(&event) { + panic!("Invalid event bigram: {:?} -> {:?}", last_event, event); + } + } + } + } Poll::Ready(()) From bc4f6083aee0884be36f2f9bbc47cdc5faac270e Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 20 Apr 2023 17:27:11 +0300 Subject: [PATCH 57/98] Improve situation a little more --- client/peerset/src/lib.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index c677c647060d9..5adfe5a47d9b6 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -362,6 +362,18 @@ impl Stream for Peerset { type Item = Message; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { + if let Poll::Ready(msg) = self.from_controllers.poll_next_unpin(cx) { + if let Some(msg) = msg { + return Poll::Ready(Some(msg)) + } else { + debug!( + target: LOG_TARGET, + "All `ProtocolController`s have terminated, terminating `Peerset`." + ); + return Poll::Ready(None) + } + } + while let Poll::Ready(action) = self.from_handle.poll_next_unpin(cx) { if let Some(action) = action { match action { @@ -387,18 +399,6 @@ impl Stream for Peerset { } } - if let Poll::Ready(msg) = self.from_controllers.poll_next_unpin(cx) { - if let Some(msg) = msg { - return Poll::Ready(Some(msg)) - } else { - debug!( - target: LOG_TARGET, - "All `ProtocolController`s have terminated, terminating `Peerset`." - ); - return Poll::Ready(None) - } - } - if let Poll::Ready(()) = self.peer_store_future.poll_unpin(cx) { debug!(target: LOG_TARGET, "`PeerStore` has terminated, terminating `PeerSet`."); return Poll::Ready(None) From 8ed59d3afc822f02764d67a2ab5b9cf13221b873 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 20 Apr 2023 18:13:51 +0300 Subject: [PATCH 58/98] Try to make it work using rendezvous channels --- Cargo.lock | 28 +++++++++++++++ client/peerset/Cargo.toml | 1 + client/peerset/src/lib.rs | 35 ++++++++++++------ client/peerset/src/protocol_controller.rs | 43 ++++++++++++++--------- client/peerset/tests/fuzz.rs | 12 +++---- 5 files changed, 86 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index caa640038dd04..4f3e19a186dfc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2214,6 +2214,19 @@ dependencies = [ "num-traits", ] +[[package]] +name = "flume" +version = "0.10.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" +dependencies = [ + "futures-core", + "futures-sink", + "nanorand", + "pin-project", + "spin 0.9.5", +] + [[package]] name = "fnv" version = "1.0.7" @@ -2844,8 +2857,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -4850,6 +4865,15 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.8", +] + [[package]] name = "netlink-packet-core" version = "0.4.2" @@ -9282,6 +9306,7 @@ dependencies = [ name = "sc-peerset" version = "4.0.0-dev" dependencies = [ + "flume", "futures", "libp2p", "log", @@ -11013,6 +11038,9 @@ name = "spin" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dccf47db1b41fa1573ed27ccf5e08e3ca771cb994f776668c5ebda893b248fc" +dependencies = [ + "lock_api", +] [[package]] name = "spki" diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index 00a05601ee879..b789c8cd035e9 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -15,6 +15,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = "0.3.21" +flume = "0.10.14" libp2p = "0.50.0" log = "0.4.17" partial_sort = "0.2.0" diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 5adfe5a47d9b6..b299a53b2e2c5 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -44,16 +44,15 @@ use futures::{ prelude::*, stream::Stream, }; +use flume::{bounded, Receiver, TryRecvError}; use log::{debug, error, trace}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use serde_json::json; use std::{ - collections::{HashMap, HashSet, VecDeque}, + collections::HashSet, pin::Pin, task::{Context, Poll}, - time::{Duration, Instant}, }; -use wasm_timer::Delay; pub use libp2p::PeerId; @@ -259,7 +258,7 @@ pub struct Peerset { protocol_controller_futures: JoinAll>, /// Commands sent from protocol controllers to `Notifications`. The size of this vector never /// changes. - from_controllers: TracingUnboundedReceiver, + from_controllers: Receiver, /// Receiver for messages from the `PeersetHandle` and from `to_self`. from_handle: TracingUnboundedReceiver, /// Sending side of `from_handle`. @@ -272,8 +271,10 @@ impl Peerset { let default_set_config = &config.sets[0]; let peer_store = PeerStore::new(default_set_config.bootnodes.clone()); - let (to_notifications, from_controllers) = - tracing_unbounded("mpsc_protocol_controllers_to_notifications", 10_000); + //let (to_notifications, from_controllers) = + // tracing_unbounded("mpsc_protocol_controllers_to_notifications", 10_000); + + let (to_notifications, from_controllers) = bounded(0); let controllers = config .sets @@ -362,16 +363,28 @@ impl Stream for Peerset { type Item = Message; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - if let Poll::Ready(msg) = self.from_controllers.poll_next_unpin(cx) { - if let Some(msg) = msg { - return Poll::Ready(Some(msg)) - } else { + // if let Poll::Ready(msg) = self.from_controllers.poll_next_unpin(cx) { + // if let Some(msg) = msg { + // return Poll::Ready(Some(msg)) + // } else { + // debug!( + // target: LOG_TARGET, + // "All `ProtocolController`s have terminated, terminating `Peerset`." + // ); + // return Poll::Ready(None) + // } + // } + + match self.from_controllers.try_recv() { + Ok(msg) => return Poll::Ready(Some(msg)), + Err(TryRecvError::Disconnected) => { debug!( target: LOG_TARGET, "All `ProtocolController`s have terminated, terminating `Peerset`." ); return Poll::Ready(None) - } + }, + Err(TryRecvError::Empty) => (), } while let Poll::Ready(action) = self.from_handle.poll_next_unpin(cx) { diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 4975028ff4e78..24c0db5be3606 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -25,12 +25,14 @@ #![allow(unused)] use futures::{channel::oneshot, future::Either, FutureExt, StreamExt}; +use flume::Sender; use libp2p::PeerId; use log::{error, info, trace, warn}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_arithmetic::traits::SaturatedConversion; use std::{ - collections::{hash_map::Entry, HashMap, HashSet}, + collections::{hash_map::Entry, HashMap, HashSet, VecDeque}, + sync::mpsc::SyncSender, time::{Duration, Instant}, }; use wasm_timer::Delay; @@ -176,8 +178,10 @@ pub struct ProtocolController { reserved_only: bool, /// Next time to allocate slots. This is done once per second. next_periodic_alloc_slots: Instant, + /// Queue of messages for `Notifications` + notif_queue: VecDeque, /// Outgoing channel for messages to `Notifications`. - to_notifications: TracingUnboundedSender, + to_notifications: Sender, /// `PeerStore` handle for checking peer reputation values and getting connection candidates /// with highest reputation. peer_store: Box, @@ -188,7 +192,7 @@ impl ProtocolController { pub fn new( set_id: SetId, config: SetConfig, - to_notifications: TracingUnboundedSender, + to_notifications: Sender, peer_store: Box, ) -> (ProtocolHandle, ProtocolController) { let (actions_tx, actions_rx) = tracing_unbounded("mpsc_api_protocol", 10_000); @@ -208,6 +212,7 @@ impl ProtocolController { reserved_nodes, reserved_only: config.reserved_only, next_periodic_alloc_slots: Instant::now(), + notif_queue: VecDeque::new(), to_notifications, peer_store, }; @@ -224,8 +229,13 @@ impl ProtocolController { /// /// Intended for tests only. Use `run` for driving [`ProtocolController`]. pub async fn next_action(&mut self) -> bool { - // Perform tasks prioritizing connection events processing. let either = loop { + for msg in self.notif_queue.drain(..) { + let _ = self.to_notifications.send_async(msg).await; + // TODO: handle errors. + } + + if let Some(event) = self.events_rx.next().now_or_never() { match event { None => return false, @@ -281,27 +291,27 @@ impl ProtocolController { } /// Send "accept" message to `Notifications`. - fn accept_connection(&self, incoming_index: IncomingIndex) { - let _ = self.to_notifications.unbounded_send(Message::Accept(incoming_index)); + fn accept_connection(&mut self, incoming_index: IncomingIndex) { + let _ = self.notif_queue.push_back(Message::Accept(incoming_index)); } /// Send "reject" message to `Notifications`. - fn reject_connection(&self, incoming_index: IncomingIndex) { - let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); + fn reject_connection(&mut self, incoming_index: IncomingIndex) { + let _ = self.notif_queue.push_back(Message::Reject(incoming_index)); } /// Send "connect" message to `Notifications`. - fn start_connection(&self, peer_id: PeerId) { + fn start_connection(&mut self, peer_id: PeerId) { let _ = self - .to_notifications - .unbounded_send(Message::Connect { set_id: self.set_id, peer_id }); + .notif_queue + .push_back(Message::Connect { set_id: self.set_id, peer_id }); } /// Send "drop" message to `Notifications`. - fn drop_connection(&self, peer_id: PeerId) { + fn drop_connection(&mut self, peer_id: PeerId) { let _ = self - .to_notifications - .unbounded_send(Message::Drop { set_id: self.set_id, peer_id }); + .notif_queue + .push_back(Message::Drop { set_id: self.set_id, peer_id }); } /// Report peer disconnect event to `PeerStore` for it to update peer's reputation accordingly. @@ -422,7 +432,8 @@ impl ProtocolController { } // Disconnect all non-reserved peers. - self.nodes.keys().for_each(|peer_id| self.drop_connection(*peer_id)); + let non_reserved: Vec<_> = self.nodes.keys().cloned().collect(); + non_reserved.iter().for_each(|peer_id| self.drop_connection(*peer_id)); self.nodes.clear(); self.num_in = 0; self.num_out = 0; @@ -649,7 +660,7 @@ impl ProtocolController { } } -#[cfg(test)] +#[cfg(disabled)] mod tests { use super::{Direction, PeerState, ProtocolController}; use crate::{ diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index 83da87a3cba33..83bf42230ac4b 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -52,7 +52,7 @@ enum Event { fn run() { sp_tracing::try_init_simple(); - for _ in 0..50 { + for _ in 0..500 { test_once(); } } @@ -63,11 +63,11 @@ fn test_once() { let allowed_events: HashMap> = [ ( Event::Disconnected, - [Event::Incoming, Event::PsmConnect, Event::PsmDrop /* synonymous */].into_iter().collect::>(), + [Event::Incoming, Event::PsmConnect/*, Event::PsmDrop synonymous */].into_iter().collect::>(), ), ( Event::Incoming, - [Event::PsmAccept, Event::PsmReject, Event::PsmConnect, Event::PsmDrop /* must be ignored, will be obsoleted by the answer to `Incoming` */] + [Event::PsmAccept, Event::PsmReject, Event::PsmConnect/*, Event::PsmDrop must be ignored, will be obsoleted by the answer to `Incoming` */] .into_iter() .collect::>(), ), @@ -83,13 +83,13 @@ fn test_once() { ), ( Event::PsmConnect, - [Event::Disconnected, Event::PsmAccept /* direction switch */, Event::PsmDrop] + [Event::Disconnected/*, Event::PsmAccept direction switch */, Event::PsmDrop] .into_iter() .collect::>(), ), ( Event::PsmDrop, - [Event::Incoming, Event::PsmReject /* synonymous */, Event::PsmConnect] + [Event::Incoming/*, Event::PsmReject synonymous */, Event::PsmConnect] .into_iter() .collect::>(), ), @@ -150,7 +150,7 @@ fn test_once() { // Perform a certain number of actions while checking that the state is consistent. If we // reach the end of the loop, the run has succeeded. - for _ in 0..25000 { + for _ in 0..250000 { // Peer we are working with. let mut current_peer = None; // Current event for event bigrams validation. From a2b9ca4c39188999d8236bc61283539cd6658081 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 26 Apr 2023 17:15:32 +0300 Subject: [PATCH 59/98] Make fuzz test skip over commands between `incoming` and `accept`/`reject` --- client/peerset/src/lib.rs | 2 +- client/peerset/src/protocol_controller.rs | 35 +-- client/peerset/tests/fuzz.rs | 250 +++++++++++++++------- 3 files changed, 195 insertions(+), 92 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index b299a53b2e2c5..8d2f6ed1703f5 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -38,13 +38,13 @@ mod protocol_controller; use peer_store::{PeerReputationProvider, PeerStore, PeerStoreHandle}; use protocol_controller::{ProtocolController, ProtocolHandle}; +use flume::{bounded, Receiver, TryRecvError}; use futures::{ channel::oneshot, future::{join_all, BoxFuture, JoinAll}, prelude::*, stream::Stream, }; -use flume::{bounded, Receiver, TryRecvError}; use log::{debug, error, trace}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use serde_json::json; diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 24c0db5be3606..703c8dd4ac62b 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -24,8 +24,8 @@ // TODO: remove this line. #![allow(unused)] -use futures::{channel::oneshot, future::Either, FutureExt, StreamExt}; use flume::Sender; +use futures::{channel::oneshot, future::Either, FutureExt, StreamExt}; use libp2p::PeerId; use log::{error, info, trace, warn}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; @@ -235,7 +235,6 @@ impl ProtocolController { // TODO: handle errors. } - if let Some(event) = self.events_rx.next().now_or_never() { match event { None => return false, @@ -302,16 +301,12 @@ impl ProtocolController { /// Send "connect" message to `Notifications`. fn start_connection(&mut self, peer_id: PeerId) { - let _ = self - .notif_queue - .push_back(Message::Connect { set_id: self.set_id, peer_id }); + let _ = self.notif_queue.push_back(Message::Connect { set_id: self.set_id, peer_id }); } /// Send "drop" message to `Notifications`. fn drop_connection(&mut self, peer_id: PeerId) { - let _ = self - .notif_queue - .push_back(Message::Drop { set_id: self.set_id, peer_id }); + let _ = self.notif_queue.push_back(Message::Drop { set_id: self.set_id, peer_id }); } /// Report peer disconnect event to `PeerStore` for it to update peer's reputation accordingly. @@ -493,8 +488,12 @@ impl ProtocolController { // Check if the node is reserved first. if let Some(state) = self.reserved_nodes.get_mut(&peer_id) { match state { - // If we're already connected, don't answer, as the docs mention. - PeerState::Connected(_) => {}, + PeerState::Connected(mut direction) => { + // We are accepting an incoming connection, so switch the direction to inbound. + // (See the note above.) + direction = Direction::Inbound; + self.accept_connection(incoming_index); + }, PeerState::NotConnected => { // FIXME: unable to call `self.is_banned()` because of borrowed `self`. if self.peer_store.is_banned(&peer_id) { @@ -508,9 +507,19 @@ impl ProtocolController { return } - // If we're already connected, don't answer, as the docs mention. - if self.nodes.contains_key(&peer_id) { - return + // If we're already connected, pretend we are not connected and decide on the node again. + // (See the note above.) + if let Some(direction) = self.nodes.remove(&peer_id) { + trace!( + target: LOG_TARGET, + "Handling incoming connection from peer {} we think we already connected as {:?}.", + peer_id, + direction, + ); + match direction { + Direction::Inbound => self.num_in -= 1, + Direction::Outbound => self.num_out -= 1, + } } if self.num_in >= self.max_in { diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index 83bf42230ac4b..dae28f23327a4 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -31,7 +31,7 @@ use std::{ task::Poll, }; -/// Peer events as observed by `Notifications` (fuzz test). +/// Peer events as observed by `Notifications` / fuzz test. #[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] enum Event { /// Either API requested to disconnect from the peer, or the peer dropped. @@ -48,50 +48,70 @@ enum Event { PsmDrop, } +/// Simplified peer state as thought by `Notifications` / fuzz test. +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +enum State { + /// Peer is not connected. + Disconnected, + /// We have an inbound connection, but have not decided yet whether to accept it. + Incoming(IncomingIndex), + /// Peer is connected via an inbound connection. + Inbound, + /// Peer is connected via an outbound connection. + Outbound, +} + +/// Bare simplified state without incoming index. +#[derive(Debug, PartialEq, Eq, Hash, Clone, Copy)] +enum BareState { + /// Peer is not connected. + Disconnected, + /// We have an inbound connection, but have not decided yet whether to accept it. + Incoming, + /// Peer is connected via an inbound connection. + Inbound, + /// Peer is connected via an outbound connection. + Outbound, +} + +fn discard_incoming_index(state: State) -> BareState { + match state { + State::Disconnected => BareState::Disconnected, + State::Incoming(_) => BareState::Incoming, + State::Inbound => BareState::Inbound, + State::Outbound => BareState::Outbound, + } +} + #[test] fn run() { sp_tracing::try_init_simple(); - for _ in 0..500 { + for _ in 0..50 { test_once(); } } fn test_once() { - // Allowed bigrams of peer events. Events marked by /* */ are invalid, but can be safely - // dropped. - let allowed_events: HashMap> = [ - ( - Event::Disconnected, - [Event::Incoming, Event::PsmConnect/*, Event::PsmDrop synonymous */].into_iter().collect::>(), - ), + // Allowed events that can be received in a specific state. + let allowed_events: HashMap> = [ ( - Event::Incoming, - [Event::PsmAccept, Event::PsmReject, Event::PsmConnect/*, Event::PsmDrop must be ignored, will be obsoleted by the answer to `Incoming` */] + BareState::Disconnected, + [Event::Incoming, Event::PsmConnect, Event::PsmDrop /* must be ignored */] .into_iter() .collect::>(), ), ( - Event::PsmAccept, - [Event::Disconnected, Event::PsmDrop].into_iter().collect::>(), + BareState::Incoming, + [Event::PsmAccept, Event::PsmReject].into_iter().collect::>(), ), ( - Event::PsmReject, - [Event::Disconnected, Event::Incoming, Event::PsmConnect] - .into_iter() - .collect::>(), - ), - ( - Event::PsmConnect, - [Event::Disconnected/*, Event::PsmAccept direction switch */, Event::PsmDrop] - .into_iter() - .collect::>(), + BareState::Inbound, + [Event::Disconnected, Event::PsmDrop].into_iter().collect::>(), ), ( - Event::PsmDrop, - [Event::Incoming/*, Event::PsmReject synonymous */, Event::PsmConnect] - .into_iter() - .collect::>(), + BareState::Outbound, + [Event::Disconnected, Event::PsmDrop].into_iter().collect::>(), ), ] .into_iter() @@ -101,31 +121,29 @@ fn test_once() { let mut rng = rand::thread_rng(); // Nodes that the peerset knows about. - let mut known_nodes = HashSet::::new(); + let mut known_nodes = HashMap::::new(); // Nodes that we have reserved. Always a subset of `known_nodes`. let mut reserved_nodes = HashSet::::new(); let (mut peerset, peerset_handle) = Peerset::from_config(PeersetConfig { sets: vec![SetConfig { - // bootnodes: (0..Uniform::new_inclusive(0, 4).sample(&mut rng)) - // .map(|_| { - // let id = PeerId::random(); - // known_nodes.insert(id); - // id - // }) - // .collect(), - // reserved_nodes: { - // (0..Uniform::new_inclusive(0, 2).sample(&mut rng)) - // .map(|_| { - // let id = PeerId::random(); - // known_nodes.insert(id); - // reserved_nodes.insert(id); - // id - // }) - // .collect() - // }, - bootnodes: Vec::new(), - reserved_nodes: HashSet::new(), + bootnodes: (0..Uniform::new_inclusive(0, 4).sample(&mut rng)) + .map(|_| { + let id = PeerId::random(); + known_nodes.insert(id, State::Disconnected); + id + }) + .collect(), + reserved_nodes: { + (0..Uniform::new_inclusive(0, 2).sample(&mut rng)) + .map(|_| { + let id = PeerId::random(); + known_nodes.insert(id, State::Disconnected); + reserved_nodes.insert(id); + id + }) + .collect() + }, in_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), out_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), reserved_only: Uniform::new_inclusive(0, 10).sample(&mut rng) == 0, @@ -133,7 +151,7 @@ fn test_once() { }); let new_id = PeerId::random(); - known_nodes.insert(new_id); + known_nodes.insert(new_id, State::Disconnected); peerset_handle.add_known_peer(new_id); futures::executor::block_on(futures::future::poll_fn(move |cx| { @@ -145,16 +163,16 @@ fn test_once() { let mut incoming_nodes = HashMap::::new(); // Next id for incoming connections. let mut next_incoming_id = IncomingIndex(0); - // Last event with the peer for bigram validation. - let mut events = HashMap::::new(); // Perform a certain number of actions while checking that the state is consistent. If we // reach the end of the loop, the run has succeeded. - for _ in 0..250000 { + for _ in 0..2500 { // Peer we are working with. let mut current_peer = None; // Current event for event bigrams validation. let mut current_event = None; + // Last peer state for allowed event validation. + let mut last_state = None; // Each of these weights corresponds to an action that we may perform. let action_weights = [150, 90, 90, 30, 30, 1, 1, 4, 4]; @@ -162,40 +180,104 @@ fn test_once() { // If we generate 0, poll the peerset. 0 => match Stream::poll_next(Pin::new(&mut peerset), cx) { Poll::Ready(Some(Message::Connect { peer_id, .. })) => { - log::info!("PSM: connecting to peer {}", peer_id); - if let Some(id) = incoming_nodes - .iter() - .find(|(_, v)| **v == peer_id) - .map(|(&id, _)| id) - { - // incoming_nodes.remove(&id); - // Do not remove the incoming index for subsequent bigram validation. + let state = known_nodes.get_mut(&peer_id).unwrap(); + if matches!(*state, State::Incoming(_)) { + log::info!( + "Awaiting incoming response, ignoring obsolete Connect from PSM for peer {}", + peer_id, + ); + continue } + + last_state = Some(*state); + *state = State::Outbound; + assert!(connected_nodes.insert(peer_id)); current_peer = Some(peer_id); current_event = Some(Event::PsmConnect); + + log::info!("PSM: connecting to peer {}", peer_id); }, Poll::Ready(Some(Message::Drop { peer_id, .. })) => { - log::info!("PSM: dropping peer {}", peer_id); - connected_nodes.remove(&peer_id); + let state = known_nodes.get_mut(&peer_id).unwrap(); + if matches!(*state, State::Incoming(_)) { + log::info!( + "Awaiting incoming response, ignoring obsolete Drop from PSM for peer {}", + peer_id, + ); + continue + } + + last_state = Some(*state); + *state = State::Disconnected; + + if !connected_nodes.remove(&peer_id) { + log::info!("Ignoring attempt to drop a disconnected peer {}", peer_id); + } current_peer = Some(peer_id); current_event = Some(Event::PsmDrop); + + log::info!("PSM: dropping peer {}", peer_id); }, Poll::Ready(Some(Message::Accept(n))) => { - log::info!("PSM: accepting index {}", n.0); - let peer_id = incoming_nodes.remove(&n).unwrap(); + + let state = known_nodes.get_mut(&peer_id).unwrap(); + match *state { + State::Incoming(incoming_index) => + if n.0 < incoming_index.0 { + log::info!( + "Ignoring obsolete Accept for {:?} while awaiting {:?} for peer {}", + n, incoming_index, peer_id, + ); + continue + } else if n.0 > incoming_index.0 { + panic!( + "Received {:?} while awaiting {:?} for peer {}", + n, incoming_index, peer_id, + ); + }, + _ => {}, + } + + last_state = Some(*state); + *state = State::Inbound; + assert!(connected_nodes.insert(peer_id)); current_peer = Some(peer_id); current_event = Some(Event::PsmAccept); + + log::info!("PSM: accepting index {}", n.0); }, Poll::Ready(Some(Message::Reject(n))) => { log::info!("PSM: rejecting index {}", n.0); let peer_id = incoming_nodes.remove(&n).unwrap(); + + let state = known_nodes.get_mut(&peer_id).unwrap(); + match *state { + State::Incoming(incoming_index) => + if n.0 < incoming_index.0 { + log::info!( + "Ignoring obsolete Reject for {:?} while awaiting {:?} for peer {}", + n, incoming_index, peer_id, + ); + continue + } else if n.0 > incoming_index.0 { + panic!( + "Received {:?} while awaiting {:?} for peer {}", + n, incoming_index, peer_id, + ); + }, + _ => {}, + } + + last_state = Some(*state); + *state = State::Disconnected; + assert!(!connected_nodes.contains(&peer_id)); current_peer = Some(peer_id); @@ -207,14 +289,14 @@ fn test_once() { // If we generate 1, discover a new node. 1 => { - // let new_id = PeerId::random(); - // known_nodes.insert(new_id); - // peerset_handle.add_known_peer(new_id); + let new_id = PeerId::random(); + known_nodes.insert(new_id, State::Disconnected); + peerset_handle.add_known_peer(new_id); }, // If we generate 2, adjust a random reputation. 2 => - if let Some(id) = known_nodes.iter().choose(&mut rng) { + if let Some(id) = known_nodes.keys().choose(&mut rng) { let val = Uniform::new_inclusive(i32::MIN, i32::MAX).sample(&mut rng); peerset_handle.report_peer(*id, ReputationChange::new(val, "")); }, @@ -224,6 +306,11 @@ fn test_once() { if let Some(id) = connected_nodes.iter().choose(&mut rng).cloned() { log::info!("Disconnected from {}", id); connected_nodes.remove(&id); + + let state = known_nodes.get_mut(&id).unwrap(); + last_state = Some(*state); + *state = State::Disconnected; + peerset.dropped(SetId::from(0), id, DropReason::Unknown); current_peer = Some(id); @@ -233,19 +320,25 @@ fn test_once() { // If we generate 4, connect to a random node. 4 => { if let Some(id) = known_nodes - .iter() + .keys() .filter(|n| { incoming_nodes.values().all(|m| m != *n) && !connected_nodes.contains(*n) }) .choose(&mut rng) + .cloned() { log::info!("Incoming connection from {}, index {}", id, next_incoming_id.0); - peerset.incoming(SetId::from(0), *id, next_incoming_id); - incoming_nodes.insert(next_incoming_id, *id); + peerset.incoming(SetId::from(0), id, next_incoming_id); + incoming_nodes.insert(next_incoming_id, id); + + let state = known_nodes.get_mut(&id).unwrap(); + last_state = Some(*state); + *state = State::Incoming(next_incoming_id); + next_incoming_id.0 += 1; - current_peer = Some(*id); + current_peer = Some(id); current_event = Some(Event::Incoming); } }, @@ -263,7 +356,7 @@ fn test_once() { // 7 and 8 are about switching a random node in or out of reserved mode. 7 => { if let Some(id) = - known_nodes.iter().filter(|n| !reserved_nodes.contains(*n)).choose(&mut rng) + known_nodes.keys().filter(|n| !reserved_nodes.contains(*n)).choose(&mut rng) { log::info!("Add reserved: {}", id); peerset_handle.add_reserved_peer(SetId::from(0), *id); @@ -280,16 +373,17 @@ fn test_once() { _ => unreachable!(), } - // Validate event bigrams. + // Validate event bigrams and state transitions. if let Some(peer_id) = current_peer { let event = current_event.unwrap(); - if let Some(last_event) = events.insert(peer_id, event) { - if !allowed_events.get(&last_event).unwrap().contains(&event) { - panic!("Invalid event bigram: {:?} -> {:?}", last_event, event); - } + let last_state = discard_incoming_index(last_state.unwrap()); + if !allowed_events.get(&last_state).unwrap().contains(&event) { + panic!( + "Invalid state transition: {:?} x {:?} for peer {}", + last_state, event, peer_id, + ); } } - } Poll::Ready(()) From 14315a05dc81c177dc42550e02556a8257aeaf1a Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 26 Apr 2023 17:22:39 +0300 Subject: [PATCH 60/98] Update comment re concurrency issues with `ProtocolController`<->`Notifications` --- client/peerset/src/protocol_controller.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 703c8dd4ac62b..0e27ff58013c8 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -474,9 +474,11 @@ impl ProtocolController { /// Note that this mechanism is orthogonal to `Connect`/`Drop`. Accepting an incoming /// connection implicitly means `Connect`, but incoming connections aren't cancelled by /// `dropped`. - // Implementation note: because of concurrency issues, it is possible that we push a `Connect` - // message to the output channel with a `PeerId`, and that `incoming` gets called with the same - // `PeerId` before that message has been read by the user. In this situation we must not answer. + // Implementation note: because of concurrency issues, `ProtocolController` has an imperfect view + // of the peers' states, and may issue commands for a peer after `Notifications` received an + // incoming request for that peer. In this case, `Notifications` ignores all the commands until + // it receives a response for the incoming request to `ProtocolController`, so we must ensure we + // handle this incoming request correctly. fn on_incoming_connection(&mut self, peer_id: PeerId, incoming_index: IncomingIndex) { trace!(target: LOG_TARGET, "Incoming connection from peer {peer_id} ({incoming_index:?}).",); @@ -489,7 +491,7 @@ impl ProtocolController { if let Some(state) = self.reserved_nodes.get_mut(&peer_id) { match state { PeerState::Connected(mut direction) => { - // We are accepting an incoming connection, so switch the direction to inbound. + // We are accepting an incoming connection, so ensure the direction is inbound. // (See the note above.) direction = Direction::Inbound; self.accept_connection(incoming_index); From 27178f81476e4c0f84e279381c744cb2aa359d36 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 27 Apr 2023 13:37:58 +0300 Subject: [PATCH 61/98] Clean things up and fix warnings --- Cargo.lock | 28 ----- client/peerset/Cargo.toml | 1 - client/peerset/src/lib.rs | 33 ++---- client/peerset/src/protocol_controller.rs | 118 ++++++++++------------ client/peerset/tests/fuzz.rs | 54 +++++----- 5 files changed, 90 insertions(+), 144 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4f3e19a186dfc..caa640038dd04 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2214,19 +2214,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "flume" -version = "0.10.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1657b4441c3403d9f7b3409e47575237dac27b1b5726df654a6ecbf92f0f7577" -dependencies = [ - "futures-core", - "futures-sink", - "nanorand", - "pin-project", - "spin 0.9.5", -] - [[package]] name = "fnv" version = "1.0.7" @@ -2857,10 +2844,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", - "js-sys", "libc", "wasi 0.11.0+wasi-snapshot-preview1", - "wasm-bindgen", ] [[package]] @@ -4865,15 +4850,6 @@ dependencies = [ "rand 0.8.5", ] -[[package]] -name = "nanorand" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" -dependencies = [ - "getrandom 0.2.8", -] - [[package]] name = "netlink-packet-core" version = "0.4.2" @@ -9306,7 +9282,6 @@ dependencies = [ name = "sc-peerset" version = "4.0.0-dev" dependencies = [ - "flume", "futures", "libp2p", "log", @@ -11038,9 +11013,6 @@ name = "spin" version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7dccf47db1b41fa1573ed27ccf5e08e3ca771cb994f776668c5ebda893b248fc" -dependencies = [ - "lock_api", -] [[package]] name = "spki" diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index b789c8cd035e9..00a05601ee879 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -15,7 +15,6 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] futures = "0.3.21" -flume = "0.10.14" libp2p = "0.50.0" log = "0.4.17" partial_sort = "0.2.0" diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 8d2f6ed1703f5..9d12cf53eb247 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -38,14 +38,13 @@ mod protocol_controller; use peer_store::{PeerReputationProvider, PeerStore, PeerStoreHandle}; use protocol_controller::{ProtocolController, ProtocolHandle}; -use flume::{bounded, Receiver, TryRecvError}; use futures::{ channel::oneshot, future::{join_all, BoxFuture, JoinAll}, prelude::*, stream::Stream, }; -use log::{debug, error, trace}; +use log::debug; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use serde_json::json; use std::{ @@ -258,7 +257,7 @@ pub struct Peerset { protocol_controller_futures: JoinAll>, /// Commands sent from protocol controllers to `Notifications`. The size of this vector never /// changes. - from_controllers: Receiver, + from_controllers: TracingUnboundedReceiver, /// Receiver for messages from the `PeersetHandle` and from `to_self`. from_handle: TracingUnboundedReceiver, /// Sending side of `from_handle`. @@ -271,10 +270,8 @@ impl Peerset { let default_set_config = &config.sets[0]; let peer_store = PeerStore::new(default_set_config.bootnodes.clone()); - //let (to_notifications, from_controllers) = - // tracing_unbounded("mpsc_protocol_controllers_to_notifications", 10_000); - - let (to_notifications, from_controllers) = bounded(0); + let (to_notifications, from_controllers) = + tracing_unbounded("mpsc_protocol_controllers_to_notifications", 10_000); let controllers = config .sets @@ -363,28 +360,16 @@ impl Stream for Peerset { type Item = Message; fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { - // if let Poll::Ready(msg) = self.from_controllers.poll_next_unpin(cx) { - // if let Some(msg) = msg { - // return Poll::Ready(Some(msg)) - // } else { - // debug!( - // target: LOG_TARGET, - // "All `ProtocolController`s have terminated, terminating `Peerset`." - // ); - // return Poll::Ready(None) - // } - // } - - match self.from_controllers.try_recv() { - Ok(msg) => return Poll::Ready(Some(msg)), - Err(TryRecvError::Disconnected) => { + if let Poll::Ready(msg) = self.from_controllers.poll_next_unpin(cx) { + if let Some(msg) = msg { + return Poll::Ready(Some(msg)) + } else { debug!( target: LOG_TARGET, "All `ProtocolController`s have terminated, terminating `Peerset`." ); return Poll::Ready(None) - }, - Err(TryRecvError::Empty) => (), + } } while let Poll::Ready(action) = self.from_handle.poll_next_unpin(cx) { diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 0e27ff58013c8..56b5f35b41a8e 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -21,25 +21,19 @@ //! respecting the inbound and outbound peer slot counts. Communicates with `Peerset` to get and //! update peer reputation values and sends commands to `Notifications`. -// TODO: remove this line. -#![allow(unused)] - -use flume::Sender; use futures::{channel::oneshot, future::Either, FutureExt, StreamExt}; use libp2p::PeerId; use log::{error, info, trace, warn}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_arithmetic::traits::SaturatedConversion; use std::{ - collections::{hash_map::Entry, HashMap, HashSet, VecDeque}, - sync::mpsc::SyncSender, + collections::{HashMap, HashSet}, time::{Duration, Instant}, }; use wasm_timer::Delay; use crate::{ - peer_store::PeerReputationProvider, DropReason, IncomingIndex, Message, PeersetHandle, - SetConfig, SetId, LOG_TARGET, + peer_store::PeerReputationProvider, IncomingIndex, Message, SetConfig, SetId, LOG_TARGET, }; #[derive(Debug)] @@ -178,10 +172,8 @@ pub struct ProtocolController { reserved_only: bool, /// Next time to allocate slots. This is done once per second. next_periodic_alloc_slots: Instant, - /// Queue of messages for `Notifications` - notif_queue: VecDeque, /// Outgoing channel for messages to `Notifications`. - to_notifications: Sender, + to_notifications: TracingUnboundedSender, /// `PeerStore` handle for checking peer reputation values and getting connection candidates /// with highest reputation. peer_store: Box, @@ -192,7 +184,7 @@ impl ProtocolController { pub fn new( set_id: SetId, config: SetConfig, - to_notifications: Sender, + to_notifications: TracingUnboundedSender, peer_store: Box, ) -> (ProtocolHandle, ProtocolController) { let (actions_tx, actions_rx) = tracing_unbounded("mpsc_api_protocol", 10_000); @@ -212,7 +204,6 @@ impl ProtocolController { reserved_nodes, reserved_only: config.reserved_only, next_periodic_alloc_slots: Instant::now(), - notif_queue: VecDeque::new(), to_notifications, peer_store, }; @@ -229,12 +220,8 @@ impl ProtocolController { /// /// Intended for tests only. Use `run` for driving [`ProtocolController`]. pub async fn next_action(&mut self) -> bool { + // Perform tasks prioritizing connection events processing. let either = loop { - for msg in self.notif_queue.drain(..) { - let _ = self.to_notifications.send_async(msg).await; - // TODO: handle errors. - } - if let Some(event) = self.events_rx.next().now_or_never() { match event { None => return false, @@ -290,23 +277,27 @@ impl ProtocolController { } /// Send "accept" message to `Notifications`. - fn accept_connection(&mut self, incoming_index: IncomingIndex) { - let _ = self.notif_queue.push_back(Message::Accept(incoming_index)); + fn accept_connection(&self, incoming_index: IncomingIndex) { + let _ = self.to_notifications.unbounded_send(Message::Accept(incoming_index)); } /// Send "reject" message to `Notifications`. - fn reject_connection(&mut self, incoming_index: IncomingIndex) { - let _ = self.notif_queue.push_back(Message::Reject(incoming_index)); + fn reject_connection(&self, incoming_index: IncomingIndex) { + let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); } /// Send "connect" message to `Notifications`. - fn start_connection(&mut self, peer_id: PeerId) { - let _ = self.notif_queue.push_back(Message::Connect { set_id: self.set_id, peer_id }); + fn start_connection(&self, peer_id: PeerId) { + let _ = self + .to_notifications + .unbounded_send(Message::Connect { set_id: self.set_id, peer_id }); } /// Send "drop" message to `Notifications`. - fn drop_connection(&mut self, peer_id: PeerId) { - let _ = self.notif_queue.push_back(Message::Drop { set_id: self.set_id, peer_id }); + fn drop_connection(&self, peer_id: PeerId) { + let _ = self + .to_notifications + .unbounded_send(Message::Drop { set_id: self.set_id, peer_id }); } /// Report peer disconnect event to `PeerStore` for it to update peer's reputation accordingly. @@ -359,7 +350,7 @@ impl ProtocolController { /// Remove the peer from the set of reserved peers. The peer is moved to the set of regular /// nodes. fn on_remove_reserved_peer(&mut self, peer_id: PeerId) { - let mut state = match self.reserved_nodes.remove(&peer_id) { + let state = match self.reserved_nodes.remove(&peer_id) { Some(state) => state, None => { warn!(target: LOG_TARGET, "Trying to remove unknown reserved node: {peer_id}."); @@ -427,8 +418,7 @@ impl ProtocolController { } // Disconnect all non-reserved peers. - let non_reserved: Vec<_> = self.nodes.keys().cloned().collect(); - non_reserved.iter().for_each(|peer_id| self.drop_connection(*peer_id)); + self.nodes.keys().for_each(|peer_id| self.drop_connection(*peer_id)); self.nodes.clear(); self.num_in = 0; self.num_out = 0; @@ -436,7 +426,7 @@ impl ProtocolController { /// Get the list of reserved peers. fn on_get_reserved_peers(&self, pending_response: oneshot::Sender>) { - pending_response.send(self.reserved_nodes.keys().cloned().collect()); + let _ = pending_response.send(self.reserved_nodes.keys().cloned().collect()); } /// Disconnect the peer. @@ -474,11 +464,11 @@ impl ProtocolController { /// Note that this mechanism is orthogonal to `Connect`/`Drop`. Accepting an incoming /// connection implicitly means `Connect`, but incoming connections aren't cancelled by /// `dropped`. - // Implementation note: because of concurrency issues, `ProtocolController` has an imperfect view - // of the peers' states, and may issue commands for a peer after `Notifications` received an - // incoming request for that peer. In this case, `Notifications` ignores all the commands until - // it receives a response for the incoming request to `ProtocolController`, so we must ensure we - // handle this incoming request correctly. + // Implementation note: because of concurrency issues, `ProtocolController` has an imperfect + // view of the peers' states, and may issue commands for a peer after `Notifications` received + // an incoming request for that peer. In this case, `Notifications` ignores all the commands + // until it receives a response for the incoming request to `ProtocolController`, so we must + // ensure we handle this incoming request correctly. fn on_incoming_connection(&mut self, peer_id: PeerId, incoming_index: IncomingIndex) { trace!(target: LOG_TARGET, "Incoming connection from peer {peer_id} ({incoming_index:?}).",); @@ -671,14 +661,13 @@ impl ProtocolController { } } -#[cfg(disabled)] +#[cfg(test)] mod tests { use super::{Direction, PeerState, ProtocolController}; use crate::{ - peer_store::PeerReputationProvider, DropReason, IncomingIndex, Message, ReputationChange, - SetConfig, SetId, + peer_store::PeerReputationProvider, IncomingIndex, Message, ReputationChange, SetConfig, + SetId, }; - use futures::FutureExt; use libp2p::PeerId; use sc_utils::mpsc::{tracing_unbounded, TryRecvError}; use std::collections::HashSet; @@ -715,7 +704,7 @@ mod tests { peer_store.expect_is_banned().times(4).return_const(false); peer_store.expect_report_disconnect().times(2).return_const(()); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Add second reserved node at runtime (this currently calls `alloc_slots` internally). @@ -776,7 +765,7 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(6).return_const(true); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Add second reserved node at runtime (this currently calls `alloc_slots` internally). @@ -828,7 +817,7 @@ mod tests { peer_store.expect_is_banned().times(4).return_const(false); peer_store.expect_report_disconnect().times(2).return_const(()); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Add second reserved node at runtime (this calls `alloc_slots` internally). @@ -871,7 +860,6 @@ mod tests { fn nodes_supplied_by_peer_store_are_connected() { let peer1 = PeerId::random(); let peer2 = PeerId::random(); - let peer3 = PeerId::random(); let candidates = vec![peer1, peer2]; let config = SetConfig { @@ -887,7 +875,7 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_outgoing_candidates().once().return_const(candidates); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. @@ -938,7 +926,7 @@ mod tests { peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. @@ -980,7 +968,7 @@ mod tests { peer_store.expect_outgoing_candidates().once().return_const(candidates2); peer_store.expect_report_disconnect().times(2).return_const(()); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. @@ -1045,9 +1033,9 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStoreHandle::new(); + let peer_store = MockPeerStoreHandle::new(); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. @@ -1071,9 +1059,9 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStoreHandle::new(); + let peer_store = MockPeerStoreHandle::new(); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); let peer = PeerId::random(); @@ -1111,7 +1099,7 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_outgoing_candidates().once().return_const(candidates); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. @@ -1158,7 +1146,7 @@ mod tests { peer_store.expect_is_banned().times(3).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); assert_eq!(controller.num_out, 0); assert_eq!(controller.num_in, 0); @@ -1214,9 +1202,9 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let mut peer_store = MockPeerStoreHandle::new(); + let peer_store = MockPeerStoreHandle::new(); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); assert_eq!(controller.reserved_nodes.len(), 2); assert_eq!(controller.nodes.len(), 0); @@ -1249,7 +1237,7 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().times(2).return_const(false); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Initiate connections. @@ -1293,7 +1281,7 @@ mod tests { peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as inbound, `peer2` as outbound. @@ -1339,7 +1327,7 @@ mod tests { peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as outbound & `peer2` as inbound. @@ -1381,7 +1369,7 @@ mod tests { peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as outbound & `peer2` as inbound. @@ -1434,7 +1422,7 @@ mod tests { peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `reserved1` as inbound & `reserved2` as outbound. @@ -1491,7 +1479,7 @@ mod tests { peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); peer_store.expect_report_disconnect().times(2).return_const(()); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as outbound & `peer2` as inbound. @@ -1543,7 +1531,7 @@ mod tests { peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as outbound & `peer2` as inbound. @@ -1589,7 +1577,7 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(false); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Connect `peer1` as inbound. @@ -1619,7 +1607,7 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(true); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); // Incoming request. @@ -1644,7 +1632,7 @@ mod tests { let mut peer_store = MockPeerStoreHandle::new(); peer_store.expect_is_banned().once().return_const(true); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); assert!(controller.reserved_nodes.contains_key(&reserved1)); @@ -1671,7 +1659,7 @@ mod tests { peer_store.expect_is_banned().once().return_const(true); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); - let (handle, mut controller) = + let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); assert!(matches!(controller.reserved_nodes.get(&reserved1), Some(PeerState::NotConnected))); diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index dae28f23327a4..44dd5ba677b00 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -127,23 +127,25 @@ fn test_once() { let (mut peerset, peerset_handle) = Peerset::from_config(PeersetConfig { sets: vec![SetConfig { - bootnodes: (0..Uniform::new_inclusive(0, 4).sample(&mut rng)) - .map(|_| { - let id = PeerId::random(); - known_nodes.insert(id, State::Disconnected); - id - }) - .collect(), - reserved_nodes: { - (0..Uniform::new_inclusive(0, 2).sample(&mut rng)) - .map(|_| { - let id = PeerId::random(); - known_nodes.insert(id, State::Disconnected); - reserved_nodes.insert(id); - id - }) - .collect() - }, + // bootnodes: (0..Uniform::new_inclusive(0, 4).sample(&mut rng)) + // .map(|_| { + // let id = PeerId::random(); + // known_nodes.insert(id, State::Disconnected); + // id + // }) + // .collect(), + // reserved_nodes: { + // (0..Uniform::new_inclusive(0, 2).sample(&mut rng)) + // .map(|_| { + // let id = PeerId::random(); + // known_nodes.insert(id, State::Disconnected); + // reserved_nodes.insert(id); + // id + // }) + // .collect() + // }, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), in_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), out_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), reserved_only: Uniform::new_inclusive(0, 10).sample(&mut rng) == 0, @@ -180,6 +182,8 @@ fn test_once() { // If we generate 0, poll the peerset. 0 => match Stream::poll_next(Pin::new(&mut peerset), cx) { Poll::Ready(Some(Message::Connect { peer_id, .. })) => { + log::info!("PSM: connecting to peer {}", peer_id); + let state = known_nodes.get_mut(&peer_id).unwrap(); if matches!(*state, State::Incoming(_)) { log::info!( @@ -196,10 +200,10 @@ fn test_once() { current_peer = Some(peer_id); current_event = Some(Event::PsmConnect); - - log::info!("PSM: connecting to peer {}", peer_id); }, Poll::Ready(Some(Message::Drop { peer_id, .. })) => { + log::info!("PSM: dropping peer {}", peer_id); + let state = known_nodes.get_mut(&peer_id).unwrap(); if matches!(*state, State::Incoming(_)) { log::info!( @@ -218,10 +222,10 @@ fn test_once() { current_peer = Some(peer_id); current_event = Some(Event::PsmDrop); - - log::info!("PSM: dropping peer {}", peer_id); }, Poll::Ready(Some(Message::Accept(n))) => { + log::info!("PSM: accepting index {}", n.0); + let peer_id = incoming_nodes.remove(&n).unwrap(); let state = known_nodes.get_mut(&peer_id).unwrap(); @@ -249,8 +253,6 @@ fn test_once() { current_peer = Some(peer_id); current_event = Some(Event::PsmAccept); - - log::info!("PSM: accepting index {}", n.0); }, Poll::Ready(Some(Message::Reject(n))) => { log::info!("PSM: rejecting index {}", n.0); @@ -289,9 +291,9 @@ fn test_once() { // If we generate 1, discover a new node. 1 => { - let new_id = PeerId::random(); - known_nodes.insert(new_id, State::Disconnected); - peerset_handle.add_known_peer(new_id); + // let new_id = PeerId::random(); + // known_nodes.insert(new_id, State::Disconnected); + // peerset_handle.add_known_peer(new_id); }, // If we generate 2, adjust a random reputation. From 1ef145c44abc767ff44c5c7dc0e7e8d260a33cdd Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 27 Apr 2023 13:46:18 +0300 Subject: [PATCH 62/98] Fix error with assigning to local variable instead of reference --- client/peerset/src/protocol_controller.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 56b5f35b41a8e..7d00b6487f2d0 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -480,10 +480,10 @@ impl ProtocolController { // Check if the node is reserved first. if let Some(state) = self.reserved_nodes.get_mut(&peer_id) { match state { - PeerState::Connected(mut direction) => { + PeerState::Connected(ref mut direction) => { // We are accepting an incoming connection, so ensure the direction is inbound. // (See the note above.) - direction = Direction::Inbound; + *direction = Direction::Inbound; self.accept_connection(incoming_index); }, PeerState::NotConnected => { From 535410be83774d6fccdbac60d0697a94ee566448 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 27 Apr 2023 14:46:39 +0300 Subject: [PATCH 63/98] Update tests with new incoming request handling logic --- client/peerset/src/protocol_controller.rs | 212 +++++++++++++++++++--- 1 file changed, 191 insertions(+), 21 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 7d00b6487f2d0..ea7493bbdb5b2 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -1513,51 +1513,221 @@ mod tests { } #[test] - fn we_dont_answer_to_incoming_requests_for_already_connected_peers() { - let peer1 = PeerId::random(); - let peer2 = PeerId::random(); - let outgoing_candidates = vec![peer1]; + fn incoming_request_for_connected_reserved_node_switches_it_to_inbound() { + let reserved1 = PeerId::random(); + let reserved2 = PeerId::random(); let config = SetConfig { in_peers: 10, out_peers: 10, bootnodes: Vec::new(), - reserved_nodes: HashSet::new(), + reserved_nodes: [reserved1, reserved2].iter().cloned().collect(), reserved_only: false, }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); - peer_store.expect_is_banned().once().return_const(false); - peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + peer_store.expect_is_banned().times(2).return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); - // Connect `peer1` as outbound & `peer2` as inbound. + // Connect `reserved1` as inbound & `reserved2` as outbound. + controller.on_incoming_connection(reserved1, IncomingIndex(1)); controller.alloc_slots(); - controller.on_incoming_connection(peer2, IncomingIndex(1)); let mut messages = Vec::new(); while let Some(message) = rx.try_recv().ok() { messages.push(message); } assert_eq!(messages.len(), 2); - assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); - assert_eq!(controller.nodes.len(), 2); - assert!(matches!(controller.nodes.get(&peer1), Some(Direction::Outbound))); - assert!(matches!(controller.nodes.get(&peer2), Some(Direction::Inbound))); - assert_eq!(controller.num_in, 1); - assert_eq!(controller.num_out, 1); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + assert!(matches!( + controller.reserved_nodes.get(&reserved1), + Some(PeerState::Connected(Direction::Inbound)) + )); + assert!(matches!( + controller.reserved_nodes.get(&reserved2), + Some(PeerState::Connected(Direction::Outbound)) + )); - // Incoming requests for `peer1` & `peer2`. - controller.on_incoming_connection(peer1, IncomingIndex(1)); - controller.on_incoming_connection(peer2, IncomingIndex(2)); + // Incoming request for `reserved1`. + controller.on_incoming_connection(reserved1, IncomingIndex(2)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(2))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!( + controller.reserved_nodes.get(&reserved1), + Some(PeerState::Connected(Direction::Inbound)) + )); - // No answer. - assert_eq!(controller.num_in, 1); - assert_eq!(controller.num_out, 1); + // Incoming request for `reserved2`. + controller.on_incoming_connection(reserved2, IncomingIndex(3)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(3))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!( + controller.reserved_nodes.get(&reserved2), + Some(PeerState::Connected(Direction::Inbound)) + )); + } + + #[test] + fn incoming_request_for_connected_regular_node_switches_it_to_inbound() { + let regular1 = PeerId::random(); + let regular2 = PeerId::random(); + let outgoing_candidates = vec![regular1]; + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_is_banned().times(3).return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + + let (_handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + + // Connect `regular1` as outbound. + controller.alloc_slots(); + assert_eq!( + rx.try_recv().ok().unwrap(), + Message::Connect { set_id: SetId(0), peer_id: regular1 } + ); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!(controller.nodes.get(®ular1).unwrap(), Direction::Outbound,)); + + // Connect `regular2` as inbound. + controller.on_incoming_connection(regular2, IncomingIndex(0)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(0))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); + + // Incoming request for `regular1`. + controller.on_incoming_connection(regular1, IncomingIndex(1)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!(controller.nodes.get(®ular1).unwrap(), Direction::Inbound,)); + + // Incoming request for `regular2`. + controller.on_incoming_connection(regular2, IncomingIndex(2)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(2))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); + } + + #[test] + fn incoming_request_for_connected_node_is_rejected_if_its_banned() { + let regular1 = PeerId::random(); + let regular2 = PeerId::random(); + let outgoing_candidates = vec![regular1]; + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_is_banned().once().return_const(false); + peer_store.expect_is_banned().times(2).return_const(true); + peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + + let (_handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + + // Connect `regular1` as outbound. + controller.alloc_slots(); + assert_eq!( + rx.try_recv().ok().unwrap(), + Message::Connect { set_id: SetId(0), peer_id: regular1 } + ); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!(controller.nodes.get(®ular1).unwrap(), Direction::Outbound,)); + + // Connect `regular2` as inbound. + controller.on_incoming_connection(regular2, IncomingIndex(0)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(0))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); + + // Incoming request for `regular1`. + controller.on_incoming_connection(regular1, IncomingIndex(1)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(!controller.nodes.contains_key(®ular1)); + + // Incoming request for `regular2`. + controller.on_incoming_connection(regular2, IncomingIndex(2)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(2))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(!controller.nodes.contains_key(®ular2)); + } + + #[test] + fn incoming_request_for_connected_node_is_rejected_if_no_slots_available() { + let regular1 = PeerId::random(); + let regular2 = PeerId::random(); + let outgoing_candidates = vec![regular1]; + + let config = SetConfig { + in_peers: 1, + out_peers: 1, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_is_banned().once().return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + + let (_handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); + assert_eq!(controller.num_out, 0); + assert_eq!(controller.num_in, 0); + + // Connect `regular1` as outbound. + controller.alloc_slots(); + assert_eq!( + rx.try_recv().ok().unwrap(), + Message::Connect { set_id: SetId(0), peer_id: regular1 } + ); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!(controller.nodes.get(®ular1).unwrap(), Direction::Outbound,)); + + // Connect `regular2` as inbound. + controller.on_incoming_connection(regular2, IncomingIndex(0)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(0))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); + + controller.max_in = 0; + + // Incoming request for `regular1`. + controller.on_incoming_connection(regular1, IncomingIndex(1)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(!controller.nodes.contains_key(®ular1)); + + // Incoming request for `regular2`. + controller.on_incoming_connection(regular2, IncomingIndex(2)); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(!controller.nodes.contains_key(®ular2)); } #[test] From 1065fc68c7c90b4c37461eead88524477586c388 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 28 Apr 2023 17:37:34 +0300 Subject: [PATCH 64/98] Make `Notifications` skip over `Peerset` commands between `incoming` and `accept`/`reject` --- .../src/protocol/notifications/behaviour.rs | 176 ++++++++++++------ client/peerset/src/lib.rs | 2 +- 2 files changed, 120 insertions(+), 58 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index e60090bc5709f..c710c7d0d729c 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -35,7 +35,7 @@ use libp2p::{ PollParameters, }, }; -use log::{error, trace, warn}; +use log::{error, info, trace, warn}; use parking_lot::RwLock; use rand::distributions::{Distribution as _, Uniform}; use sc_peerset::DropReason; @@ -231,6 +231,9 @@ enum PeerState { /// If `Some`, any dial attempts to this peer are delayed until the given `Instant`. backoff_until: Option, + /// Incoming index tracking this connection. + incoming_index: sc_peerset::IncomingIndex, + /// List of connections with this peer, and their state. connections: SmallVec<[(ConnectionId, ConnectionState); crate::MAX_CONNECTIONS_PER_PEER]>, }, @@ -493,7 +496,7 @@ impl Notifications { // Incoming => Disabled. // Ongoing opening requests from the remote are rejected. - PeerState::Incoming { mut connections, backoff_until } => { + PeerState::Incoming { mut connections, backoff_until, .. } => { let inc = if let Some(inc) = self .incoming .iter_mut() @@ -698,41 +701,14 @@ impl Notifications { } }, - // Incoming => Enabled - PeerState::Incoming { mut connections, .. } => { - trace!(target: "sub-libp2p", "PSM => Connect({}, {:?}): Enabling connections.", - occ_entry.key().0, set_id); - if let Some(inc) = self - .incoming - .iter_mut() - .find(|i| i.peer_id == occ_entry.key().0 && i.set_id == set_id && i.alive) - { - inc.alive = false; - } else { - error!( - target: "sub-libp2p", - "State mismatch in libp2p: no entry in incoming for incoming peer", - ) - } - - debug_assert!(connections - .iter() - .any(|(_, s)| matches!(s, ConnectionState::OpenDesiredByRemote))); - for (connec_id, connec_state) in connections - .iter_mut() - .filter(|(_, s)| matches!(s, ConnectionState::OpenDesiredByRemote)) - { - trace!(target: "sub-libp2p", "Handler({:?}, {:?}) <= Open({:?})", - occ_entry.key(), *connec_id, set_id); - self.events.push_back(NetworkBehaviourAction::NotifyHandler { - peer_id: occ_entry.key().0, - handler: NotifyHandler::One(*connec_id), - event: NotifsHandlerIn::Open { protocol_index: set_id.into() }, - }); - *connec_state = ConnectionState::Opening; - } - - *occ_entry.into_mut() = PeerState::Enabled { connections }; + // Incoming => Incoming + st @ PeerState::Incoming { .. } => { + info!( + target: "sub-libp2p", + "PSM => Connect({}, {:?}): Ignoring obsolete connect, we are awaiting for accept/reject.", + occ_entry.key().0, set_id + ); + *occ_entry.into_mut() = st; }, // Other states are kept as-is. @@ -858,10 +834,12 @@ impl Notifications { // Invalid state transitions. st @ PeerState::Incoming { .. } => { - error!(target: "sub-libp2p", "PSM => Drop({}, {:?}): Not enabled (Incoming).", - entry.key().0, set_id); + info!( + target: "sub-libp2p", + "PSM => Drop({}, {:?}): Ignoring obsolete disconnect, we are awaiting for accept/reject.", + entry.key().0, set_id, + ); *entry.into_mut() = st; - debug_assert!(false); }, PeerState::Poisoned => { error!(target: "sub-libp2p", "State of {:?} is poisoned", entry.key()); @@ -906,7 +884,24 @@ impl Notifications { match mem::replace(state, PeerState::Poisoned) { // Incoming => Enabled - PeerState::Incoming { mut connections, .. } => { + PeerState::Incoming { mut connections, incoming_index, .. } => { + if index < incoming_index { + warn!( + target: "sub-libp2p", + "PSM => Accept({:?}, {}, {:?}): Ignoring obsolete incoming index, we are already awaiting {:?}.", + index, incoming.peer_id, incoming.set_id, incoming_index + ); + return + } else if index > incoming_index { + error!( + target: "sub-libp2p", + "PSM => Accept({:?}, {}, {:?}): Ignoring incoming index from the future, we are awaiting {:?}.", + index, incoming.peer_id, incoming.set_id, incoming_index + ); + debug_assert!(false); + return + } + trace!(target: "sub-libp2p", "PSM => Accept({:?}, {}, {:?}): Enabling connections.", index, incoming.peer_id, incoming.set_id); @@ -966,7 +961,24 @@ impl Notifications { match mem::replace(state, PeerState::Poisoned) { // Incoming => Disabled - PeerState::Incoming { mut connections, backoff_until } => { + PeerState::Incoming { mut connections, backoff_until, incoming_index } => { + if index < incoming_index { + warn!( + target: "sub-libp2p", + "PSM => Reject({:?}, {}, {:?}): Ignoring obsolete incoming index, we are already awaiting {:?}.", + index, incoming.peer_id, incoming.set_id, incoming_index + ); + return + } else if index > incoming_index { + error!( + target: "sub-libp2p", + "PSM => Reject({:?}, {}, {:?}): Ignoring incoming index from the future, we are awaiting {:?}.", + index, incoming.peer_id, incoming.set_id, incoming_index + ); + debug_assert!(false); + return + } + trace!(target: "sub-libp2p", "PSM => Reject({:?}, {}, {:?}): Rejecting connections.", index, incoming.peer_id, incoming.set_id); @@ -1164,7 +1176,7 @@ impl NetworkBehaviour for Notifications { }, // Incoming => Incoming | Disabled | Backoff | Ø - PeerState::Incoming { mut connections, backoff_until } => { + PeerState::Incoming { mut connections, backoff_until, incoming_index } => { trace!( target: "sub-libp2p", "Libp2p => Disconnected({}, {:?}, {:?}): OpenDesiredByRemote.", @@ -1238,8 +1250,11 @@ impl NetworkBehaviour for Notifications { *entry.get_mut() = PeerState::Disabled { connections, backoff_until }; } else { - *entry.get_mut() = - PeerState::Incoming { connections, backoff_until }; + *entry.get_mut() = PeerState::Incoming { + connections, + backoff_until, + incoming_index, + }; } }, @@ -1463,7 +1478,7 @@ impl NetworkBehaviour for Notifications { match mem::replace(entry.get_mut(), PeerState::Poisoned) { // Incoming => Incoming - PeerState::Incoming { mut connections, backoff_until } => { + PeerState::Incoming { mut connections, backoff_until, incoming_index } => { debug_assert!(connections .iter() .any(|(_, s)| matches!(s, ConnectionState::OpenDesiredByRemote))); @@ -1491,7 +1506,8 @@ impl NetworkBehaviour for Notifications { debug_assert!(false); } - *entry.into_mut() = PeerState::Incoming { connections, backoff_until }; + *entry.into_mut() = + PeerState::Incoming { connections, backoff_until, incoming_index }; }, PeerState::Enabled { mut connections } => { @@ -1556,8 +1572,11 @@ impl NetworkBehaviour for Notifications { incoming_id, }); - *entry.into_mut() = - PeerState::Incoming { connections, backoff_until }; + *entry.into_mut() = PeerState::Incoming { + connections, + backoff_until, + incoming_index: incoming_id, + }; } else { // Connections in `OpeningThenClosing` and `Closing` state can be // in a Closed phase, and as such can emit `OpenDesiredByRemote` @@ -2066,6 +2085,7 @@ mod tests { core::ConnectedPoint, swarm::{behaviour::FromSwarm, AddressRecord}, }; + use sc_peerset::IncomingIndex; use std::{collections::HashSet, iter}; impl PartialEq for ConnectionState { @@ -2258,7 +2278,7 @@ mod tests { NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, ); - if let Some(&PeerState::Incoming { ref connections, backoff_until: None }) = + if let Some(&PeerState::Incoming { ref connections, backoff_until: None, .. }) = notif.peers.get(&(peer, 0.into())) { assert_eq!(connections.len(), 1); @@ -2403,8 +2423,10 @@ mod tests { NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, ); - // attempt to connect to the peer and verify that the peer state is `Enabled` - notif.peerset_report_connect(peer, set_id); + // attempt to connect to the peer and verify that the peer state is `Enabled`; + // we rely on implementation detail that incoming indices are counted from 0 + // to not mock the `Peerset` + notif.peerset_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); } @@ -2481,7 +2503,9 @@ mod tests { conn, NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, ); - notif.peerset_report_connect(peer, set_id); + // we rely on the implementation detail that incoming indices are counted from 0 + // to not mock the `Peerset` + notif.peerset_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // disconnect peer and verify that the state is `Disabled` @@ -2838,7 +2862,9 @@ mod tests { ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); - notif.peerset_report_connect(peer, set_id); + // We rely on the implementation detail that incoming indices are counted + // from 0 to not mock the `Peerset`. + notif.peerset_report_accept(sc_peerset::IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); // open new substream @@ -3681,7 +3707,9 @@ mod tests { ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); - notif.peerset_report_connect(peer, set_id); + // we rely on the implementation detail that incoming indices are counted from 0 + // to not mock the `Peerset` + notif.peerset_report_accept(IncomingIndex(0)); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Enabled { .. }))); let event = conn_yielder.open_substream(peer, 0, connected, vec![1, 2, 3, 4]); @@ -3922,9 +3950,8 @@ mod tests { } #[test] - #[should_panic] #[cfg(debug_assertions)] - fn peerset_report_disconnect_with_incoming_peer() { + fn peerset_report_connect_with_incoming_peer() { let (mut notif, _peerset) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); @@ -3951,10 +3978,45 @@ mod tests { conn, NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, ); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); + notif.peerset_report_connect(peer, set_id); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); + } + + #[test] + #[cfg(debug_assertions)] + fn peerset_report_disconnect_with_incoming_peer() { + let (mut notif, _peerset) = development_notifs(); + let peer = PeerId::random(); + let set_id = sc_peerset::SetId::from(0); + let conn = ConnectionId::new(0usize); + let connected = ConnectedPoint::Listener { + local_addr: Multiaddr::empty(), + send_back_addr: Multiaddr::empty(), + }; + + notif.on_swarm_event(FromSwarm::ConnectionEstablished( + libp2p::swarm::behaviour::ConnectionEstablished { + peer_id: peer, + connection_id: conn, + endpoint: &connected, + failed_addresses: &[], + other_established: 0usize, + }, + )); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Disabled { .. }))); + + // remote opens a substream, verify that peer state is updated to `Incoming` + notif.on_connection_handler_event( + peer, + conn, + NotifsHandlerOut::OpenDesiredByRemote { protocol_index: 0 }, + ); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); notif.peerset_report_disconnect(peer, set_id); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Incoming { .. }))); } #[test] diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 9d12cf53eb247..3cf806f80ce63 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -201,7 +201,7 @@ pub enum Message { } /// Opaque identifier for an incoming connection. Allocated by the network. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct IncomingIndex(pub u64); impl From for IncomingIndex { From 9730a412089ba65e54fa98b405c021309539297e Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Sat, 29 Apr 2023 20:03:31 +0300 Subject: [PATCH 65/98] Fix `reconnect_after_disconnect` test --- client/network/src/protocol/notifications/tests.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/client/network/src/protocol/notifications/tests.rs b/client/network/src/protocol/notifications/tests.rs index 9ca6974e4cdde..8137cd68c648c 100644 --- a/client/network/src/protocol/notifications/tests.rs +++ b/client/network/src/protocol/notifications/tests.rs @@ -264,8 +264,20 @@ fn reconnect_after_disconnect() { _ => {}, } + // Due to the bug in `Notifications`, the disconnected node does not always detect that + // it was disconnected. The closed inbound substream is tolerated by design, and the + // closed outbound substream is not detected until something is sent into it. + // See [PR #13396](https://github.com/paritytech/substrate/pull/13396). + // This happens if the disconnecting node reconnects to it fast enough. + // In this case the disconnected node does not transit via `ServiceState::NotConnected` + // and stays in `ServiceState::FirstConnec`. + // TODO: update this once the fix is finally merged. if service1_state == ServiceState::ConnectedAgain && - service2_state == ServiceState::ConnectedAgain + service2_state == ServiceState::ConnectedAgain || + service1_state == ServiceState::ConnectedAgain && + service2_state == ServiceState::FirstConnec || + service1_state == ServiceState::FirstConnec && + service2_state == ServiceState::ConnectedAgain { break } From 42f14b93deea77b9792fe7cb26470c70b5a87cae Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Sun, 30 Apr 2023 08:56:06 +0300 Subject: [PATCH 66/98] Fix rustdoc --- client/peerset/src/lib.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 3cf806f80ce63..6c35fc2d55bc0 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -421,8 +421,6 @@ pub enum DropReason { Unknown, /// Substream or connection has been explicitly refused by the target. In other words, the /// peer doesn't actually belong to this set. - /// - /// This has the side effect of calling [`PeersetHandle::remove_from_peers_set`]. Refused, } From 752d70bba208be93fa13121bb4260ece98a2c324 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Sun, 30 Apr 2023 09:05:37 +0300 Subject: [PATCH 67/98] Remove unused `ProtocolHandle::disconnect_peer` --- client/peerset/src/protocol_controller.rs | 146 ---------------------- 1 file changed, 146 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index ea7493bbdb5b2..af446297ef536 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -42,7 +42,6 @@ enum Action { RemoveReservedPeer(PeerId), SetReservedPeers(HashSet), SetReservedOnly(bool), - DisconnectPeer(PeerId), GetReservedPeers(oneshot::Sender>), } @@ -92,12 +91,6 @@ impl ProtocolHandle { let _ = self.actions_tx.unbounded_send(Action::SetReservedOnly(reserved)); } - /// Disconnect peer. You should remove the `PeerId` from the `PeerStore` first - /// to not connect to the peer again during the next slot allocation. - pub fn disconnect_peer(&self, peer_id: PeerId) { - let _ = self.actions_tx.unbounded_send(Action::DisconnectPeer(peer_id)); - } - /// Get the list of reserved peers. pub fn reserved_peers(&self, pending_response: oneshot::Sender>) { let _ = self.actions_tx.unbounded_send(Action::GetReservedPeers(pending_response)); @@ -270,7 +263,6 @@ impl ProtocolController { Action::RemoveReservedPeer(peer_id) => self.on_remove_reserved_peer(peer_id), Action::SetReservedPeers(peer_ids) => self.on_set_reserved_peers(peer_ids), Action::SetReservedOnly(reserved_only) => self.on_set_reserved_only(reserved_only), - Action::DisconnectPeer(peer_id) => self.on_disconnect_peer(peer_id), Action::GetReservedPeers(pending_response) => self.on_get_reserved_peers(pending_response), } @@ -429,35 +421,6 @@ impl ProtocolController { let _ = pending_response.send(self.reserved_nodes.keys().cloned().collect()); } - /// Disconnect the peer. - fn on_disconnect_peer(&mut self, peer_id: PeerId) { - // Don't do anything if the node is reserved. - if self.reserved_nodes.contains_key(&peer_id) { - warn!( - target: LOG_TARGET, - "Ignoring request to disconnect reserved peer {} from {:?}.", peer_id, self.set_id, - ); - return - } - - match self.nodes.remove(&peer_id) { - Some(direction) => { - trace!(target: LOG_TARGET, "Disconnecting peer {peer_id} ({direction:?})."); - match direction { - Direction::Inbound => self.num_in -= 1, - Direction::Outbound => self.num_out -= 1, - } - self.drop_connection(peer_id); - }, - None => { - warn!( - target: LOG_TARGET, - "Trying to disconnect unknown peer {} from {:?}.", peer_id, self.set_id, - ); - }, - } - } - /// Indicate that we received an incoming connection. Must be answered either with /// a corresponding `Accept` or `Reject`, except if we were already connected to this peer. /// @@ -1350,115 +1313,6 @@ mod tests { assert_eq!(controller.num_out, 0); } - #[test] - fn disconnecting_regular_peers_work() { - let peer1 = PeerId::random(); - let peer2 = PeerId::random(); - let outgoing_candidates = vec![peer1]; - - let config = SetConfig { - in_peers: 10, - out_peers: 10, - bootnodes: Vec::new(), - reserved_nodes: HashSet::new(), - reserved_only: false, - }; - let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - - let mut peer_store = MockPeerStoreHandle::new(); - peer_store.expect_is_banned().once().return_const(false); - peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); - - let (_handle, mut controller) = - ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); - - // Connect `peer1` as outbound & `peer2` as inbound. - controller.alloc_slots(); - controller.on_incoming_connection(peer2, IncomingIndex(1)); - let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { - messages.push(message); - } - assert_eq!(messages.len(), 2); - assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); - assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); - assert_eq!(controller.nodes.len(), 2); - assert!(matches!(controller.nodes.get(&peer1), Some(Direction::Outbound))); - assert!(matches!(controller.nodes.get(&peer2), Some(Direction::Inbound))); - assert_eq!(controller.num_in, 1); - assert_eq!(controller.num_out, 1); - - controller.on_disconnect_peer(peer1); - assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: peer1 }); - assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); - assert_eq!(controller.nodes.len(), 1); - assert!(!controller.nodes.contains_key(&peer1)); - assert_eq!(controller.num_in, 1); - assert_eq!(controller.num_out, 0); - - controller.on_disconnect_peer(peer2); - assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: peer2 }); - assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); - assert_eq!(controller.nodes.len(), 0); - assert_eq!(controller.num_in, 0); - assert_eq!(controller.num_out, 0); - } - - #[test] - fn disconnecting_reserved_peers_is_a_noop() { - let reserved1 = PeerId::random(); - let reserved2 = PeerId::random(); - - let config = SetConfig { - in_peers: 10, - out_peers: 10, - bootnodes: Vec::new(), - reserved_nodes: [reserved1, reserved2].iter().cloned().collect(), - reserved_only: false, - }; - let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - - let mut peer_store = MockPeerStoreHandle::new(); - peer_store.expect_is_banned().times(2).return_const(false); - peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); - - let (_handle, mut controller) = - ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); - - // Connect `reserved1` as inbound & `reserved2` as outbound. - controller.on_incoming_connection(reserved1, IncomingIndex(1)); - controller.alloc_slots(); - let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { - messages.push(message); - } - assert_eq!(messages.len(), 2); - assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); - assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); - assert!(matches!( - controller.reserved_nodes.get(&reserved1), - Some(PeerState::Connected(Direction::Inbound)) - )); - assert!(matches!( - controller.reserved_nodes.get(&reserved2), - Some(PeerState::Connected(Direction::Outbound)) - )); - - controller.on_disconnect_peer(reserved1); - assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); - assert!(matches!( - controller.reserved_nodes.get(&reserved1), - Some(PeerState::Connected(Direction::Inbound)) - )); - - controller.on_disconnect_peer(reserved2); - assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); - assert!(matches!( - controller.reserved_nodes.get(&reserved2), - Some(PeerState::Connected(Direction::Outbound)) - )); - } - #[test] fn dropping_regular_peers_work() { let peer1 = PeerId::random(); From 2fbfe6c1168352558056eae094125ffb89509911 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Sun, 30 Apr 2023 09:54:32 +0300 Subject: [PATCH 68/98] Add logging of reputation changes --- client/peerset/src/peer_store.rs | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 6615611e42f89..a05c47ce1a5b0 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -38,7 +38,7 @@ const DISCONNECT_REPUTATION_CHANGE: i32 = -256; /// decrement of 50 we decrease absolute value of the reputation by 1/50. This corresponds to a /// factor of `k = 0.98`. It takes ~ `ln(0.5) / ln(k)` seconds to reduce the reputation by half, /// or 34.3 seconds for the values above. In this setup the maximum allowed absolute value of -/// `i32::MAX` becomes 0 in ~1100 seconds. +/// `i32::MAX` becomes 0 in ~1100 seconds (actually less due to integer arithmetic). const INVERSE_DECREMENT: i32 = 50; /// Amount of time between the moment we last updated the [`PeerStore`] entry and the moment we /// remove it, once the reputation value reaches 0. @@ -179,14 +179,30 @@ impl PeerStoreInner { } fn report_disconnect(&mut self, peer_id: PeerId) { - self.peers - .entry(peer_id) - .or_default() - .add_reputation(DISCONNECT_REPUTATION_CHANGE); + let peer_info = self.peers.entry(peer_id).or_default(); + peer_info.add_reputation(DISCONNECT_REPUTATION_CHANGE); + + log::trace!( + target: LOG_TARGET, + "Peer {} disconnected, reputation: {:+} to {}", + peer_id, + DISCONNECT_REPUTATION_CHANGE, + peer_info.reputation, + ); } fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange) { - self.peers.entry(peer_id).or_default().add_reputation(change.value); + let peer_info = self.peers.entry(peer_id).or_default(); + peer_info.add_reputation(change.value); + + log::trace!( + target: LOG_TARGET, + "Report {}: {:+} to {}. Reason: {}.", + peer_id, + change.value, + peer_info.reputation, + change.reason, + ); } fn peer_reputation(&self, peer_id: &PeerId) -> i32 { From dc85f1f92ad2f20eb9d1b8f19210d918b9e0c2db Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Sun, 30 Apr 2023 09:55:06 +0300 Subject: [PATCH 69/98] Fix warnings --- client/network/src/service/traits.rs | 4 ---- client/peerset/src/protocol_controller.rs | 10 +++++----- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 1da890c0bae2a..1dc11cd213bf1 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -156,10 +156,6 @@ pub trait NetworkPeers { /// Disconnect from a node as soon as possible. /// /// This triggers the same effects as if the connection had closed itself spontaneously. - /// - /// See also [`NetworkPeers::remove_from_peers_set`], which has the same effect but also - /// prevents the local node from re-establishing an outgoing substream to this peer until it - /// is added again. fn disconnect_peer(&self, who: PeerId, protocol: ProtocolName); /// Connect to unreserved peers and allow unreserved peers to connect for syncing purposes. diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index af446297ef536..f4531b4e753a1 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -353,7 +353,7 @@ impl ProtocolController { if let PeerState::Connected(direction) = state { if self.reserved_only { // Disconnect the node. - info!( + trace!( target: LOG_TARGET, "Disconnecting previously reserved node {} ({:?}) on {:?}.", peer_id, @@ -533,7 +533,7 @@ impl ProtocolController { *state = PeerState::NotConnected; Ok(true) } else { - Err(peer_id.clone()) + Err(*peer_id) } } @@ -960,8 +960,8 @@ mod tests { assert_eq!(controller.num_in, 0); // Drop peers. - controller.on_peer_dropped(peer1.clone()); - controller.on_peer_dropped(peer2.clone()); + controller.on_peer_dropped(peer1); + controller.on_peer_dropped(peer2); // Slots are freed. assert_eq!(controller.num_out, 0); @@ -1047,7 +1047,7 @@ mod tests { fn disabling_reserved_only_mode_allows_to_connect_to_peers() { let peer1 = PeerId::random(); let peer2 = PeerId::random(); - let candidates = vec![peer1.clone(), peer2.clone()]; + let candidates = vec![peer1, peer2]; let config = SetConfig { in_peers: 0, From 9b332973f8cde69b025a2e76c926aa34c2e77d6b Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Sun, 30 Apr 2023 09:59:01 +0300 Subject: [PATCH 70/98] minor: change log level --- client/peerset/src/protocol_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index f4531b4e753a1..3ac2ed437267c 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -498,7 +498,7 @@ impl ProtocolController { // We do not assert here, because due to asynchronous nature of communication // between `ProtocolController` and `Notifications` we can receive `Action::Dropped` // for a peer we already disconnected ourself. - warn!( + trace!( target: LOG_TARGET, "Received `Action::Dropped` for not connected peer {} on {:?}.", peer_id, From 198504ff36e712cbc1f8d4a07a02ec246d31131f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Sun, 30 Apr 2023 10:08:32 +0300 Subject: [PATCH 71/98] minor: remove unused import --- client/peerset/src/protocol_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 3ac2ed437267c..d51f304a73123 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -23,7 +23,7 @@ use futures::{channel::oneshot, future::Either, FutureExt, StreamExt}; use libp2p::PeerId; -use log::{error, info, trace, warn}; +use log::{error, trace, warn}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_arithmetic::traits::SaturatedConversion; use std::{ From 9e86b387f01c7f9e1c50720a60ebe210a20f23dd Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 8 May 2023 12:03:48 +0300 Subject: [PATCH 72/98] Apply suggestions from code review Co-authored-by: Aaro Altonen <48052676+altonen@users.noreply.github.com> --- client/peerset/src/peer_store.rs | 9 +++++---- client/peerset/src/protocol_controller.rs | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index a05c47ce1a5b0..2bdeaf8db64d2 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -48,16 +48,16 @@ pub trait PeerReputationProvider: Debug + Send { /// Check whether the peer is banned. fn is_banned(&self, peer_id: &PeerId) -> bool; - /// Report the peer disconnect for reputation adjustement. + /// Report peer disconnection for reputation adjustment. fn report_disconnect(&mut self, peer_id: PeerId); - /// Adjust peer reputation value. + /// Adjust peer reputation. fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); - /// Get peer reputation value. + /// Get peer reputation. fn peer_reputation(&self, peer_id: &PeerId) -> i32; - /// Get the candidates for initiating outgoing connections. + /// Get candidates with highest reputations for initiating outgoing connections. fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec; } @@ -233,6 +233,7 @@ impl PeerStoreInner { self.peers .iter_mut() .for_each(|(_, info)| info.decay_reputation(seconds_passed)); + // Retain only entries with non-zero reputation values or not expired ones. let now = Instant::now(); self.peers diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index d51f304a73123..ab84f1db8a04d 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -598,22 +598,22 @@ impl ProtocolController { (!self.reserved_nodes.contains_key(&peer_id) && !self.nodes.contains_key(&peer_id)) .then_some(peer_id) .or_else(|| { - debug_assert!(false, "`PeerStore` returned a node we asked to ignore."); error!( target: LOG_TARGET, "`PeerStore` returned a node we asked to ignore: {peer_id}.", ); + debug_assert!(false, "`PeerStore` returned a node we asked to ignore."); None }) }) .collect::>(); if candidates.len() > available_slots { - debug_assert!(false, "`PeerStore` returned more nodes than there are slots available."); error!( target: LOG_TARGET, "`PeerStore` returned more nodes than there are slots available.", - ) + ); + debug_assert!(false, "`PeerStore` returned more nodes than there are slots available."); } candidates.into_iter().take(available_slots).for_each(|peer_id| { From b3a72ada1cade261e1751ac62443c3e54212be1c Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 8 May 2023 16:27:03 +0300 Subject: [PATCH 73/98] minor: add more logging --- client/peerset/src/protocol_controller.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index ab84f1db8a04d..a2060829bdb58 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -270,16 +270,19 @@ impl ProtocolController { /// Send "accept" message to `Notifications`. fn accept_connection(&self, incoming_index: IncomingIndex) { + trace!(target: LOG_TARGET, "Accepting {incoming_index:?}."); let _ = self.to_notifications.unbounded_send(Message::Accept(incoming_index)); } /// Send "reject" message to `Notifications`. fn reject_connection(&self, incoming_index: IncomingIndex) { + trace!(target: LOG_TARGET, "Rejecting {incoming_index:?}."); let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); } /// Send "connect" message to `Notifications`. fn start_connection(&self, peer_id: PeerId) { + trace!(target: LOG_TARGET, "Connecting to {peer_id}."); let _ = self .to_notifications .unbounded_send(Message::Connect { set_id: self.set_id, peer_id }); @@ -287,6 +290,7 @@ impl ProtocolController { /// Send "drop" message to `Notifications`. fn drop_connection(&self, peer_id: PeerId) { + trace!(target: LOG_TARGET, "Dropping {peer_id}."); let _ = self .to_notifications .unbounded_send(Message::Drop { set_id: self.set_id, peer_id }); From e03f852a1860fb5e99a3840d7a8d32d8cf13cc09 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 9 May 2023 11:37:10 +0300 Subject: [PATCH 74/98] Document event prioritization requirements in `ProtocolController` --- client/peerset/src/protocol_controller.rs | 34 +++++++++++++++++++++-- 1 file changed, 32 insertions(+), 2 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index a2060829bdb58..b33062b129704 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -18,8 +18,28 @@ //! Protocol Controller. Generic implementation of peer management for protocols. //! Responsible for accepting/rejecting incoming connections and initiating outgoing connections, -//! respecting the inbound and outbound peer slot counts. Communicates with `Peerset` to get and +//! respecting the inbound and outbound peer slot counts. Communicates with `PeerStore` to get and //! update peer reputation values and sends commands to `Notifications`. +//! +//! Due to asynchronous nature of communication between `ProtocolController` and `Notifications`, +//! `ProtocolController` has an imperfect view of the states of the peers. This can lead to sending +//! confusing commands to `Notifications`. To mitigate this issue, the following measures are taken: +//! +//! 1. `Notifications` ignores all commands from `ProtocolController` after sending "incoming" +//! request, except the answer to this "incoming" request. +//! 2. `ProtocolController` does not modify the peers state after a command to `Notifications` +//! was sent, but before ACK for this command was heard back. +//! 3. Network peer events from `Notifictions` are prioritized over actions from external API and +//! internal actions by `ProtocolController` (like slot allocation). +//! +//! Even though the fuzz test from the crate does not reveal any inconsistencies in peer states +//! observed by `Notifications`, measures above do not provide 100% guarantee against confusing +//! commands from `ProtocolController`. This might happen, for example, if a network state event +//! from `Notifications` is processed after an external API action sent at the same time. +//! For this reason, `Notifications` must employ defensive programming and behave in a defined way +//! if seemingly impossible sequences of commands are received. For example, `Notifications` must +//! correctly handle a "connect" command for the peer it thinks is already connected, and a "drop" +//! command for a peer that was previously dropped. use futures::{channel::oneshot, future::Either, FutureExt, StreamExt}; use libp2p::PeerId; @@ -36,18 +56,27 @@ use crate::{ peer_store::PeerReputationProvider, IncomingIndex, Message, SetConfig, SetId, LOG_TARGET, }; +/// External API actions. #[derive(Debug)] enum Action { + /// Add a reserved peer or mark already connected peer as reserved. AddReservedPeer(PeerId), + /// Remove a reserved peer. RemoveReservedPeer(PeerId), + /// Update reserved peers to match the provided set. SetReservedPeers(HashSet), + /// Set/unset reserved-only mode. SetReservedOnly(bool), + /// Get the list of reserved peers. GetReservedPeers(oneshot::Sender>), } +/// Network events from `Notifications`. #[derive(Debug)] enum Event { + /// Incoming connection from the peer. IncomingConnection(PeerId, IncomingIndex), + /// Connection with the peer dropped. Dropped(PeerId), } @@ -213,7 +242,8 @@ impl ProtocolController { /// /// Intended for tests only. Use `run` for driving [`ProtocolController`]. pub async fn next_action(&mut self) -> bool { - // Perform tasks prioritizing connection events processing. + // Perform tasks prioritizing connection events processing + // (see the module documentation for details). let either = loop { if let Some(event) = self.events_rx.next().now_or_never() { match event { From 9a51e226a338432a02c02b98458ad5722e1aa9e6 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 9 May 2023 11:56:18 +0300 Subject: [PATCH 75/98] Rename `PeerReputationProvider` -> `PeerStoreProvider` --- client/peerset/src/lib.rs | 2 +- client/peerset/src/peer_store.rs | 4 ++-- client/peerset/src/protocol_controller.rs | 13 +++++-------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 6c35fc2d55bc0..ad284875b25ef 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -35,7 +35,7 @@ mod peer_store; mod protocol_controller; -use peer_store::{PeerReputationProvider, PeerStore, PeerStoreHandle}; +use peer_store::{PeerStore, PeerStoreHandle, PeerStoreProvider}; use protocol_controller::{ProtocolController, ProtocolHandle}; use futures::{ diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 2bdeaf8db64d2..2c8fbb0893139 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -44,7 +44,7 @@ const INVERSE_DECREMENT: i32 = 50; /// remove it, once the reputation value reaches 0. const FORGET_AFTER: Duration = Duration::from_secs(3600); -pub trait PeerReputationProvider: Debug + Send { +pub trait PeerStoreProvider: Debug + Send { /// Check whether the peer is banned. fn is_banned(&self, peer_id: &PeerId) -> bool; @@ -66,7 +66,7 @@ pub struct PeerStoreHandle { inner: Arc>, } -impl PeerReputationProvider for PeerStoreHandle { +impl PeerStoreProvider for PeerStoreHandle { fn is_banned(&self, peer_id: &PeerId) -> bool { self.inner.lock().unwrap().is_banned(peer_id) } diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index b33062b129704..abc77bf02dccf 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -52,9 +52,7 @@ use std::{ }; use wasm_timer::Delay; -use crate::{ - peer_store::PeerReputationProvider, IncomingIndex, Message, SetConfig, SetId, LOG_TARGET, -}; +use crate::{peer_store::PeerStoreProvider, IncomingIndex, Message, SetConfig, SetId, LOG_TARGET}; /// External API actions. #[derive(Debug)] @@ -198,7 +196,7 @@ pub struct ProtocolController { to_notifications: TracingUnboundedSender, /// `PeerStore` handle for checking peer reputation values and getting connection candidates /// with highest reputation. - peer_store: Box, + peer_store: Box, } impl ProtocolController { @@ -207,7 +205,7 @@ impl ProtocolController { set_id: SetId, config: SetConfig, to_notifications: TracingUnboundedSender, - peer_store: Box, + peer_store: Box, ) -> (ProtocolHandle, ProtocolController) { let (actions_tx, actions_rx) = tracing_unbounded("mpsc_api_protocol", 10_000); let (events_tx, events_rx) = tracing_unbounded("mpsc_notifications_protocol", 10_000); @@ -662,8 +660,7 @@ impl ProtocolController { mod tests { use super::{Direction, PeerState, ProtocolController}; use crate::{ - peer_store::PeerReputationProvider, IncomingIndex, Message, ReputationChange, SetConfig, - SetId, + peer_store::PeerStoreProvider, IncomingIndex, Message, ReputationChange, SetConfig, SetId, }; use libp2p::PeerId; use sc_utils::mpsc::{tracing_unbounded, TryRecvError}; @@ -673,7 +670,7 @@ mod tests { #[derive(Debug)] pub PeerStoreHandle {} - impl PeerReputationProvider for PeerStoreHandle { + impl PeerStoreProvider for PeerStoreHandle { fn is_banned(&self, peer_id: &PeerId) -> bool; fn report_disconnect(&mut self, peer_id: PeerId); fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); From a4e32a6329ee97d2c4405b5e24b820550bd649a0 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 9 May 2023 12:11:12 +0300 Subject: [PATCH 76/98] Add comment re dropped entries and `num_known_peers` --- client/peerset/src/peer_store.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 2c8fbb0893139..5976c4f92f28e 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -90,8 +90,10 @@ impl PeerStoreProvider for PeerStoreHandle { impl PeerStoreHandle { /// Get the number of known peers. + /// + /// This number might not include some connected peers in rare cases when their reputation + /// was not updated for one hour, because their entries in [`PeerStore`] were dropped. pub fn num_known_peers(&self) -> usize { - // FIXME: how do we use this info? May be better ask for numbers from protocol controllers? self.inner.lock().unwrap().peers.len() } From 0322d3d9b951054452cf6f177062267255f48c20 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 9 May 2023 12:36:44 +0300 Subject: [PATCH 77/98] Revert "Remove unused `ProtocolHandle::disconnect_peer`" This reverts commit 752d70bba208be93fa13121bb4260ece98a2c324. --- client/peerset/src/protocol_controller.rs | 147 ++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index abc77bf02dccf..31be32d6a114e 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -65,6 +65,8 @@ enum Action { SetReservedPeers(HashSet), /// Set/unset reserved-only mode. SetReservedOnly(bool), + /// Disconnect a peer. + DisconnectPeer(PeerId), /// Get the list of reserved peers. GetReservedPeers(oneshot::Sender>), } @@ -118,6 +120,12 @@ impl ProtocolHandle { let _ = self.actions_tx.unbounded_send(Action::SetReservedOnly(reserved)); } + /// Disconnect peer. You should remove the `PeerId` from the `PeerStore` first + /// to not connect to the peer again during the next slot allocation. + pub fn disconnect_peer(&self, peer_id: PeerId) { + let _ = self.actions_tx.unbounded_send(Action::DisconnectPeer(peer_id)); + } + /// Get the list of reserved peers. pub fn reserved_peers(&self, pending_response: oneshot::Sender>) { let _ = self.actions_tx.unbounded_send(Action::GetReservedPeers(pending_response)); @@ -291,6 +299,7 @@ impl ProtocolController { Action::RemoveReservedPeer(peer_id) => self.on_remove_reserved_peer(peer_id), Action::SetReservedPeers(peer_ids) => self.on_set_reserved_peers(peer_ids), Action::SetReservedOnly(reserved_only) => self.on_set_reserved_only(reserved_only), + Action::DisconnectPeer(peer_id) => self.on_disconnect_peer(peer_id), Action::GetReservedPeers(pending_response) => self.on_get_reserved_peers(pending_response), } @@ -453,6 +462,35 @@ impl ProtocolController { let _ = pending_response.send(self.reserved_nodes.keys().cloned().collect()); } + /// Disconnect the peer. + fn on_disconnect_peer(&mut self, peer_id: PeerId) { + // Don't do anything if the node is reserved. + if self.reserved_nodes.contains_key(&peer_id) { + warn!( + target: LOG_TARGET, + "Ignoring request to disconnect reserved peer {} from {:?}.", peer_id, self.set_id, + ); + return + } + + match self.nodes.remove(&peer_id) { + Some(direction) => { + trace!(target: LOG_TARGET, "Disconnecting peer {peer_id} ({direction:?})."); + match direction { + Direction::Inbound => self.num_in -= 1, + Direction::Outbound => self.num_out -= 1, + } + self.drop_connection(peer_id); + }, + None => { + warn!( + target: LOG_TARGET, + "Trying to disconnect unknown peer {} from {:?}.", peer_id, self.set_id, + ); + }, + } + } + /// Indicate that we received an incoming connection. Must be answered either with /// a corresponding `Accept` or `Reject`, except if we were already connected to this peer. /// @@ -1344,6 +1382,115 @@ mod tests { assert_eq!(controller.num_out, 0); } + #[test] + fn disconnecting_regular_peers_work() { + let peer1 = PeerId::random(); + let peer2 = PeerId::random(); + let outgoing_candidates = vec![peer1]; + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: HashSet::new(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_is_banned().once().return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); + + let (_handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); + + // Connect `peer1` as outbound & `peer2` as inbound. + controller.alloc_slots(); + controller.on_incoming_connection(peer2, IncomingIndex(1)); + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); + assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); + assert_eq!(controller.nodes.len(), 2); + assert!(matches!(controller.nodes.get(&peer1), Some(Direction::Outbound))); + assert!(matches!(controller.nodes.get(&peer2), Some(Direction::Inbound))); + assert_eq!(controller.num_in, 1); + assert_eq!(controller.num_out, 1); + + controller.on_disconnect_peer(peer1); + assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: peer1 }); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.nodes.len(), 1); + assert!(!controller.nodes.contains_key(&peer1)); + assert_eq!(controller.num_in, 1); + assert_eq!(controller.num_out, 0); + + controller.on_disconnect_peer(peer2); + assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: peer2 }); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert_eq!(controller.nodes.len(), 0); + assert_eq!(controller.num_in, 0); + assert_eq!(controller.num_out, 0); + } + + #[test] + fn disconnecting_reserved_peers_is_a_noop() { + let reserved1 = PeerId::random(); + let reserved2 = PeerId::random(); + + let config = SetConfig { + in_peers: 10, + out_peers: 10, + bootnodes: Vec::new(), + reserved_nodes: [reserved1, reserved2].iter().cloned().collect(), + reserved_only: false, + }; + let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); + + let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_is_banned().times(2).return_const(false); + peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); + + let (_handle, mut controller) = + ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); + + // Connect `reserved1` as inbound & `reserved2` as outbound. + controller.on_incoming_connection(reserved1, IncomingIndex(1)); + controller.alloc_slots(); + let mut messages = Vec::new(); + while let Some(message) = rx.try_recv().ok() { + messages.push(message); + } + assert_eq!(messages.len(), 2); + assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); + assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + assert!(matches!( + controller.reserved_nodes.get(&reserved1), + Some(PeerState::Connected(Direction::Inbound)) + )); + assert!(matches!( + controller.reserved_nodes.get(&reserved2), + Some(PeerState::Connected(Direction::Outbound)) + )); + + controller.on_disconnect_peer(reserved1); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!( + controller.reserved_nodes.get(&reserved1), + Some(PeerState::Connected(Direction::Inbound)) + )); + + controller.on_disconnect_peer(reserved2); + assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); + assert!(matches!( + controller.reserved_nodes.get(&reserved2), + Some(PeerState::Connected(Direction::Outbound)) + )); + } + #[test] fn dropping_regular_peers_work() { let peer1 = PeerId::random(); From 0944c8e31424c68fc0a30b037181d7f7626a7113 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 9 May 2023 12:55:36 +0300 Subject: [PATCH 78/98] Disconnect peers with reputation below `BANNED_THRESHOLD` --- client/peerset/src/peer_store.rs | 44 ++++++++++++++++++----- client/peerset/src/protocol_controller.rs | 4 ++- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 5976c4f92f28e..692b961b46f9c 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -28,7 +28,7 @@ use std::{ }; use wasm_timer::Delay; -use crate::{ReputationChange, LOG_TARGET}; +use crate::{protocol_controller::ProtocolHandle, ReputationChange, LOG_TARGET}; /// We don't accept nodes whose reputation is under this value. pub const BANNED_THRESHOLD: i32 = 82 * (i32::MIN / 100); @@ -48,6 +48,9 @@ pub trait PeerStoreProvider: Debug + Send { /// Check whether the peer is banned. fn is_banned(&self, peer_id: &PeerId) -> bool; + /// Register a protocol handle to disconnect peers whose reputation drops below the threshold. + fn register_protocol(&self, protocol_handle: ProtocolHandle); + /// Report peer disconnection for reputation adjustment. fn report_disconnect(&mut self, peer_id: PeerId); @@ -71,6 +74,10 @@ impl PeerStoreProvider for PeerStoreHandle { self.inner.lock().unwrap().is_banned(peer_id) } + fn register_protocol(&self, protocol_handle: ProtocolHandle) { + self.inner.lock().unwrap().register_protocol(protocol_handle); + } + fn report_disconnect(&mut self, peer_id: PeerId) { self.inner.lock().unwrap().report_disconnect(peer_id) } @@ -173,6 +180,7 @@ impl PeerInfo { #[derive(Debug)] struct PeerStoreInner { peers: HashMap, + protocols: Vec, } impl PeerStoreInner { @@ -180,6 +188,10 @@ impl PeerStoreInner { self.peers.get(peer_id).map_or(false, |info| info.is_banned()) } + fn register_protocol(&mut self, protocol_handle: ProtocolHandle) { + self.protocols.push(protocol_handle); + } + fn report_disconnect(&mut self, peer_id: PeerId) { let peer_info = self.peers.entry(peer_id).or_default(); peer_info.add_reputation(DISCONNECT_REPUTATION_CHANGE); @@ -197,14 +209,27 @@ impl PeerStoreInner { let peer_info = self.peers.entry(peer_id).or_default(); peer_info.add_reputation(change.value); - log::trace!( - target: LOG_TARGET, - "Report {}: {:+} to {}. Reason: {}.", - peer_id, - change.value, - peer_info.reputation, - change.reason, - ); + if peer_info.reputation < BANNED_THRESHOLD { + self.protocols.iter().for_each(|handle| handle.disconnect_peer(peer_id)); + + log::trace!( + target: LOG_TARGET, + "Report {}: {:+} to {}. Reason: {}. Banned, disconnecting.", + peer_id, + change.value, + peer_info.reputation, + change.reason, + ); + } else { + log::trace!( + target: LOG_TARGET, + "Report {}: {:+} to {}. Reason: {}.", + peer_id, + change.value, + peer_info.reputation, + change.reason, + ); + } } fn peer_reputation(&self, peer_id: &PeerId) -> i32 { @@ -273,6 +298,7 @@ impl PeerStore { .into_iter() .map(|peer_id| (peer_id, PeerInfo::default())) .collect(), + protocols: Vec::new(), })), } } diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 31be32d6a114e..844c2ca77f20d 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -218,6 +218,7 @@ impl ProtocolController { let (actions_tx, actions_rx) = tracing_unbounded("mpsc_api_protocol", 10_000); let (events_tx, events_rx) = tracing_unbounded("mpsc_notifications_protocol", 10_000); let handle = ProtocolHandle { actions_tx, events_tx }; + peer_store.register_protocol(handle.clone()); let reserved_nodes = config.reserved_nodes.iter().map(|p| (*p, PeerState::NotConnected)).collect(); let controller = ProtocolController { @@ -696,7 +697,7 @@ impl ProtocolController { #[cfg(test)] mod tests { - use super::{Direction, PeerState, ProtocolController}; + use super::{Direction, PeerState, ProtocolController, ProtocolHandle}; use crate::{ peer_store::PeerStoreProvider, IncomingIndex, Message, ReputationChange, SetConfig, SetId, }; @@ -710,6 +711,7 @@ mod tests { impl PeerStoreProvider for PeerStoreHandle { fn is_banned(&self, peer_id: &PeerId) -> bool; + fn register_protocol(&self, protocol_handle: ProtocolHandle); fn report_disconnect(&mut self, peer_id: PeerId); fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange); fn peer_reputation(&self, peer_id: &PeerId) -> i32; From fba6f2de849d151e628358f1dab4e2dfdef9743f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 9 May 2023 15:27:40 +0300 Subject: [PATCH 79/98] Switch to `parking_lot::Mutex` --- Cargo.lock | 1 + client/peerset/Cargo.toml | 1 + client/peerset/src/peer_store.rs | 21 +++++++++++---------- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dba98681dc1f5..770366f9e1071 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9349,6 +9349,7 @@ dependencies = [ "libp2p", "log", "mockall", + "parking_lot 0.12.1", "partial_sort", "rand 0.8.5", "sc-utils", diff --git a/client/peerset/Cargo.toml b/client/peerset/Cargo.toml index 00a05601ee879..688f550296957 100644 --- a/client/peerset/Cargo.toml +++ b/client/peerset/Cargo.toml @@ -17,6 +17,7 @@ targets = ["x86_64-unknown-linux-gnu"] futures = "0.3.21" libp2p = "0.50.0" log = "0.4.17" +parking_lot = "0.12.1" partial_sort = "0.2.0" serde_json = "1.0.85" wasm-timer = "0.2" diff --git a/client/peerset/src/peer_store.rs b/client/peerset/src/peer_store.rs index 692b961b46f9c..88659fd00edcf 100644 --- a/client/peerset/src/peer_store.rs +++ b/client/peerset/src/peer_store.rs @@ -18,12 +18,13 @@ use libp2p::PeerId; use log::trace; +use parking_lot::Mutex; use partial_sort::PartialSort; use std::{ cmp::{Ord, Ordering, PartialOrd}, collections::{hash_map::Entry, HashMap, HashSet}, fmt::Debug, - sync::{Arc, Mutex}, + sync::Arc, time::{Duration, Instant}, }; use wasm_timer::Delay; @@ -71,27 +72,27 @@ pub struct PeerStoreHandle { impl PeerStoreProvider for PeerStoreHandle { fn is_banned(&self, peer_id: &PeerId) -> bool { - self.inner.lock().unwrap().is_banned(peer_id) + self.inner.lock().is_banned(peer_id) } fn register_protocol(&self, protocol_handle: ProtocolHandle) { - self.inner.lock().unwrap().register_protocol(protocol_handle); + self.inner.lock().register_protocol(protocol_handle); } fn report_disconnect(&mut self, peer_id: PeerId) { - self.inner.lock().unwrap().report_disconnect(peer_id) + self.inner.lock().report_disconnect(peer_id) } fn report_peer(&mut self, peer_id: PeerId, change: ReputationChange) { - self.inner.lock().unwrap().report_peer(peer_id, change) + self.inner.lock().report_peer(peer_id, change) } fn peer_reputation(&self, peer_id: &PeerId) -> i32 { - self.inner.lock().unwrap().peer_reputation(peer_id) + self.inner.lock().peer_reputation(peer_id) } fn outgoing_candidates(&self, count: usize, ignored: HashSet<&PeerId>) -> Vec { - self.inner.lock().unwrap().outgoing_candidates(count, ignored) + self.inner.lock().outgoing_candidates(count, ignored) } } @@ -101,12 +102,12 @@ impl PeerStoreHandle { /// This number might not include some connected peers in rare cases when their reputation /// was not updated for one hour, because their entries in [`PeerStore`] were dropped. pub fn num_known_peers(&self) -> usize { - self.inner.lock().unwrap().peers.len() + self.inner.lock().peers.len() } /// Add known peer. pub fn add_known_peer(&mut self, peer_id: PeerId) { - self.inner.lock().unwrap().add_known_peer(peer_id); + self.inner.lock().add_known_peer(peer_id); } } @@ -324,7 +325,7 @@ impl PeerStore { elapsed_now.as_secs() - elapsed_latest.as_secs() }; - self.inner.lock().unwrap().progress_time(seconds_passed); + self.inner.lock().progress_time(seconds_passed); let _ = Delay::new(Duration::from_secs(1)).await; } } From c94ea21966785abf15bab3142b89eeb57452627c Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Wed, 10 May 2023 18:00:17 +0300 Subject: [PATCH 80/98] WIP: introduce outstanding events/actions handling --- client/peerset/src/protocol_controller.rs | 117 +++++++++++++++++++--- 1 file changed, 104 insertions(+), 13 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 844c2ca77f20d..5a6144fb17482 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -47,7 +47,7 @@ use log::{error, trace, warn}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_arithmetic::traits::SaturatedConversion; use std::{ - collections::{HashMap, HashSet}, + collections::{HashMap, HashSet, VecDeque}, time::{Duration, Instant}, }; use wasm_timer::Delay; @@ -80,6 +80,28 @@ enum Event { Dropped(PeerId), } +impl Action { + fn peer_id(&self) -> Option<&PeerId> { + match self { + Self::AddReservedPeer(peer_id) => Some(peer_id), + Self::RemoveReservedPeer(peer_id) => Some(peer_id), + Self::SetReservedPeers(_) => None, + Self::SetReservedOnly(_) => None, + Self::DisconnectPeer(peer_id) => Some(peer_id), + Self::GetReservedPeers(_) => None, + } + } +} + +impl Event { + fn peer_id(&self) -> &PeerId { + match self { + Self::IncomingConnection(peer_id, _) => peer_id, + Self::Dropped(peer_id) => peer_id, + } + } +} + /// Shared handle to [`ProtocolController`]. Distributed around the code outside of the /// protocol implementation. #[derive(Debug, Clone)] @@ -173,6 +195,34 @@ impl Default for PeerState { } } +/// Outstanding events and actions for a peer that we can't process because we are waiting +/// for an ACK. +#[derive(Debug, Default)] +struct Outstanding { + pending_ack: Option>, + events: VecDeque, + actions: VecDeque, +} + +impl Outstanding { + fn try_resolve_ack(&mut self) -> bool { + match self.pending_ack { + Some(ref mut pending_ack) => + if pending_ack.try_recv().ok().flatten().is_some() { + self.pending_ack = None; + true + } else { + false + }, + None => true, + } + } + + fn is_empty(&self) -> bool { + self.pending_ack.is_none() && self.events.is_empty() && self.actions.is_empty() + } +} + /// Side of [`ProtocolHandle`] responsible for all the logic. Currently all instances are /// owned by [`crate::Peerset`], but they should eventually be moved to corresponding protocols. #[derive(Debug)] @@ -205,6 +255,12 @@ pub struct ProtocolController { /// `PeerStore` handle for checking peer reputation values and getting connection candidates /// with highest reputation. peer_store: Box, + /// Pending ACKS + /// Outstanding events and actions per peer. + outstanding: HashMap, + /// If we have outstanding (global) `SetReservedOnly` action. It's processed once we receive + /// all pending ACKs. + outstanding_set_reserved_only: Option, } impl ProtocolController { @@ -235,6 +291,8 @@ impl ProtocolController { next_periodic_alloc_slots: Instant::now(), to_notifications, peer_store, + outstanding: HashMap::new(), + outstanding_set_reserved_only: None, }; (handle, controller) } @@ -251,16 +309,29 @@ impl ProtocolController { pub async fn next_action(&mut self) -> bool { // Perform tasks prioritizing connection events processing // (see the module documentation for details). - let either = loop { - if let Some(event) = self.events_rx.next().now_or_never() { - match event { - None => return false, - Some(event) => break Either::Left(event), + let mut emptied_outstanding = None; + let either = 'outer: loop { + // Receive ACKs from `Notifications` and process outstanding events & actions. + for (peer_id, outstanding) in self.outstanding.iter_mut() { + if outstanding.try_resolve_ack() { + if let Some(event) = outstanding.events.pop_front() { + emptied_outstanding = outstanding.is_empty().then_some(*peer_id); + break 'outer Either::Left(event) + } + if let Some(action) = outstanding.actions.pop_front() { + emptied_outstanding = outstanding.is_empty().then_some(*peer_id); + break 'outer Either::Right(action) + } } } + // If we have received some ACKs, we might be able to remove some obsolete entries. + self.outstanding.retain(|_, outstanding| !outstanding.is_empty()); + let mut next_alloc_slots = Delay::new_at(self.next_periodic_alloc_slots).fuse(); - futures::select! { + + // See the module doc for why we use `select_biased!`. + futures::select_biased! { event = self.events_rx.next() => match event { Some(event) => break Either::Left(event), None => return false, @@ -276,6 +347,9 @@ impl ProtocolController { } }; + // Remove outstanding entry that became empty. + emptied_outstanding.map(|peer_id| self.outstanding.remove(&peer_id)); + match either { Either::Left(event) => self.process_event(event), Either::Right(action) => self.process_action(action), @@ -286,20 +360,37 @@ impl ProtocolController { /// Process connection event. fn process_event(&mut self, event: Event) { - match event { - Event::IncomingConnection(peer_id, index) => - self.on_incoming_connection(peer_id, index), - Event::Dropped(peer_id) => self.on_peer_dropped(peer_id), + if let Some(outstanding) = self.outstanding.get_mut(event.peer_id()) { + outstanding.events.push_back(event); + } else { + match event { + Event::IncomingConnection(peer_id, index) => + self.on_incoming_connection(peer_id, index), + Event::Dropped(peer_id) => self.on_peer_dropped(peer_id), + } } } /// Process action command. fn process_action(&mut self, action: Action) { + if let Some(peer_id) = action.peer_id() { + if let Some(outstanding) = self.outstanding.get_mut(peer_id) { + outstanding.actions.push_back(action); + return + } + } + match action { Action::AddReservedPeer(peer_id) => self.on_add_reserved_peer(peer_id), Action::RemoveReservedPeer(peer_id) => self.on_remove_reserved_peer(peer_id), - Action::SetReservedPeers(peer_ids) => self.on_set_reserved_peers(peer_ids), - Action::SetReservedOnly(reserved_only) => self.on_set_reserved_only(reserved_only), + Action::SetReservedPeers(peer_ids) => { + todo!("Decompose into individual peer actions if `self.outstanding` is not empty."); + self.on_set_reserved_peers(peer_ids); + }, + Action::SetReservedOnly(reserved_only) => { + todo!("Postpone operation if `self.outstanding` is not empty."); + self.on_set_reserved_only(reserved_only); + }, Action::DisconnectPeer(peer_id) => self.on_disconnect_peer(peer_id), Action::GetReservedPeers(pending_response) => self.on_get_reserved_peers(pending_response), From 34b2d1b00c8a4a32eaa7533f3e900e12908c1d3e Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 11 May 2023 11:49:46 +0300 Subject: [PATCH 81/98] WIP: defer/process outstanding `SetReservedPeers` and `SetReservedOnly` --- client/peerset/src/protocol_controller.rs | 61 +++++++++++++++++------ 1 file changed, 46 insertions(+), 15 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 5a6144fb17482..fc3c35d3c53b1 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -325,9 +325,18 @@ impl ProtocolController { } } - // If we have received some ACKs, we might be able to remove some obsolete entries. + // If we have received some ACKs, we might be able to remove obsolete entries. self.outstanding.retain(|_, outstanding| !outstanding.is_empty()); + // Apply postponed `SetReservedOnly` action + // TODO: we might need to do it after all events processing. + if self.outstanding.is_empty() { + if let Some(reserved_only) = self.outstanding_set_reserved_only.take() { + self.on_set_reserved_only(reserved_only); + return true + } + } + let mut next_alloc_slots = Delay::new_at(self.next_periodic_alloc_slots).fuse(); // See the module doc for why we use `select_biased!`. @@ -383,14 +392,18 @@ impl ProtocolController { match action { Action::AddReservedPeer(peer_id) => self.on_add_reserved_peer(peer_id), Action::RemoveReservedPeer(peer_id) => self.on_remove_reserved_peer(peer_id), - Action::SetReservedPeers(peer_ids) => { - todo!("Decompose into individual peer actions if `self.outstanding` is not empty."); - self.on_set_reserved_peers(peer_ids); - }, - Action::SetReservedOnly(reserved_only) => { - todo!("Postpone operation if `self.outstanding` is not empty."); - self.on_set_reserved_only(reserved_only); - }, + Action::SetReservedPeers(peer_ids) => self.on_set_reserved_peers(peer_ids), + Action::SetReservedOnly(reserved_only) => + if !self.outstanding.is_empty() { + // we need to postpone the action if there are outstanding events/actions. + if reserved_only == self.reserved_only { + self.outstanding_set_reserved_only = None; + } else { + self.outstanding_set_reserved_only = Some(reserved_only); + } + } else { + self.on_set_reserved_only(reserved_only); + }, Action::DisconnectPeer(peer_id) => self.on_disconnect_peer(peer_id), Action::GetReservedPeers(pending_response) => self.on_get_reserved_peers(pending_response), @@ -522,12 +535,30 @@ impl ProtocolController { let to_insert = peer_ids.difference(¤t).cloned().collect::>(); let to_remove = current.difference(&peer_ids).cloned().collect::>(); - for node in to_insert { - self.on_add_reserved_peer(node); - } - - for node in to_remove { - self.on_remove_reserved_peer(node); + if self.outstanding.is_empty() { + // Process peers immediately if we have no oustanding events/actions. + for peer_id in to_insert { + self.on_add_reserved_peer(peer_id); + } + for peer_id in to_remove { + self.on_remove_reserved_peer(peer_id); + } + } else { + // Push actions to outstanding queues if there are outstanding events/actions. + for peer_id in to_insert { + self.outstanding + .entry(peer_id) + .or_default() + .actions + .push_back(Action::AddReservedPeer(peer_id)); + } + for peer_id in to_remove { + self.outstanding + .entry(peer_id) + .or_default() + .actions + .push_back(Action::RemoveReservedPeer(peer_id)); + } } } From ace2dd0b4d8ffbd144968a066610759c6c069230 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 11 May 2023 17:55:30 +0300 Subject: [PATCH 82/98] WIP: implement ACKing of `ProtocolController` messages --- client/peerset/src/lib.rs | 21 +- client/peerset/src/protocol_controller.rs | 256 +++++++++++++++------- 2 files changed, 195 insertions(+), 82 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index ad284875b25ef..2b99df041b7fc 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -181,6 +181,7 @@ pub enum Message { /// Request to open a connection to the given peer. From the point of view of the PSM, we are /// immediately connected. Connect { + /// Set id to connect on. set_id: SetId, /// Peer to connect to. peer_id: PeerId, @@ -188,6 +189,7 @@ pub enum Message { /// Drop the connection to the given peer, or cancel the connection attempt after a `Connect`. Drop { + /// Set id to disconnect on. set_id: SetId, /// Peer to disconnect from. peer_id: PeerId, @@ -200,6 +202,18 @@ pub enum Message { Reject(IncomingIndex), } +/// Message with oneshot ACK channel. +#[derive(Debug)] +pub struct AckedMessage(Message, oneshot::Sender<()>); + +#[cfg(test)] +impl AckedMessage { + fn take(self) -> Message { + let _ = self.1.send(()); + self.0 + } +} + /// Opaque identifier for an incoming connection. Allocated by the network. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct IncomingIndex(pub u64); @@ -257,7 +271,7 @@ pub struct Peerset { protocol_controller_futures: JoinAll>, /// Commands sent from protocol controllers to `Notifications`. The size of this vector never /// changes. - from_controllers: TracingUnboundedReceiver, + from_controllers: TracingUnboundedReceiver, /// Receiver for messages from the `PeersetHandle` and from `to_self`. from_handle: TracingUnboundedReceiver, /// Sending side of `from_handle`. @@ -361,7 +375,10 @@ impl Stream for Peerset { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { if let Poll::Ready(msg) = self.from_controllers.poll_next_unpin(cx) { - if let Some(msg) = msg { + if let Some(AckedMessage(msg, tx_ack)) = msg { + // `Peerset` is polled directly by `Notifications`, so we can already send ACK here. + // Once `Peerset` is eliminated, ACKs must be sent by `Notifications`. + let _ = tx_ack.send(()); return Poll::Ready(Some(msg)) } else { debug!( diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index fc3c35d3c53b1..0a0e1aa26d2cf 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -52,7 +52,10 @@ use std::{ }; use wasm_timer::Delay; -use crate::{peer_store::PeerStoreProvider, IncomingIndex, Message, SetConfig, SetId, LOG_TARGET}; +use crate::{ + peer_store::PeerStoreProvider, AckedMessage, IncomingIndex, Message, SetConfig, SetId, + LOG_TARGET, +}; /// External API actions. #[derive(Debug)] @@ -251,7 +254,7 @@ pub struct ProtocolController { /// Next time to allocate slots. This is done once per second. next_periodic_alloc_slots: Instant, /// Outgoing channel for messages to `Notifications`. - to_notifications: TracingUnboundedSender, + to_notifications: TracingUnboundedSender, /// `PeerStore` handle for checking peer reputation values and getting connection candidates /// with highest reputation. peer_store: Box, @@ -268,7 +271,7 @@ impl ProtocolController { pub fn new( set_id: SetId, config: SetConfig, - to_notifications: TracingUnboundedSender, + to_notifications: TracingUnboundedSender, peer_store: Box, ) -> (ProtocolHandle, ProtocolController) { let (actions_tx, actions_rx) = tracing_unbounded("mpsc_api_protocol", 10_000); @@ -411,31 +414,57 @@ impl ProtocolController { } /// Send "accept" message to `Notifications`. - fn accept_connection(&self, incoming_index: IncomingIndex) { + fn accept_connection(&mut self, peer_id: PeerId, incoming_index: IncomingIndex) { trace!(target: LOG_TARGET, "Accepting {incoming_index:?}."); - let _ = self.to_notifications.unbounded_send(Message::Accept(incoming_index)); + + let tx_ack = self.insert_outstanding_ack(peer_id); + let _ = self + .to_notifications + .unbounded_send(AckedMessage(Message::Accept(incoming_index), tx_ack)); } /// Send "reject" message to `Notifications`. - fn reject_connection(&self, incoming_index: IncomingIndex) { + fn reject_connection(&mut self, peer_id: PeerId, incoming_index: IncomingIndex) { trace!(target: LOG_TARGET, "Rejecting {incoming_index:?}."); - let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); + + let tx_ack = self.insert_outstanding_ack(peer_id); + let _ = self + .to_notifications + .unbounded_send(AckedMessage(Message::Reject(incoming_index), tx_ack)); } /// Send "connect" message to `Notifications`. - fn start_connection(&self, peer_id: PeerId) { + fn start_connection(&mut self, peer_id: PeerId) { trace!(target: LOG_TARGET, "Connecting to {peer_id}."); + + let tx_ack = self.insert_outstanding_ack(peer_id); let _ = self .to_notifications - .unbounded_send(Message::Connect { set_id: self.set_id, peer_id }); + .unbounded_send(AckedMessage(Message::Connect { set_id: self.set_id, peer_id }, tx_ack)); } /// Send "drop" message to `Notifications`. - fn drop_connection(&self, peer_id: PeerId) { + fn drop_connection(&mut self, peer_id: PeerId) { trace!(target: LOG_TARGET, "Dropping {peer_id}."); + + let tx_ack = self.insert_outstanding_ack(peer_id); let _ = self .to_notifications - .unbounded_send(Message::Drop { set_id: self.set_id, peer_id }); + .unbounded_send(AckedMessage(Message::Drop { set_id: self.set_id, peer_id }, tx_ack)); + } + + /// Insert outstanding entry and/or initialize oneshot channel for ACK. + fn insert_outstanding_ack(&mut self, peer_id: PeerId) -> oneshot::Sender<()> { + let outstanding = self.outstanding.entry(peer_id).or_default(); + debug_assert!( + outstanding.pending_ack.is_none(), + "Discarding previous pending ACK channel before it was received." + ); + + let (tx, rx) = oneshot::channel(); + outstanding.pending_ack = Some(rx); + + tx } /// Report peer disconnect event to `PeerStore` for it to update peer's reputation accordingly. @@ -574,7 +603,12 @@ impl ProtocolController { } // Disconnect all non-reserved peers. - self.nodes.keys().for_each(|peer_id| self.drop_connection(*peer_id)); + self.nodes + .keys() + .cloned() + .collect::>() + .iter() + .for_each(|peer_id| self.drop_connection(*peer_id)); self.nodes.clear(); self.num_in = 0; self.num_out = 0; @@ -629,7 +663,7 @@ impl ProtocolController { trace!(target: LOG_TARGET, "Incoming connection from peer {peer_id} ({incoming_index:?}).",); if self.reserved_only && !self.reserved_nodes.contains_key(&peer_id) { - self.reject_connection(incoming_index); + self.reject_connection(peer_id, incoming_index); return } @@ -640,15 +674,15 @@ impl ProtocolController { // We are accepting an incoming connection, so ensure the direction is inbound. // (See the note above.) *direction = Direction::Inbound; - self.accept_connection(incoming_index); + self.accept_connection(peer_id, incoming_index); }, PeerState::NotConnected => { // FIXME: unable to call `self.is_banned()` because of borrowed `self`. if self.peer_store.is_banned(&peer_id) { - self.reject_connection(incoming_index); + self.reject_connection(peer_id, incoming_index); } else { *state = PeerState::Connected(Direction::Inbound); - self.accept_connection(incoming_index); + self.accept_connection(peer_id, incoming_index); } }, } @@ -671,18 +705,18 @@ impl ProtocolController { } if self.num_in >= self.max_in { - self.reject_connection(incoming_index); + self.reject_connection(peer_id, incoming_index); return } if self.is_banned(&peer_id) { - self.reject_connection(incoming_index); + self.reject_connection(peer_id, incoming_index); return } self.num_in += 1; self.nodes.insert(peer_id, Direction::Inbound); - self.accept_connection(incoming_index); + self.accept_connection(peer_id, incoming_index); } /// Indicate that a connection with the peer was dropped. @@ -750,11 +784,14 @@ impl ProtocolController { /// Initiate outgoing connections trying to connect all reserved nodes and fill in all outgoing /// slots. fn alloc_slots(&mut self) { - // Try connecting to reserved nodes first. + // Try connecting to reserved nodes first, ignoring nodes with outstanding events/actions. self.reserved_nodes .iter_mut() .filter_map(|(peer_id, state)| { - (!state.is_connected() && !self.peer_store.is_banned(peer_id)).then(|| { + (!state.is_connected() && + !self.peer_store.is_banned(peer_id) && + !self.outstanding.contains_key(peer_id)) + .then(|| { *state = PeerState::Connected(Direction::Outbound); peer_id }) @@ -774,13 +811,17 @@ impl ProtocolController { // Fill available slots. let available_slots = (self.max_out - self.num_out).saturated_into(); - // Ignore reserved nodes (connected above) and already connected nodes. + // Ignore reserved nodes (connected above), already connected nodes, and nodes with + // outstanding events/actions. let ignored = self .reserved_nodes .keys() .collect::>() .union(&self.nodes.keys().collect::>()) .cloned() + .collect::>() + .union(&self.outstanding.keys().collect::>()) + .cloned() .collect(); let candidates = self @@ -788,16 +829,18 @@ impl ProtocolController { .outgoing_candidates(available_slots, ignored) .into_iter() .filter_map(|peer_id| { - (!self.reserved_nodes.contains_key(&peer_id) && !self.nodes.contains_key(&peer_id)) - .then_some(peer_id) - .or_else(|| { - error!( - target: LOG_TARGET, - "`PeerStore` returned a node we asked to ignore: {peer_id}.", - ); - debug_assert!(false, "`PeerStore` returned a node we asked to ignore."); - None - }) + (!self.reserved_nodes.contains_key(&peer_id) && + !self.nodes.contains_key(&peer_id) && + !self.outstanding.contains_key(&peer_id)) + .then_some(peer_id) + .or_else(|| { + error!( + target: LOG_TARGET, + "`PeerStore` returned a node we asked to ignore: {peer_id}.", + ); + debug_assert!(false, "`PeerStore` returned a node we asked to ignore."); + None + }) }) .collect::>(); @@ -821,7 +864,8 @@ impl ProtocolController { mod tests { use super::{Direction, PeerState, ProtocolController, ProtocolHandle}; use crate::{ - peer_store::PeerStoreProvider, IncomingIndex, Message, ReputationChange, SetConfig, SetId, + peer_store::PeerStoreProvider, AckedMessage, IncomingIndex, Message, ReputationChange, + SetConfig, SetId, }; use libp2p::PeerId; use sc_utils::mpsc::{tracing_unbounded, TryRecvError}; @@ -857,6 +901,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().times(4).return_const(false); peer_store.expect_report_disconnect().times(2).return_const(()); @@ -871,8 +916,9 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); @@ -889,13 +935,13 @@ mod tests { // Incoming connection from `reserved1`. let incoming1 = IncomingIndex(1); controller.on_incoming_connection(reserved1, incoming1); - assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming1)); + assert_eq!(rx.try_recv().unwrap().take(), Message::Accept(incoming1)); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Incoming connection from `reserved2`. let incoming2 = IncomingIndex(2); controller.on_incoming_connection(reserved2, incoming2); - assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming2)); + assert_eq!(rx.try_recv().unwrap().take(), Message::Accept(incoming2)); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Reserved peers do not occupy slots. @@ -919,6 +965,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().times(6).return_const(true); let (_handle, mut controller) = @@ -940,13 +987,13 @@ mod tests { // Incoming connection from `reserved1`. let incoming1 = IncomingIndex(1); controller.on_incoming_connection(reserved1, incoming1); - assert_eq!(rx.try_recv().unwrap(), Message::Reject(incoming1)); + assert_eq!(rx.try_recv().unwrap().take(), Message::Reject(incoming1)); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Incoming connection from `reserved2`. let incoming2 = IncomingIndex(2); controller.on_incoming_connection(reserved2, incoming2); - assert_eq!(rx.try_recv().unwrap(), Message::Reject(incoming2)); + assert_eq!(rx.try_recv().unwrap().take(), Message::Reject(incoming2)); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // No slots occupied. @@ -970,6 +1017,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().times(4).return_const(false); peer_store.expect_report_disconnect().times(2).return_const(()); @@ -983,8 +1031,9 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); @@ -999,8 +1048,9 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); @@ -1029,6 +1079,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_outgoing_candidates().once().return_const(candidates); let (_handle, mut controller) = @@ -1038,8 +1089,9 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } // Only first two peers are connected (we only have 2 slots). @@ -1079,6 +1131,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1089,8 +1142,9 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 4); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); @@ -1120,6 +1174,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_outgoing_candidates().once().return_const(candidates1); peer_store.expect_outgoing_candidates().once().return_const(candidates2); peer_store.expect_report_disconnect().times(2).return_const(()); @@ -1131,8 +1186,9 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } // Only first two peers are connected (we only have 2 slots). @@ -1164,8 +1220,9 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } // Peers are connected. @@ -1189,7 +1246,8 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let peer_store = MockPeerStoreHandle::new(); + let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); @@ -1215,7 +1273,8 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let peer_store = MockPeerStoreHandle::new(); + let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); @@ -1225,8 +1284,9 @@ mod tests { controller.on_incoming_connection(peer, incoming_index); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } // Peer is rejected. @@ -1253,6 +1313,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_outgoing_candidates().once().return_const(candidates); let (_handle, mut controller) = @@ -1270,8 +1331,9 @@ mod tests { controller.on_set_reserved_only(false); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); @@ -1299,6 +1361,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().times(3).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1311,8 +1374,9 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 3); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); @@ -1324,7 +1388,7 @@ mod tests { // Connect `regular2` as inbound. let incoming_index = IncomingIndex(1); controller.on_incoming_connection(regular2, incoming_index); - assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming_index)); + assert_eq!(rx.try_recv().unwrap().take(), Message::Accept(incoming_index)); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert_eq!(controller.num_out, 1); assert_eq!(controller.num_in, 1); @@ -1333,8 +1397,9 @@ mod tests { controller.on_set_reserved_only(true); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Drop { set_id: SetId(0), peer_id: regular1 })); @@ -1358,7 +1423,8 @@ mod tests { }; let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); - let peer_store = MockPeerStoreHandle::new(); + let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); let (_handle, mut controller) = ProtocolController::new(SetId(0), config, tx, Box::new(peer_store)); @@ -1391,6 +1457,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().times(2).return_const(false); let (_handle, mut controller) = @@ -1399,8 +1466,9 @@ mod tests { // Initiate connections. controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); @@ -1412,7 +1480,10 @@ mod tests { // Remove reserved node controller.on_remove_reserved_peer(reserved1); - assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: reserved1 }); + assert_eq!( + rx.try_recv().unwrap().take(), + Message::Drop { set_id: SetId(0), peer_id: reserved1 } + ); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert_eq!(controller.reserved_nodes.len(), 1); assert!(controller.reserved_nodes.contains_key(&reserved2)); @@ -1434,6 +1505,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); @@ -1444,8 +1516,9 @@ mod tests { controller.on_incoming_connection(peer1, IncomingIndex(1)); controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); @@ -1480,6 +1553,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1490,8 +1564,9 @@ mod tests { controller.alloc_slots(); controller.on_incoming_connection(peer2, IncomingIndex(1)); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); @@ -1522,6 +1597,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1532,8 +1608,9 @@ mod tests { controller.alloc_slots(); controller.on_incoming_connection(peer2, IncomingIndex(1)); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); @@ -1545,7 +1622,10 @@ mod tests { assert_eq!(controller.num_out, 1); controller.on_disconnect_peer(peer1); - assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: peer1 }); + assert_eq!( + rx.try_recv().unwrap().take(), + Message::Drop { set_id: SetId(0), peer_id: peer1 } + ); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert_eq!(controller.nodes.len(), 1); assert!(!controller.nodes.contains_key(&peer1)); @@ -1553,7 +1633,10 @@ mod tests { assert_eq!(controller.num_out, 0); controller.on_disconnect_peer(peer2); - assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: peer2 }); + assert_eq!( + rx.try_recv().unwrap().take(), + Message::Drop { set_id: SetId(0), peer_id: peer2 } + ); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert_eq!(controller.nodes.len(), 0); assert_eq!(controller.num_in, 0); @@ -1575,6 +1658,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); @@ -1585,8 +1669,9 @@ mod tests { controller.on_incoming_connection(reserved1, IncomingIndex(1)); controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); @@ -1631,6 +1716,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); peer_store.expect_report_disconnect().times(2).return_const(()); @@ -1642,8 +1728,9 @@ mod tests { controller.alloc_slots(); controller.on_incoming_connection(peer2, IncomingIndex(1)); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); @@ -1683,6 +1770,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().times(2).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); @@ -1693,8 +1781,9 @@ mod tests { controller.on_incoming_connection(reserved1, IncomingIndex(1)); controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(message) = rx.try_recv().ok() { + while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { messages.push(message); + let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); @@ -1710,7 +1799,7 @@ mod tests { // Incoming request for `reserved1`. controller.on_incoming_connection(reserved1, IncomingIndex(2)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(2))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!( controller.reserved_nodes.get(&reserved1), @@ -1719,7 +1808,7 @@ mod tests { // Incoming request for `reserved2`. controller.on_incoming_connection(reserved2, IncomingIndex(3)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(3))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(3))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!( controller.reserved_nodes.get(&reserved2), @@ -1743,6 +1832,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().times(3).return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1754,7 +1844,7 @@ mod tests { // Connect `regular1` as outbound. controller.alloc_slots(); assert_eq!( - rx.try_recv().ok().unwrap(), + rx.try_recv().ok().unwrap().take(), Message::Connect { set_id: SetId(0), peer_id: regular1 } ); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); @@ -1762,19 +1852,19 @@ mod tests { // Connect `regular2` as inbound. controller.on_incoming_connection(regular2, IncomingIndex(0)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(0))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(0))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); // Incoming request for `regular1`. controller.on_incoming_connection(regular1, IncomingIndex(1)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(1))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular1).unwrap(), Direction::Inbound,)); // Incoming request for `regular2`. controller.on_incoming_connection(regular2, IncomingIndex(2)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(2))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); } @@ -1795,6 +1885,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().once().return_const(false); peer_store.expect_is_banned().times(2).return_const(true); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1807,7 +1898,7 @@ mod tests { // Connect `regular1` as outbound. controller.alloc_slots(); assert_eq!( - rx.try_recv().ok().unwrap(), + rx.try_recv().ok().unwrap().take(), Message::Connect { set_id: SetId(0), peer_id: regular1 } ); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); @@ -1815,19 +1906,19 @@ mod tests { // Connect `regular2` as inbound. controller.on_incoming_connection(regular2, IncomingIndex(0)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(0))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(0))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); // Incoming request for `regular1`. controller.on_incoming_connection(regular1, IncomingIndex(1)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Reject(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(!controller.nodes.contains_key(®ular1)); // Incoming request for `regular2`. controller.on_incoming_connection(regular2, IncomingIndex(2)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(2))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Reject(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(!controller.nodes.contains_key(®ular2)); } @@ -1848,6 +1939,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().once().return_const(false); peer_store.expect_outgoing_candidates().once().return_const(outgoing_candidates); @@ -1859,7 +1951,7 @@ mod tests { // Connect `regular1` as outbound. controller.alloc_slots(); assert_eq!( - rx.try_recv().ok().unwrap(), + rx.try_recv().ok().unwrap().take(), Message::Connect { set_id: SetId(0), peer_id: regular1 } ); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); @@ -1867,7 +1959,7 @@ mod tests { // Connect `regular2` as inbound. controller.on_incoming_connection(regular2, IncomingIndex(0)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(0))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(0))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); @@ -1875,13 +1967,13 @@ mod tests { // Incoming request for `regular1`. controller.on_incoming_connection(regular1, IncomingIndex(1)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Reject(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(!controller.nodes.contains_key(®ular1)); // Incoming request for `regular2`. controller.on_incoming_connection(regular2, IncomingIndex(2)); - assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(2))); + assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Reject(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(!controller.nodes.contains_key(®ular2)); } @@ -1901,6 +1993,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().once().return_const(false); let (_handle, mut controller) = @@ -1908,12 +2001,12 @@ mod tests { // Connect `peer1` as inbound. controller.on_incoming_connection(peer1, IncomingIndex(1)); - assert_eq!(rx.try_recv().unwrap(), Message::Accept(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap().take(), Message::Accept(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Incoming requests for `peer2`. controller.on_incoming_connection(peer2, IncomingIndex(2)); - assert_eq!(rx.try_recv().unwrap(), Message::Reject(IncomingIndex(2))); + assert_eq!(rx.try_recv().unwrap().take(), Message::Reject(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); } @@ -1931,6 +2024,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().once().return_const(true); let (_handle, mut controller) = @@ -1938,7 +2032,7 @@ mod tests { // Incoming request. controller.on_incoming_connection(peer1, IncomingIndex(1)); - assert_eq!(rx.try_recv().unwrap(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap().take(), Message::Reject(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); } @@ -1956,6 +2050,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().once().return_const(true); let (_handle, mut controller) = @@ -1964,7 +2059,7 @@ mod tests { // Incoming request. controller.on_incoming_connection(reserved1, IncomingIndex(1)); - assert_eq!(rx.try_recv().unwrap(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap().take(), Message::Reject(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); } @@ -1982,6 +2077,7 @@ mod tests { let (tx, mut rx) = tracing_unbounded("mpsc_test_to_notifications", 100); let mut peer_store = MockPeerStoreHandle::new(); + peer_store.expect_register_protocol().once().return_const(()); peer_store.expect_is_banned().once().return_const(true); peer_store.expect_outgoing_candidates().once().return_const(Vec::new()); From 76d41325c9edcf89ed9f9acdd5b9e413a57eb372 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 12 May 2023 16:56:05 +0300 Subject: [PATCH 83/98] WIP: ACKs in `ProtocolController` (mostly done, but fuzz fails) --- client/peerset/src/protocol_controller.rs | 133 ++++++++++++++++------ client/peerset/tests/fuzz.rs | 44 ++++--- 2 files changed, 119 insertions(+), 58 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 0a0e1aa26d2cf..bbadc1416bfc2 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -310,27 +310,19 @@ impl ProtocolController { /// /// Intended for tests only. Use `run` for driving [`ProtocolController`]. pub async fn next_action(&mut self) -> bool { - // Perform tasks prioritizing connection events processing - // (see the module documentation for details). - let mut emptied_outstanding = None; - let either = 'outer: loop { - // Receive ACKs from `Notifications` and process outstanding events & actions. - for (peer_id, outstanding) in self.outstanding.iter_mut() { - if outstanding.try_resolve_ack() { - if let Some(event) = outstanding.events.pop_front() { - emptied_outstanding = outstanding.is_empty().then_some(*peer_id); - break 'outer Either::Left(event) - } - if let Some(action) = outstanding.actions.pop_front() { - emptied_outstanding = outstanding.is_empty().then_some(*peer_id); - break 'outer Either::Right(action) - } + let either = loop { + log::debug!(target: LOG_TARGET, "Outstanding: {:?}", self.outstanding); + + // Resolve ACKs & process outstanding events/actions. We might want to do this after + // `alloc_slots` + if let Some(either) = self.resolve_acks_once() { + match either { + Either::Left(event) => self.process_event(event), + Either::Right(action) => self.process_action(action), } + return true } - // If we have received some ACKs, we might be able to remove obsolete entries. - self.outstanding.retain(|_, outstanding| !outstanding.is_empty()); - // Apply postponed `SetReservedOnly` action // TODO: we might need to do it after all events processing. if self.outstanding.is_empty() { @@ -359,32 +351,70 @@ impl ProtocolController { } }; - // Remove outstanding entry that became empty. - emptied_outstanding.map(|peer_id| self.outstanding.remove(&peer_id)); - match either { - Either::Left(event) => self.process_event(event), - Either::Right(action) => self.process_action(action), + Either::Left(event) => self.push_event(event), + Either::Right(action) => self.push_action(action), } true } - /// Process connection event. - fn process_event(&mut self, event: Event) { + /// Receive ACKs from `Notifications` and process outstanding events & actions. + /// This function must be called repeatedly intil it yields `None`. + fn resolve_acks_once(&mut self) -> Option> { + let mut result = None; + let mut emptied_entries = Vec::new(); + + for (peer_id, outstanding) in self.outstanding.iter_mut() { + if outstanding.try_resolve_ack() { + if let Some(event) = outstanding.events.pop_front() { + if outstanding.is_empty() { + emptied_entries.push(*peer_id); + } + result = Some(Either::Left(event)); + break + } + + if let Some(action) = outstanding.actions.pop_front() { + if outstanding.is_empty() { + emptied_entries.push(*peer_id); + } + result = Some(Either::Right(action)); + break + } + + emptied_entries.push(*peer_id); + } + } + + // Remove outstanding entries that became empty. + emptied_entries.iter().for_each(|peer_id| { + self.outstanding.remove(&peer_id); + }); + + result + } + + /// Push connection event for processing + fn push_event(&mut self, event: Event) { if let Some(outstanding) = self.outstanding.get_mut(event.peer_id()) { outstanding.events.push_back(event); } else { - match event { - Event::IncomingConnection(peer_id, index) => - self.on_incoming_connection(peer_id, index), - Event::Dropped(peer_id) => self.on_peer_dropped(peer_id), - } + self.process_event(event); } } - /// Process action command. - fn process_action(&mut self, action: Action) { + /// Process connection event. + fn process_event(&mut self, event: Event) { + match event { + Event::IncomingConnection(peer_id, index) => + self.on_incoming_connection(peer_id, index), + Event::Dropped(peer_id) => self.on_peer_dropped(peer_id), + } + } + + /// Push action command for processing + fn push_action(&mut self, action: Action) { if let Some(peer_id) = action.peer_id() { if let Some(outstanding) = self.outstanding.get_mut(peer_id) { outstanding.actions.push_back(action); @@ -392,6 +422,11 @@ impl ProtocolController { } } + self.process_action(action); + } + + /// Process action command. + fn process_action(&mut self, action: Action) { match action { Action::AddReservedPeer(peer_id) => self.on_add_reserved_peer(peer_id), Action::RemoveReservedPeer(peer_id) => self.on_remove_reserved_peer(peer_id), @@ -438,9 +473,10 @@ impl ProtocolController { trace!(target: LOG_TARGET, "Connecting to {peer_id}."); let tx_ack = self.insert_outstanding_ack(peer_id); - let _ = self - .to_notifications - .unbounded_send(AckedMessage(Message::Connect { set_id: self.set_id, peer_id }, tx_ack)); + let _ = self.to_notifications.unbounded_send(AckedMessage( + Message::Connect { set_id: self.set_id, peer_id }, + tx_ack, + )); } /// Send "drop" message to `Notifications`. @@ -928,6 +964,9 @@ mod tests { assert_eq!(controller.num_out, 0); assert_eq!(controller.num_in, 0); + // Resolve ACKs before manually injecting more events for the same peers. + assert!(controller.resolve_acks_once().is_none()); + // Drop connections to be able to accept reserved nodes. controller.on_peer_dropped(reserved1); controller.on_peer_dropped(reserved2); @@ -1040,6 +1079,9 @@ mod tests { assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); + // Resolve ACKs before manually injecting more events for the same peers. + assert!(controller.resolve_acks_once().is_none()); + // Drop both reserved nodes. controller.on_peer_dropped(reserved1); controller.on_peer_dropped(reserved2); @@ -1393,6 +1435,9 @@ mod tests { assert_eq!(controller.num_out, 1); assert_eq!(controller.num_in, 1); + // Resolve ACKs before manually injecting more events for the same peers. + assert!(controller.resolve_acks_once().is_none()); + // Switch to reserved-only mode. controller.on_set_reserved_only(true); @@ -1478,6 +1523,9 @@ mod tests { assert!(controller.reserved_nodes.contains_key(&reserved2)); assert!(controller.nodes.is_empty()); + // Resolve ACKs before manually injecting more events for the same peers. + assert!(controller.resolve_acks_once().is_none()); + // Remove reserved node controller.on_remove_reserved_peer(reserved1); assert_eq!( @@ -1621,6 +1669,9 @@ mod tests { assert_eq!(controller.num_in, 1); assert_eq!(controller.num_out, 1); + // Resolve ACKs before manually injecting more events for the same peers. + assert!(controller.resolve_acks_once().is_none()); + controller.on_disconnect_peer(peer1); assert_eq!( rx.try_recv().unwrap().take(), @@ -1797,6 +1848,9 @@ mod tests { Some(PeerState::Connected(Direction::Outbound)) )); + // Resolve ACKs before manually injecting more events for the same peers. + assert!(controller.resolve_acks_once().is_none()); + // Incoming request for `reserved1`. controller.on_incoming_connection(reserved1, IncomingIndex(2)); assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(2))); @@ -1856,6 +1910,9 @@ mod tests { assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); + // Resolve ACKs before manually injecting more events for the same peers. + assert!(controller.resolve_acks_once().is_none()); + // Incoming request for `regular1`. controller.on_incoming_connection(regular1, IncomingIndex(1)); assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(1))); @@ -1910,6 +1967,9 @@ mod tests { assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); + // Resolve ACKs before manually injecting more events for the same peers. + assert!(controller.resolve_acks_once().is_none()); + // Incoming request for `regular1`. controller.on_incoming_connection(regular1, IncomingIndex(1)); assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Reject(IncomingIndex(1))); @@ -1965,6 +2025,9 @@ mod tests { controller.max_in = 0; + // Resolve ACKs before manually injecting more events for the same peers. + assert!(controller.resolve_acks_once().is_none()); + // Incoming request for `regular1`. controller.on_incoming_connection(regular1, IncomingIndex(1)); assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Reject(IncomingIndex(1))); diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index 44dd5ba677b00..1215b0b6d59f9 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -127,25 +127,23 @@ fn test_once() { let (mut peerset, peerset_handle) = Peerset::from_config(PeersetConfig { sets: vec![SetConfig { - // bootnodes: (0..Uniform::new_inclusive(0, 4).sample(&mut rng)) - // .map(|_| { - // let id = PeerId::random(); - // known_nodes.insert(id, State::Disconnected); - // id - // }) - // .collect(), - // reserved_nodes: { - // (0..Uniform::new_inclusive(0, 2).sample(&mut rng)) - // .map(|_| { - // let id = PeerId::random(); - // known_nodes.insert(id, State::Disconnected); - // reserved_nodes.insert(id); - // id - // }) - // .collect() - // }, - bootnodes: Vec::new(), - reserved_nodes: HashSet::new(), + bootnodes: (0..Uniform::new_inclusive(0, 4).sample(&mut rng)) + .map(|_| { + let id = PeerId::random(); + known_nodes.insert(id, State::Disconnected); + id + }) + .collect(), + reserved_nodes: { + (0..Uniform::new_inclusive(0, 2).sample(&mut rng)) + .map(|_| { + let id = PeerId::random(); + known_nodes.insert(id, State::Disconnected); + reserved_nodes.insert(id); + id + }) + .collect() + }, in_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), out_peers: Uniform::new_inclusive(0, 25).sample(&mut rng), reserved_only: Uniform::new_inclusive(0, 10).sample(&mut rng) == 0, @@ -168,7 +166,7 @@ fn test_once() { // Perform a certain number of actions while checking that the state is consistent. If we // reach the end of the loop, the run has succeeded. - for _ in 0..2500 { + for _ in 0..25000 { // Peer we are working with. let mut current_peer = None; // Current event for event bigrams validation. @@ -291,9 +289,9 @@ fn test_once() { // If we generate 1, discover a new node. 1 => { - // let new_id = PeerId::random(); - // known_nodes.insert(new_id, State::Disconnected); - // peerset_handle.add_known_peer(new_id); + let new_id = PeerId::random(); + known_nodes.insert(new_id, State::Disconnected); + peerset_handle.add_known_peer(new_id); }, // If we generate 2, adjust a random reputation. From 406699597059f566c67b9edf190210254bd5944f Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 12 May 2023 16:58:52 +0300 Subject: [PATCH 84/98] minor: remove debugging logging --- client/peerset/src/protocol_controller.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index bbadc1416bfc2..91de8c93add21 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -311,8 +311,6 @@ impl ProtocolController { /// Intended for tests only. Use `run` for driving [`ProtocolController`]. pub async fn next_action(&mut self) -> bool { let either = loop { - log::debug!(target: LOG_TARGET, "Outstanding: {:?}", self.outstanding); - // Resolve ACKs & process outstanding events/actions. We might want to do this after // `alloc_slots` if let Some(either) = self.resolve_acks_once() { From 1fc523d959c532939009e94d6686c4fec056272e Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 12 May 2023 17:29:05 +0300 Subject: [PATCH 85/98] Fix failed merge of `master` --- client/network/src/protocol/notifications/behaviour.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 7214144c314d0..90b22761016c3 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -697,7 +697,7 @@ impl Notifications { st @ PeerState::Incoming { .. } => { info!( target: "sub-libp2p", - "PSM => Connect({}, {:?}): Ignoring obsolete connect, we are awaiting for accept/reject.", + "PSM => Connect({}, {:?}): Ignoring obsolete connect, we are awaiting accept/reject.", occ_entry.key().0, set_id ); *occ_entry.into_mut() = st; @@ -828,7 +828,7 @@ impl Notifications { st @ PeerState::Incoming { .. } => { info!( target: "sub-libp2p", - "PSM => Drop({}, {:?}): Ignoring obsolete disconnect, we are awaiting for accept/reject.", + "PSM => Drop({}, {:?}): Ignoring obsolete disconnect, we are awaiting accept/reject.", entry.key().0, set_id, ); *entry.into_mut() = st; @@ -4014,7 +4014,7 @@ mod tests { let (mut notif, _peerset) = development_notifs(); let peer = PeerId::random(); let set_id = sc_peerset::SetId::from(0); - let conn = ConnectionId::new(0usize); + let conn = ConnectionId::new_unchecked(0); let connected = ConnectedPoint::Listener { local_addr: Multiaddr::empty(), send_back_addr: Multiaddr::empty(), From ccdde89d5e7a1cca1c2dbb10ea7c55d1e7559435 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Fri, 12 May 2023 17:32:02 +0300 Subject: [PATCH 86/98] minor: revert number of iterations in `fuzz` test --- client/peerset/tests/fuzz.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index 632dca44990f7..0cdf9e9ea5f48 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -166,7 +166,7 @@ fn test_once() { // Perform a certain number of actions while checking that the state is consistent. If we // reach the end of the loop, the run has succeeded. - for _ in 0..25000 { + for _ in 0..2500 { // Peer we are working with. let mut current_peer = None; // Current event for event bigrams validation. From 2415e2ac9cd33cfd3508389d578a91cc315d20a6 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 15 May 2023 15:27:27 +0300 Subject: [PATCH 87/98] Relax event order requirements in fuzz test --- client/peerset/src/protocol_controller.rs | 29 +++++++++++++++++++++++ client/peerset/tests/fuzz.rs | 19 +++++++++++---- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index eea57b32dccb4..65a53840f5379 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -57,6 +57,10 @@ use crate::{ LOG_TARGET, }; +/// Queueing that many outstanding events means that `Notifications` and `ProtocolController` +/// are significantly out of sync. +const OUTSTANDING_QUEUE_WARNING: usize = 10; + /// External API actions. #[derive(Debug)] enum Action { @@ -396,6 +400,18 @@ impl ProtocolController { /// Push connection event for processing fn push_event(&mut self, event: Event) { if let Some(outstanding) = self.outstanding.get_mut(event.peer_id()) { + if outstanding.events.len() == OUTSTANDING_QUEUE_WARNING { + // The following warning means that either `ProtocolController` is not fast enough + // to process network events, or `Notifications` is not fast enough to ACK connection + // messages. + warn!( + target: LOG_TARGET, + "Number of oustanding connection events for peer {} exceeded {}.", + event.peer_id(), + OUTSTANDING_QUEUE_WARNING, + ); + } + outstanding.events.push_back(event); } else { self.process_event(event); @@ -415,7 +431,20 @@ impl ProtocolController { fn push_action(&mut self, action: Action) { if let Some(peer_id) = action.peer_id() { if let Some(outstanding) = self.outstanding.get_mut(peer_id) { + if outstanding.actions.len() == OUTSTANDING_QUEUE_WARNING { + // The following warning means that either `ProtocolController` is not fast + // enough to process outer API actions, or `Notifications` is not fast enough + // to ACK connection messages. + warn!( + target: LOG_TARGET, + "Number of oustanding connection actions for peer {} exceeded {}.", + peer_id, + OUTSTANDING_QUEUE_WARNING, + ); + } + outstanding.actions.push_back(action); + return } } diff --git a/client/peerset/tests/fuzz.rs b/client/peerset/tests/fuzz.rs index 0cdf9e9ea5f48..855d2339eda12 100644 --- a/client/peerset/tests/fuzz.rs +++ b/client/peerset/tests/fuzz.rs @@ -107,11 +107,15 @@ fn test_once() { ), ( BareState::Inbound, - [Event::Disconnected, Event::PsmDrop].into_iter().collect::>(), + [Event::Disconnected, Event::PsmDrop, Event::PsmConnect /* must be ignored */] + .into_iter() + .collect::>(), ), ( BareState::Outbound, - [Event::Disconnected, Event::PsmDrop].into_iter().collect::>(), + [Event::Disconnected, Event::PsmDrop, Event::PsmConnect /* must be ignored */] + .into_iter() + .collect::>(), ), ] .into_iter() @@ -166,6 +170,8 @@ fn test_once() { // Perform a certain number of actions while checking that the state is consistent. If we // reach the end of the loop, the run has succeeded. + // Note that with the ACKing and event postponing mechanism in `ProtocolController` + // the test time grows quadratically with the number of iterations below. for _ in 0..2500 { // Peer we are working with. let mut current_peer = None; @@ -192,9 +198,14 @@ fn test_once() { } last_state = Some(*state); - *state = State::Outbound; - assert!(connected_nodes.insert(peer_id)); + if *state != State::Inbound { + *state = State::Outbound; + } + + if !connected_nodes.insert(peer_id) { + log::info!("Request to connect to an already connected node {peer_id}"); + } current_peer = Some(peer_id); current_event = Some(Event::PsmConnect); From 4ee2a3231a59c85564a5f401b5a129e4e1e44a9c Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 15 May 2023 15:34:01 +0300 Subject: [PATCH 88/98] Relax `ProtocolController` message order requirements in `Notifications` --- .../network/src/protocol/notifications/behaviour.rs | 13 +++++-------- client/peerset/src/protocol_controller.rs | 4 ++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 90b22761016c3..2168f3e126754 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -35,7 +35,7 @@ use libp2p::{ }, PeerId, }; -use log::{error, info, trace, warn}; +use log::{debug, error, info, trace, warn}; use parking_lot::RwLock; use rand::distributions::{Distribution as _, Uniform}; use sc_peerset::DropReason; @@ -695,7 +695,7 @@ impl Notifications { }, // Incoming => Incoming st @ PeerState::Incoming { .. } => { - info!( + debug!( target: "sub-libp2p", "PSM => Connect({}, {:?}): Ignoring obsolete connect, we are awaiting accept/reject.", occ_entry.key().0, set_id @@ -705,25 +705,22 @@ impl Notifications { // Other states are kept as-is. st @ PeerState::Enabled { .. } => { - warn!(target: "sub-libp2p", + debug!(target: "sub-libp2p", "PSM => Connect({}, {:?}): Already connected.", occ_entry.key().0, set_id); *occ_entry.into_mut() = st; - debug_assert!(false); }, st @ PeerState::DisabledPendingEnable { .. } => { - warn!(target: "sub-libp2p", + debug!(target: "sub-libp2p", "PSM => Connect({}, {:?}): Already pending enabling.", occ_entry.key().0, set_id); *occ_entry.into_mut() = st; - debug_assert!(false); }, st @ PeerState::Requested { .. } | st @ PeerState::PendingRequest { .. } => { - warn!(target: "sub-libp2p", + debug!(target: "sub-libp2p", "PSM => Connect({}, {:?}): Duplicate request.", occ_entry.key().0, set_id); *occ_entry.into_mut() = st; - debug_assert!(false); }, PeerState::Poisoned => { diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 65a53840f5379..81b07b0e1142c 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -402,8 +402,8 @@ impl ProtocolController { if let Some(outstanding) = self.outstanding.get_mut(event.peer_id()) { if outstanding.events.len() == OUTSTANDING_QUEUE_WARNING { // The following warning means that either `ProtocolController` is not fast enough - // to process network events, or `Notifications` is not fast enough to ACK connection - // messages. + // to process network events, or `Notifications` is not fast enough to ACK + // connection messages. warn!( target: LOG_TARGET, "Number of oustanding connection events for peer {} exceeded {}.", From 6fcd68cf1513e9d4a5d097ef18ae2748023d966b Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 15 May 2023 17:52:32 +0300 Subject: [PATCH 89/98] Get rid of ACKs --- client/peerset/src/lib.rs | 19 +- client/peerset/src/protocol_controller.rs | 448 ++++------------------ 2 files changed, 86 insertions(+), 381 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 3e39f35d01f1b..336075dc75446 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -202,18 +202,6 @@ pub enum Message { Reject(IncomingIndex), } -/// Message with oneshot ACK channel. -#[derive(Debug)] -pub struct AckedMessage(Message, oneshot::Sender<()>); - -#[cfg(test)] -impl AckedMessage { - fn take(self) -> Message { - let _ = self.1.send(()); - self.0 - } -} - /// Opaque identifier for an incoming connection. Allocated by the network. #[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct IncomingIndex(pub u64); @@ -271,7 +259,7 @@ pub struct Peerset { protocol_controller_futures: JoinAll>, /// Commands sent from protocol controllers to `Notifications`. The size of this vector never /// changes. - from_controllers: TracingUnboundedReceiver, + from_controllers: TracingUnboundedReceiver, /// Receiver for messages from the `PeersetHandle` and from `to_self`. from_handle: TracingUnboundedReceiver, /// Sending side of `from_handle`. @@ -375,10 +363,7 @@ impl Stream for Peerset { fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll> { if let Poll::Ready(msg) = self.from_controllers.poll_next_unpin(cx) { - if let Some(AckedMessage(msg, tx_ack)) = msg { - // `Peerset` is polled directly by `Notifications`, so we can already send ACK here. - // Once `Peerset` is eliminated, ACKs must be sent by `Notifications`. - let _ = tx_ack.send(()); + if let Some(msg) = msg { return Poll::Ready(Some(msg)) } else { debug!( diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 81b07b0e1142c..5549e96d84ef8 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -47,19 +47,12 @@ use log::{error, trace, warn}; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedReceiver, TracingUnboundedSender}; use sp_arithmetic::traits::SaturatedConversion; use std::{ - collections::{HashMap, HashSet, VecDeque}, + collections::{HashMap, HashSet}, time::{Duration, Instant}, }; use wasm_timer::Delay; -use crate::{ - peer_store::PeerStoreProvider, AckedMessage, IncomingIndex, Message, SetConfig, SetId, - LOG_TARGET, -}; - -/// Queueing that many outstanding events means that `Notifications` and `ProtocolController` -/// are significantly out of sync. -const OUTSTANDING_QUEUE_WARNING: usize = 10; +use crate::{peer_store::PeerStoreProvider, IncomingIndex, Message, SetConfig, SetId, LOG_TARGET}; /// External API actions. #[derive(Debug)] @@ -87,28 +80,6 @@ enum Event { Dropped(PeerId), } -impl Action { - fn peer_id(&self) -> Option<&PeerId> { - match self { - Self::AddReservedPeer(peer_id) => Some(peer_id), - Self::RemoveReservedPeer(peer_id) => Some(peer_id), - Self::SetReservedPeers(_) => None, - Self::SetReservedOnly(_) => None, - Self::DisconnectPeer(peer_id) => Some(peer_id), - Self::GetReservedPeers(_) => None, - } - } -} - -impl Event { - fn peer_id(&self) -> &PeerId { - match self { - Self::IncomingConnection(peer_id, _) => peer_id, - Self::Dropped(peer_id) => peer_id, - } - } -} - /// Shared handle to [`ProtocolController`]. Distributed around the code outside of the /// protocol implementation. #[derive(Debug, Clone)] @@ -202,34 +173,6 @@ impl Default for PeerState { } } -/// Outstanding events and actions for a peer that we can't process because we are waiting -/// for an ACK. -#[derive(Debug, Default)] -struct Outstanding { - pending_ack: Option>, - events: VecDeque, - actions: VecDeque, -} - -impl Outstanding { - fn try_resolve_ack(&mut self) -> bool { - match self.pending_ack { - Some(ref mut pending_ack) => - if pending_ack.try_recv().ok().flatten().is_some() { - self.pending_ack = None; - true - } else { - false - }, - None => true, - } - } - - fn is_empty(&self) -> bool { - self.pending_ack.is_none() && self.events.is_empty() && self.actions.is_empty() - } -} - /// Side of [`ProtocolHandle`] responsible for all the logic. Currently all instances are /// owned by [`crate::Peerset`], but they should eventually be moved to corresponding protocols. #[derive(Debug)] @@ -258,16 +201,10 @@ pub struct ProtocolController { /// Next time to allocate slots. This is done once per second. next_periodic_alloc_slots: Instant, /// Outgoing channel for messages to `Notifications`. - to_notifications: TracingUnboundedSender, + to_notifications: TracingUnboundedSender, /// `PeerStore` handle for checking peer reputation values and getting connection candidates /// with highest reputation. peer_store: Box, - /// Pending ACKS - /// Outstanding events and actions per peer. - outstanding: HashMap, - /// If we have outstanding (global) `SetReservedOnly` action. It's processed once we receive - /// all pending ACKs. - outstanding_set_reserved_only: Option, } impl ProtocolController { @@ -275,7 +212,7 @@ impl ProtocolController { pub fn new( set_id: SetId, config: SetConfig, - to_notifications: TracingUnboundedSender, + to_notifications: TracingUnboundedSender, peer_store: Box, ) -> (ProtocolHandle, ProtocolController) { let (actions_tx, actions_rx) = tracing_unbounded("mpsc_api_protocol", 10_000); @@ -298,8 +235,6 @@ impl ProtocolController { next_periodic_alloc_slots: Instant::now(), to_notifications, peer_store, - outstanding: HashMap::new(), - outstanding_set_reserved_only: None, }; (handle, controller) } @@ -315,25 +250,6 @@ impl ProtocolController { /// Intended for tests only. Use `run` for driving [`ProtocolController`]. pub async fn next_action(&mut self) -> bool { let either = loop { - // Resolve ACKs & process outstanding events/actions. We might want to do this after - // `alloc_slots` - if let Some(either) = self.resolve_acks_once() { - match either { - Either::Left(event) => self.process_event(event), - Either::Right(action) => self.process_action(action), - } - return true - } - - // Apply postponed `SetReservedOnly` action - // TODO: we might need to do it after all events processing. - if self.outstanding.is_empty() { - if let Some(reserved_only) = self.outstanding_set_reserved_only.take() { - self.on_set_reserved_only(reserved_only); - return true - } - } - let mut next_alloc_slots = Delay::new_at(self.next_periodic_alloc_slots).fuse(); // See the module doc for why we use `select_biased!`. @@ -354,70 +270,13 @@ impl ProtocolController { }; match either { - Either::Left(event) => self.push_event(event), - Either::Right(action) => self.push_action(action), + Either::Left(event) => self.process_event(event), + Either::Right(action) => self.process_action(action), } true } - /// Receive ACKs from `Notifications` and process outstanding events & actions. - /// This function must be called repeatedly intil it yields `None`. - fn resolve_acks_once(&mut self) -> Option> { - let mut result = None; - let mut emptied_entries = Vec::new(); - - for (peer_id, outstanding) in self.outstanding.iter_mut() { - if outstanding.try_resolve_ack() { - if let Some(event) = outstanding.events.pop_front() { - if outstanding.is_empty() { - emptied_entries.push(*peer_id); - } - result = Some(Either::Left(event)); - break - } - - if let Some(action) = outstanding.actions.pop_front() { - if outstanding.is_empty() { - emptied_entries.push(*peer_id); - } - result = Some(Either::Right(action)); - break - } - - emptied_entries.push(*peer_id); - } - } - - // Remove outstanding entries that became empty. - emptied_entries.iter().for_each(|peer_id| { - self.outstanding.remove(&peer_id); - }); - - result - } - - /// Push connection event for processing - fn push_event(&mut self, event: Event) { - if let Some(outstanding) = self.outstanding.get_mut(event.peer_id()) { - if outstanding.events.len() == OUTSTANDING_QUEUE_WARNING { - // The following warning means that either `ProtocolController` is not fast enough - // to process network events, or `Notifications` is not fast enough to ACK - // connection messages. - warn!( - target: LOG_TARGET, - "Number of oustanding connection events for peer {} exceeded {}.", - event.peer_id(), - OUTSTANDING_QUEUE_WARNING, - ); - } - - outstanding.events.push_back(event); - } else { - self.process_event(event); - } - } - /// Process connection event. fn process_event(&mut self, event: Event) { match event { @@ -427,48 +286,13 @@ impl ProtocolController { } } - /// Push action command for processing - fn push_action(&mut self, action: Action) { - if let Some(peer_id) = action.peer_id() { - if let Some(outstanding) = self.outstanding.get_mut(peer_id) { - if outstanding.actions.len() == OUTSTANDING_QUEUE_WARNING { - // The following warning means that either `ProtocolController` is not fast - // enough to process outer API actions, or `Notifications` is not fast enough - // to ACK connection messages. - warn!( - target: LOG_TARGET, - "Number of oustanding connection actions for peer {} exceeded {}.", - peer_id, - OUTSTANDING_QUEUE_WARNING, - ); - } - - outstanding.actions.push_back(action); - - return - } - } - - self.process_action(action); - } - /// Process action command. fn process_action(&mut self, action: Action) { match action { Action::AddReservedPeer(peer_id) => self.on_add_reserved_peer(peer_id), Action::RemoveReservedPeer(peer_id) => self.on_remove_reserved_peer(peer_id), Action::SetReservedPeers(peer_ids) => self.on_set_reserved_peers(peer_ids), - Action::SetReservedOnly(reserved_only) => - if !self.outstanding.is_empty() { - // we need to postpone the action if there are outstanding events/actions. - if reserved_only == self.reserved_only { - self.outstanding_set_reserved_only = None; - } else { - self.outstanding_set_reserved_only = Some(reserved_only); - } - } else { - self.on_set_reserved_only(reserved_only); - }, + Action::SetReservedOnly(reserved_only) => self.on_set_reserved_only(reserved_only), Action::DisconnectPeer(peer_id) => self.on_disconnect_peer(peer_id), Action::GetReservedPeers(pending_response) => self.on_get_reserved_peers(pending_response), @@ -476,58 +300,35 @@ impl ProtocolController { } /// Send "accept" message to `Notifications`. - fn accept_connection(&mut self, peer_id: PeerId, incoming_index: IncomingIndex) { + fn accept_connection(&mut self, incoming_index: IncomingIndex) { trace!(target: LOG_TARGET, "Accepting {incoming_index:?}."); - let tx_ack = self.insert_outstanding_ack(peer_id); - let _ = self - .to_notifications - .unbounded_send(AckedMessage(Message::Accept(incoming_index), tx_ack)); + let _ = self.to_notifications.unbounded_send(Message::Accept(incoming_index)); } /// Send "reject" message to `Notifications`. - fn reject_connection(&mut self, peer_id: PeerId, incoming_index: IncomingIndex) { + fn reject_connection(&mut self, incoming_index: IncomingIndex) { trace!(target: LOG_TARGET, "Rejecting {incoming_index:?}."); - let tx_ack = self.insert_outstanding_ack(peer_id); - let _ = self - .to_notifications - .unbounded_send(AckedMessage(Message::Reject(incoming_index), tx_ack)); + let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); } /// Send "connect" message to `Notifications`. fn start_connection(&mut self, peer_id: PeerId) { trace!(target: LOG_TARGET, "Connecting to {peer_id}."); - let tx_ack = self.insert_outstanding_ack(peer_id); - let _ = self.to_notifications.unbounded_send(AckedMessage( - Message::Connect { set_id: self.set_id, peer_id }, - tx_ack, - )); + let _ = self + .to_notifications + .unbounded_send(Message::Connect { set_id: self.set_id, peer_id }); } /// Send "drop" message to `Notifications`. fn drop_connection(&mut self, peer_id: PeerId) { trace!(target: LOG_TARGET, "Dropping {peer_id}."); - let tx_ack = self.insert_outstanding_ack(peer_id); let _ = self .to_notifications - .unbounded_send(AckedMessage(Message::Drop { set_id: self.set_id, peer_id }, tx_ack)); - } - - /// Insert outstanding entry and/or initialize oneshot channel for ACK. - fn insert_outstanding_ack(&mut self, peer_id: PeerId) -> oneshot::Sender<()> { - let outstanding = self.outstanding.entry(peer_id).or_default(); - debug_assert!( - outstanding.pending_ack.is_none(), - "Discarding previous pending ACK channel before it was received." - ); - - let (tx, rx) = oneshot::channel(); - outstanding.pending_ack = Some(rx); - - tx + .unbounded_send(Message::Drop { set_id: self.set_id, peer_id }); } /// Report peer disconnect event to `PeerStore` for it to update peer's reputation accordingly. @@ -627,30 +428,12 @@ impl ProtocolController { let to_insert = peer_ids.difference(¤t).cloned().collect::>(); let to_remove = current.difference(&peer_ids).cloned().collect::>(); - if self.outstanding.is_empty() { - // Process peers immediately if we have no oustanding events/actions. - for peer_id in to_insert { - self.on_add_reserved_peer(peer_id); - } - for peer_id in to_remove { - self.on_remove_reserved_peer(peer_id); - } - } else { - // Push actions to outstanding queues if there are outstanding events/actions. - for peer_id in to_insert { - self.outstanding - .entry(peer_id) - .or_default() - .actions - .push_back(Action::AddReservedPeer(peer_id)); - } - for peer_id in to_remove { - self.outstanding - .entry(peer_id) - .or_default() - .actions - .push_back(Action::RemoveReservedPeer(peer_id)); - } + for node in to_insert { + self.on_add_reserved_peer(node); + } + + for node in to_remove { + self.on_remove_reserved_peer(node); } } @@ -726,7 +509,7 @@ impl ProtocolController { trace!(target: LOG_TARGET, "Incoming connection from peer {peer_id} ({incoming_index:?}).",); if self.reserved_only && !self.reserved_nodes.contains_key(&peer_id) { - self.reject_connection(peer_id, incoming_index); + self.reject_connection(incoming_index); return } @@ -737,15 +520,15 @@ impl ProtocolController { // We are accepting an incoming connection, so ensure the direction is inbound. // (See the note above.) *direction = Direction::Inbound; - self.accept_connection(peer_id, incoming_index); + self.accept_connection(incoming_index); }, PeerState::NotConnected => { // FIXME: unable to call `self.is_banned()` because of borrowed `self`. if self.peer_store.is_banned(&peer_id) { - self.reject_connection(peer_id, incoming_index); + self.reject_connection(incoming_index); } else { *state = PeerState::Connected(Direction::Inbound); - self.accept_connection(peer_id, incoming_index); + self.accept_connection(incoming_index); } }, } @@ -768,18 +551,18 @@ impl ProtocolController { } if self.num_in >= self.max_in { - self.reject_connection(peer_id, incoming_index); + self.reject_connection(incoming_index); return } if self.is_banned(&peer_id) { - self.reject_connection(peer_id, incoming_index); + self.reject_connection(incoming_index); return } self.num_in += 1; self.nodes.insert(peer_id, Direction::Inbound); - self.accept_connection(peer_id, incoming_index); + self.accept_connection(incoming_index); } /// Indicate that a connection with the peer was dropped. @@ -851,10 +634,7 @@ impl ProtocolController { self.reserved_nodes .iter_mut() .filter_map(|(peer_id, state)| { - (!state.is_connected() && - !self.peer_store.is_banned(peer_id) && - !self.outstanding.contains_key(peer_id)) - .then(|| { + (!state.is_connected() && !self.peer_store.is_banned(peer_id)).then(|| { *state = PeerState::Connected(Direction::Outbound); peer_id }) @@ -882,9 +662,6 @@ impl ProtocolController { .collect::>() .union(&self.nodes.keys().collect::>()) .cloned() - .collect::>() - .union(&self.outstanding.keys().collect::>()) - .cloned() .collect(); let candidates = self @@ -892,18 +669,16 @@ impl ProtocolController { .outgoing_candidates(available_slots, ignored) .into_iter() .filter_map(|peer_id| { - (!self.reserved_nodes.contains_key(&peer_id) && - !self.nodes.contains_key(&peer_id) && - !self.outstanding.contains_key(&peer_id)) - .then_some(peer_id) - .or_else(|| { - error!( - target: LOG_TARGET, - "`PeerStore` returned a node we asked to ignore: {peer_id}.", - ); - debug_assert!(false, "`PeerStore` returned a node we asked to ignore."); - None - }) + (!self.reserved_nodes.contains_key(&peer_id) && !self.nodes.contains_key(&peer_id)) + .then_some(peer_id) + .or_else(|| { + error!( + target: LOG_TARGET, + "`PeerStore` returned a node we asked to ignore: {peer_id}.", + ); + debug_assert!(false, "`PeerStore` returned a node we asked to ignore."); + None + }) }) .collect::>(); @@ -927,8 +702,7 @@ impl ProtocolController { mod tests { use super::{Direction, PeerState, ProtocolController, ProtocolHandle}; use crate::{ - peer_store::PeerStoreProvider, AckedMessage, IncomingIndex, Message, ReputationChange, - SetConfig, SetId, + peer_store::PeerStoreProvider, IncomingIndex, Message, ReputationChange, SetConfig, SetId, }; use libp2p_identity::PeerId; use sc_utils::mpsc::{tracing_unbounded, TryRecvError}; @@ -979,9 +753,8 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); @@ -991,9 +764,6 @@ mod tests { assert_eq!(controller.num_out, 0); assert_eq!(controller.num_in, 0); - // Resolve ACKs before manually injecting more events for the same peers. - assert!(controller.resolve_acks_once().is_none()); - // Drop connections to be able to accept reserved nodes. controller.on_peer_dropped(reserved1); controller.on_peer_dropped(reserved2); @@ -1001,13 +771,13 @@ mod tests { // Incoming connection from `reserved1`. let incoming1 = IncomingIndex(1); controller.on_incoming_connection(reserved1, incoming1); - assert_eq!(rx.try_recv().unwrap().take(), Message::Accept(incoming1)); + assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming1)); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Incoming connection from `reserved2`. let incoming2 = IncomingIndex(2); controller.on_incoming_connection(reserved2, incoming2); - assert_eq!(rx.try_recv().unwrap().take(), Message::Accept(incoming2)); + assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming2)); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Reserved peers do not occupy slots. @@ -1053,13 +823,13 @@ mod tests { // Incoming connection from `reserved1`. let incoming1 = IncomingIndex(1); controller.on_incoming_connection(reserved1, incoming1); - assert_eq!(rx.try_recv().unwrap().take(), Message::Reject(incoming1)); + assert_eq!(rx.try_recv().unwrap(), Message::Reject(incoming1)); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Incoming connection from `reserved2`. let incoming2 = IncomingIndex(2); controller.on_incoming_connection(reserved2, incoming2); - assert_eq!(rx.try_recv().unwrap().take(), Message::Reject(incoming2)); + assert_eq!(rx.try_recv().unwrap(), Message::Reject(incoming2)); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // No slots occupied. @@ -1097,18 +867,14 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved2 })); - // Resolve ACKs before manually injecting more events for the same peers. - assert!(controller.resolve_acks_once().is_none()); - // Drop both reserved nodes. controller.on_peer_dropped(reserved1); controller.on_peer_dropped(reserved2); @@ -1117,9 +883,8 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); @@ -1158,9 +923,8 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } // Only first two peers are connected (we only have 2 slots). @@ -1211,9 +975,8 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 4); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); @@ -1255,9 +1018,8 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } // Only first two peers are connected (we only have 2 slots). @@ -1289,9 +1051,8 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } // Peers are connected. @@ -1353,9 +1114,8 @@ mod tests { controller.on_incoming_connection(peer, incoming_index); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } // Peer is rejected. @@ -1400,9 +1160,8 @@ mod tests { controller.on_set_reserved_only(false); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); @@ -1443,9 +1202,8 @@ mod tests { controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 3); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); @@ -1457,21 +1215,17 @@ mod tests { // Connect `regular2` as inbound. let incoming_index = IncomingIndex(1); controller.on_incoming_connection(regular2, incoming_index); - assert_eq!(rx.try_recv().unwrap().take(), Message::Accept(incoming_index)); + assert_eq!(rx.try_recv().unwrap(), Message::Accept(incoming_index)); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert_eq!(controller.num_out, 1); assert_eq!(controller.num_in, 1); - // Resolve ACKs before manually injecting more events for the same peers. - assert!(controller.resolve_acks_once().is_none()); - // Switch to reserved-only mode. controller.on_set_reserved_only(true); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Drop { set_id: SetId(0), peer_id: regular1 })); @@ -1538,9 +1292,8 @@ mod tests { // Initiate connections. controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: reserved1 })); @@ -1550,15 +1303,9 @@ mod tests { assert!(controller.reserved_nodes.contains_key(&reserved2)); assert!(controller.nodes.is_empty()); - // Resolve ACKs before manually injecting more events for the same peers. - assert!(controller.resolve_acks_once().is_none()); - // Remove reserved node controller.on_remove_reserved_peer(reserved1); - assert_eq!( - rx.try_recv().unwrap().take(), - Message::Drop { set_id: SetId(0), peer_id: reserved1 } - ); + assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: reserved1 }); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert_eq!(controller.reserved_nodes.len(), 1); assert!(controller.reserved_nodes.contains_key(&reserved2)); @@ -1591,9 +1338,8 @@ mod tests { controller.on_incoming_connection(peer1, IncomingIndex(1)); controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); @@ -1639,9 +1385,8 @@ mod tests { controller.alloc_slots(); controller.on_incoming_connection(peer2, IncomingIndex(1)); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); @@ -1683,9 +1428,8 @@ mod tests { controller.alloc_slots(); controller.on_incoming_connection(peer2, IncomingIndex(1)); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); @@ -1696,14 +1440,8 @@ mod tests { assert_eq!(controller.num_in, 1); assert_eq!(controller.num_out, 1); - // Resolve ACKs before manually injecting more events for the same peers. - assert!(controller.resolve_acks_once().is_none()); - controller.on_disconnect_peer(peer1); - assert_eq!( - rx.try_recv().unwrap().take(), - Message::Drop { set_id: SetId(0), peer_id: peer1 } - ); + assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: peer1 }); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert_eq!(controller.nodes.len(), 1); assert!(!controller.nodes.contains_key(&peer1)); @@ -1711,10 +1449,7 @@ mod tests { assert_eq!(controller.num_out, 0); controller.on_disconnect_peer(peer2); - assert_eq!( - rx.try_recv().unwrap().take(), - Message::Drop { set_id: SetId(0), peer_id: peer2 } - ); + assert_eq!(rx.try_recv().unwrap(), Message::Drop { set_id: SetId(0), peer_id: peer2 }); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert_eq!(controller.nodes.len(), 0); assert_eq!(controller.num_in, 0); @@ -1747,9 +1482,8 @@ mod tests { controller.on_incoming_connection(reserved1, IncomingIndex(1)); controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); @@ -1806,9 +1540,8 @@ mod tests { controller.alloc_slots(); controller.on_incoming_connection(peer2, IncomingIndex(1)); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Connect { set_id: SetId(0), peer_id: peer1 })); @@ -1859,9 +1592,8 @@ mod tests { controller.on_incoming_connection(reserved1, IncomingIndex(1)); controller.alloc_slots(); let mut messages = Vec::new(); - while let Some(AckedMessage(message, tx_ack)) = rx.try_recv().ok() { + while let Some(message) = rx.try_recv().ok() { messages.push(message); - let _ = tx_ack.send(()); } assert_eq!(messages.len(), 2); assert!(messages.contains(&Message::Accept(IncomingIndex(1)))); @@ -1875,12 +1607,9 @@ mod tests { Some(PeerState::Connected(Direction::Outbound)) )); - // Resolve ACKs before manually injecting more events for the same peers. - assert!(controller.resolve_acks_once().is_none()); - // Incoming request for `reserved1`. controller.on_incoming_connection(reserved1, IncomingIndex(2)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(2))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!( controller.reserved_nodes.get(&reserved1), @@ -1889,7 +1618,7 @@ mod tests { // Incoming request for `reserved2`. controller.on_incoming_connection(reserved2, IncomingIndex(3)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(3))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(3))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!( controller.reserved_nodes.get(&reserved2), @@ -1925,7 +1654,7 @@ mod tests { // Connect `regular1` as outbound. controller.alloc_slots(); assert_eq!( - rx.try_recv().ok().unwrap().take(), + rx.try_recv().ok().unwrap(), Message::Connect { set_id: SetId(0), peer_id: regular1 } ); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); @@ -1933,22 +1662,19 @@ mod tests { // Connect `regular2` as inbound. controller.on_incoming_connection(regular2, IncomingIndex(0)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(0))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(0))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); - // Resolve ACKs before manually injecting more events for the same peers. - assert!(controller.resolve_acks_once().is_none()); - // Incoming request for `regular1`. controller.on_incoming_connection(regular1, IncomingIndex(1)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(1))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular1).unwrap(), Direction::Inbound,)); // Incoming request for `regular2`. controller.on_incoming_connection(regular2, IncomingIndex(2)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(2))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); } @@ -1982,7 +1708,7 @@ mod tests { // Connect `regular1` as outbound. controller.alloc_slots(); assert_eq!( - rx.try_recv().ok().unwrap().take(), + rx.try_recv().ok().unwrap(), Message::Connect { set_id: SetId(0), peer_id: regular1 } ); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); @@ -1990,22 +1716,19 @@ mod tests { // Connect `regular2` as inbound. controller.on_incoming_connection(regular2, IncomingIndex(0)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(0))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(0))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); - // Resolve ACKs before manually injecting more events for the same peers. - assert!(controller.resolve_acks_once().is_none()); - // Incoming request for `regular1`. controller.on_incoming_connection(regular1, IncomingIndex(1)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(!controller.nodes.contains_key(®ular1)); // Incoming request for `regular2`. controller.on_incoming_connection(regular2, IncomingIndex(2)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Reject(IncomingIndex(2))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(!controller.nodes.contains_key(®ular2)); } @@ -2038,7 +1761,7 @@ mod tests { // Connect `regular1` as outbound. controller.alloc_slots(); assert_eq!( - rx.try_recv().ok().unwrap().take(), + rx.try_recv().ok().unwrap(), Message::Connect { set_id: SetId(0), peer_id: regular1 } ); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); @@ -2046,24 +1769,21 @@ mod tests { // Connect `regular2` as inbound. controller.on_incoming_connection(regular2, IncomingIndex(0)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Accept(IncomingIndex(0))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Accept(IncomingIndex(0))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(matches!(controller.nodes.get(®ular2).unwrap(), Direction::Inbound,)); controller.max_in = 0; - // Resolve ACKs before manually injecting more events for the same peers. - assert!(controller.resolve_acks_once().is_none()); - // Incoming request for `regular1`. controller.on_incoming_connection(regular1, IncomingIndex(1)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(!controller.nodes.contains_key(®ular1)); // Incoming request for `regular2`. controller.on_incoming_connection(regular2, IncomingIndex(2)); - assert_eq!(rx.try_recv().ok().unwrap().take(), Message::Reject(IncomingIndex(2))); + assert_eq!(rx.try_recv().ok().unwrap(), Message::Reject(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); assert!(!controller.nodes.contains_key(®ular2)); } @@ -2091,12 +1811,12 @@ mod tests { // Connect `peer1` as inbound. controller.on_incoming_connection(peer1, IncomingIndex(1)); - assert_eq!(rx.try_recv().unwrap().take(), Message::Accept(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap(), Message::Accept(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); // Incoming requests for `peer2`. controller.on_incoming_connection(peer2, IncomingIndex(2)); - assert_eq!(rx.try_recv().unwrap().take(), Message::Reject(IncomingIndex(2))); + assert_eq!(rx.try_recv().unwrap(), Message::Reject(IncomingIndex(2))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); } @@ -2122,7 +1842,7 @@ mod tests { // Incoming request. controller.on_incoming_connection(peer1, IncomingIndex(1)); - assert_eq!(rx.try_recv().unwrap().take(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap(), Message::Reject(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); } @@ -2149,7 +1869,7 @@ mod tests { // Incoming request. controller.on_incoming_connection(reserved1, IncomingIndex(1)); - assert_eq!(rx.try_recv().unwrap().take(), Message::Reject(IncomingIndex(1))); + assert_eq!(rx.try_recv().unwrap(), Message::Reject(IncomingIndex(1))); assert_eq!(rx.try_recv().unwrap_err(), TryRecvError::Empty); } From 9dfd6fe500975f18b50ace9994baeb2beb1ead73 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 15 May 2023 18:11:13 +0300 Subject: [PATCH 90/98] Update docs to match the implementation --- client/peerset/src/protocol_controller.rs | 29 +++++++++++------------ 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 5549e96d84ef8..5f6917da9bf46 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -22,24 +22,23 @@ //! update peer reputation values and sends commands to `Notifications`. //! //! Due to asynchronous nature of communication between `ProtocolController` and `Notifications`, -//! `ProtocolController` has an imperfect view of the states of the peers. This can lead to sending -//! confusing commands to `Notifications`. To mitigate this issue, the following measures are taken: +//! `ProtocolController` has an imperfect view of the states of the peers. To reduce this +//! desynchronization, the following measures are taken: //! -//! 1. `Notifications` ignores all commands from `ProtocolController` after sending "incoming" -//! request, except the answer to this "incoming" request. -//! 2. `ProtocolController` does not modify the peers state after a command to `Notifications` -//! was sent, but before ACK for this command was heard back. -//! 3. Network peer events from `Notifictions` are prioritized over actions from external API and +//! 1. Network peer events from `Notifictions` are prioritized over actions from external API and //! internal actions by `ProtocolController` (like slot allocation). +//! 2. `Notifications` ignores all commands from `ProtocolController` after sending "incoming" +//! request until receiving the answer to this "incoming" request. +//! 3. After sending a "connect" message, `ProtocolController` switchs the state of the peer from +//! `Outbound` to `Inbound` if receives an "incoming" request from `Notifications` for this peer. //! -//! Even though the fuzz test from the crate does not reveal any inconsistencies in peer states -//! observed by `Notifications`, measures above do not provide 100% guarantee against confusing -//! commands from `ProtocolController`. This might happen, for example, if a network state event -//! from `Notifications` is processed after an external API action sent at the same time. -//! For this reason, `Notifications` must employ defensive programming and behave in a defined way -//! if seemingly impossible sequences of commands are received. For example, `Notifications` must -//! correctly handle a "connect" command for the peer it thinks is already connected, and a "drop" -//! command for a peer that was previously dropped. +//! These measures do not eliminate confusing commands from `ProtocolController` completely, +//! so `Notifications` must correctly handle seemingly inconsistent commands, like a "connect" +//! command for the peer it thinks is already connected, and a "drop" command for a peer that +//! was previously dropped. +//! +//! Even though this does not guarantee that `ProtocolController` and `Notifications` have the same +//! view of the peers' states at any given moment, the eventual consistency is maintained. use futures::{channel::oneshot, future::Either, FutureExt, StreamExt}; use libp2p_identity::PeerId; From 601e4f67d0050891d4e2f0269e300abaf2fea2ca Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 16 May 2023 11:32:00 +0300 Subject: [PATCH 91/98] Fix `Notifications` tests --- .../src/protocol/notifications/behaviour.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/client/network/src/protocol/notifications/behaviour.rs b/client/network/src/protocol/notifications/behaviour.rs index 2168f3e126754..8f1324dfa8c06 100644 --- a/client/network/src/protocol/notifications/behaviour.rs +++ b/client/network/src/protocol/notifications/behaviour.rs @@ -3861,7 +3861,6 @@ mod tests { } #[test] - #[should_panic] #[cfg(debug_assertions)] fn peerset_report_connect_with_disabled_pending_enable_peer() { let (mut notif, _peerset) = development_notifs(); @@ -3899,11 +3898,15 @@ mod tests { Some(&PeerState::DisabledPendingEnable { .. }) )); + // duplicate "connect" must not change the state notif.peerset_report_connect(peer, set_id); + assert!(std::matches!( + notif.peers.get(&(peer, set_id)), + Some(&PeerState::DisabledPendingEnable { .. }) + )); } #[test] - #[should_panic] #[cfg(debug_assertions)] fn peerset_report_connect_with_requested_peer() { let (mut notif, _peerset) = development_notifs(); @@ -3914,11 +3917,12 @@ mod tests { notif.peerset_report_connect(peer, set_id); assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Requested))); + // Duplicate "connect" must not change the state. notif.peerset_report_connect(peer, set_id); + assert!(std::matches!(notif.peers.get(&(peer, set_id)), Some(&PeerState::Requested))); } #[test] - #[should_panic] #[cfg(debug_assertions)] fn peerset_report_connect_with_pending_requested() { let (mut notif, _peerset) = development_notifs(); @@ -3967,7 +3971,12 @@ mod tests { Some(&PeerState::PendingRequest { .. }) )); + // duplicate "connect" must not change the state notif.peerset_report_connect(peer, set_id); + assert!(std::matches!( + notif.peers.get(&(peer, set_id)), + Some(&PeerState::PendingRequest { .. }) + )); } #[test] From 263db27f53d328b3fc1570d86c477949640abde0 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Tue, 16 May 2023 14:48:41 +0300 Subject: [PATCH 92/98] Delete `Peerset` test duplicating `ProtocolController` tests --- client/peerset/src/lib.rs | 226 -------------------------------------- 1 file changed, 226 deletions(-) diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index 336075dc75446..a6258fb52226e 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -425,229 +425,3 @@ pub enum DropReason { /// peer doesn't actually belong to this set. Refused, } - -/* -#[cfg(test)] -mod tests { - use super::{ - IncomingIndex, Message, Peerset, PeersetConfig, ReputationChange, SetConfig, SetId, - BANNED_THRESHOLD, - }; - use futures::prelude::*; - use libp2p_identity::PeerId; - use std::{pin::Pin, task::Poll, thread, time::Duration}; - - fn assert_messages(mut peerset: Peerset, messages: Vec) -> Peerset { - for expected_message in messages { - let (message, p) = next_message(peerset).expect("expected message"); - assert_eq!(message, expected_message); - peerset = p; - } - peerset - } - - fn next_message(mut peerset: Peerset) -> Result<(Message, Peerset), ()> { - let next = futures::executor::block_on_stream(&mut peerset).next(); - let message = next.ok_or(())?; - Ok((message, peerset)) - } - - #[test] - fn test_peerset_add_reserved_peer() { - let bootnode = PeerId::random(); - let reserved_peer = PeerId::random(); - let reserved_peer2 = PeerId::random(); - let config = PeersetConfig { - sets: vec![SetConfig { - in_peers: 0, - out_peers: 2, - bootnodes: vec![bootnode], - reserved_nodes: Default::default(), - reserved_only: true, - }], - }; - - let (peerset, handle) = Peerset::from_config(config); - handle.add_reserved_peer(SetId::from(0), reserved_peer); - handle.add_reserved_peer(SetId::from(0), reserved_peer2); - - assert_messages( - peerset, - vec![ - Message::Connect { set_id: SetId::from(0), peer_id: reserved_peer }, - Message::Connect { set_id: SetId::from(0), peer_id: reserved_peer2 }, - ], - ); - } - - #[test] - fn test_peerset_incoming() { - let bootnode = PeerId::random(); - let incoming = PeerId::random(); - let incoming2 = PeerId::random(); - let incoming3 = PeerId::random(); - let ii = IncomingIndex(1); - let ii2 = IncomingIndex(2); - let ii3 = IncomingIndex(3); - let ii4 = IncomingIndex(3); - let config = PeersetConfig { - sets: vec![SetConfig { - in_peers: 2, - out_peers: 1, - bootnodes: vec![bootnode], - reserved_nodes: Default::default(), - reserved_only: false, - }], - }; - - let (mut peerset, _handle) = Peerset::from_config(config); - peerset.incoming(SetId::from(0), incoming, ii); - peerset.incoming(SetId::from(0), incoming, ii4); - peerset.incoming(SetId::from(0), incoming2, ii2); - peerset.incoming(SetId::from(0), incoming3, ii3); - - assert_messages( - peerset, - vec![ - Message::Connect { set_id: SetId::from(0), peer_id: bootnode }, - Message::Accept(ii), - Message::Accept(ii2), - Message::Reject(ii3), - ], - ); - } - - #[test] - fn test_peerset_reject_incoming_in_reserved_only() { - let incoming = PeerId::random(); - let ii = IncomingIndex(1); - let config = PeersetConfig { - sets: vec![SetConfig { - in_peers: 50, - out_peers: 50, - bootnodes: vec![], - reserved_nodes: Default::default(), - reserved_only: true, - }], - }; - - let (mut peerset, _) = Peerset::from_config(config); - peerset.incoming(SetId::from(0), incoming, ii); - - assert_messages(peerset, vec![Message::Reject(ii)]); - } - - #[test] - fn test_peerset_discovered() { - let bootnode = PeerId::random(); - let discovered = PeerId::random(); - let discovered2 = PeerId::random(); - let config = PeersetConfig { - sets: vec![SetConfig { - in_peers: 0, - out_peers: 2, - bootnodes: vec![bootnode], - reserved_nodes: Default::default(), - reserved_only: false, - }], - }; - - let (mut peerset, _handle) = Peerset::from_config(config); - peerset.add_to_peers_set(SetId::from(0), discovered); - peerset.add_to_peers_set(SetId::from(0), discovered); - peerset.add_to_peers_set(SetId::from(0), discovered2); - - assert_messages( - peerset, - vec![ - Message::Connect { set_id: SetId::from(0), peer_id: bootnode }, - Message::Connect { set_id: SetId::from(0), peer_id: discovered }, - ], - ); - } - - #[test] - fn test_peerset_banned() { - let (mut peerset, handle) = Peerset::from_config(PeersetConfig { - sets: vec![SetConfig { - in_peers: 25, - out_peers: 25, - bootnodes: vec![], - reserved_nodes: Default::default(), - reserved_only: false, - }], - }); - - // We ban a node by setting its reputation under the threshold. - let peer_id = PeerId::random(); - handle.report_peer(peer_id, ReputationChange::new(BANNED_THRESHOLD - 1, "")); - - let fut = futures::future::poll_fn(move |cx| { - // We need one polling for the message to be processed. - assert_eq!(Stream::poll_next(Pin::new(&mut peerset), cx), Poll::Pending); - - // Check that an incoming connection from that node gets refused. - peerset.incoming(SetId::from(0), peer_id, IncomingIndex(1)); - if let Poll::Ready(msg) = Stream::poll_next(Pin::new(&mut peerset), cx) { - assert_eq!(msg.unwrap(), Message::Reject(IncomingIndex(1))); - } else { - panic!() - } - - // Wait a bit for the node's reputation to go above the threshold. - thread::sleep(Duration::from_millis(1500)); - - // Try again. This time the node should be accepted. - peerset.incoming(SetId::from(0), peer_id, IncomingIndex(2)); - while let Poll::Ready(msg) = Stream::poll_next(Pin::new(&mut peerset), cx) { - assert_eq!(msg.unwrap(), Message::Accept(IncomingIndex(2))); - } - - Poll::Ready(()) - }); - - futures::executor::block_on(fut); - } - - #[test] - fn test_relloc_after_banned() { - let (mut peerset, handle) = Peerset::from_config(PeersetConfig { - sets: vec![SetConfig { - in_peers: 25, - out_peers: 25, - bootnodes: vec![], - reserved_nodes: Default::default(), - reserved_only: false, - }], - }); - - // We ban a node by setting its reputation under the threshold. - let peer_id = PeerId::random(); - handle.report_peer(peer_id, ReputationChange::new(BANNED_THRESHOLD - 1, "")); - - let fut = futures::future::poll_fn(move |cx| { - // We need one polling for the message to be processed. - assert_eq!(Stream::poll_next(Pin::new(&mut peerset), cx), Poll::Pending); - - // Check that an incoming connection from that node gets refused. - // This is already tested in other tests, but it is done again here because it doesn't - // hurt. - peerset.incoming(SetId::from(0), peer_id, IncomingIndex(1)); - if let Poll::Ready(msg) = Stream::poll_next(Pin::new(&mut peerset), cx) { - assert_eq!(msg.unwrap(), Message::Reject(IncomingIndex(1))); - } else { - panic!() - } - - // Wait for the peerset to change its mind and actually connect to it. - while let Poll::Ready(msg) = Stream::poll_next(Pin::new(&mut peerset), cx) { - assert_eq!(msg.unwrap(), Message::Connect { set_id: SetId::from(0), peer_id }); - } - - Poll::Ready(()) - }); - - futures::executor::block_on(fut); - } -} -*/ From 47edf311b7565cb1fe4e79dfaee32189d6c39ca6 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 18 May 2023 12:41:11 +0300 Subject: [PATCH 93/98] minor: add issue references in TODOs --- client/network/src/service/traits.rs | 1 + client/peerset/src/lib.rs | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/client/network/src/service/traits.rs b/client/network/src/service/traits.rs index 6f2b09f5c2b86..22af3816bd15d 100644 --- a/client/network/src/service/traits.rs +++ b/client/network/src/service/traits.rs @@ -238,6 +238,7 @@ where // TODO: when we get rid of `Peerset`, we'll likely need to add some kind of async // interface to `PeerStore`, otherwise we'll have trouble calling functions accepting // `&mut self` via `Arc`. + // See https://github.com/paritytech/substrate/issues/14170. T::report_peer(self, who, cost_benefit) } diff --git a/client/peerset/src/lib.rs b/client/peerset/src/lib.rs index a6258fb52226e..8a5ed7ddcd134 100644 --- a/client/peerset/src/lib.rs +++ b/client/peerset/src/lib.rs @@ -348,7 +348,8 @@ impl Peerset { /// Produces a JSON object containing the state of the peerset manager, for debugging purposes. pub fn debug_info(&mut self) -> serde_json::Value { - // TODO: check what info we can include here. + // TODO: Check what info we can include here. + // Issue reference: https://github.com/paritytech/substrate/issues/14160. json!("unimplemented") } From b50d998e5916d02899f18d636c2b6f9dce535754 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 18 May 2023 12:44:45 +0300 Subject: [PATCH 94/98] Apply suggestions from code review Co-authored-by: Anton --- client/peerset/src/protocol_controller.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 5f6917da9bf46..d90aa0a3bdfc4 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -25,12 +25,12 @@ //! `ProtocolController` has an imperfect view of the states of the peers. To reduce this //! desynchronization, the following measures are taken: //! -//! 1. Network peer events from `Notifictions` are prioritized over actions from external API and +//! 1. Network peer events from `Notifications` are prioritized over actions from external API and //! internal actions by `ProtocolController` (like slot allocation). //! 2. `Notifications` ignores all commands from `ProtocolController` after sending "incoming" //! request until receiving the answer to this "incoming" request. -//! 3. After sending a "connect" message, `ProtocolController` switchs the state of the peer from -//! `Outbound` to `Inbound` if receives an "incoming" request from `Notifications` for this peer. +//! 3. After sending a "connect" message, `ProtocolController` switches the state of the peer from +//! `Outbound` to `Inbound` if it receives an "incoming" request from `Notifications` for this peer. //! //! These measures do not eliminate confusing commands from `ProtocolController` completely, //! so `Notifications` must correctly handle seemingly inconsistent commands, like a "connect" @@ -522,7 +522,6 @@ impl ProtocolController { self.accept_connection(incoming_index); }, PeerState::NotConnected => { - // FIXME: unable to call `self.is_banned()` because of borrowed `self`. if self.peer_store.is_banned(&peer_id) { self.reject_connection(incoming_index); } else { From 2cd0188a03d59cee64b644e6eb244b224d300504 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 18 May 2023 12:47:35 +0300 Subject: [PATCH 95/98] rustfmt --- client/peerset/src/protocol_controller.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index d90aa0a3bdfc4..0fde1c7e043eb 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -30,7 +30,8 @@ //! 2. `Notifications` ignores all commands from `ProtocolController` after sending "incoming" //! request until receiving the answer to this "incoming" request. //! 3. After sending a "connect" message, `ProtocolController` switches the state of the peer from -//! `Outbound` to `Inbound` if it receives an "incoming" request from `Notifications` for this peer. +//! `Outbound` to `Inbound` if it receives an "incoming" request from `Notifications` for this +//! peer. //! //! These measures do not eliminate confusing commands from `ProtocolController` completely, //! so `Notifications` must correctly handle seemingly inconsistent commands, like a "connect" @@ -517,18 +518,17 @@ impl ProtocolController { match state { PeerState::Connected(ref mut direction) => { // We are accepting an incoming connection, so ensure the direction is inbound. - // (See the note above.) + // (See the implementation note above.) *direction = Direction::Inbound; self.accept_connection(incoming_index); }, - PeerState::NotConnected => { + PeerState::NotConnected => if self.peer_store.is_banned(&peer_id) { self.reject_connection(incoming_index); } else { *state = PeerState::Connected(Direction::Inbound); self.accept_connection(incoming_index); - } - }, + }, } return } From a0dcac4adafbe4e05d96ab86a71e793c89a73e46 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Thu, 18 May 2023 15:14:39 +0300 Subject: [PATCH 96/98] monor: format after `rustfmt` --- client/peerset/src/protocol_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 0fde1c7e043eb..20f42befcfcf5 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -31,7 +31,7 @@ //! request until receiving the answer to this "incoming" request. //! 3. After sending a "connect" message, `ProtocolController` switches the state of the peer from //! `Outbound` to `Inbound` if it receives an "incoming" request from `Notifications` for this -//! peer. +//! peer. //! //! These measures do not eliminate confusing commands from `ProtocolController` completely, //! so `Notifications` must correctly handle seemingly inconsistent commands, like a "connect" From 6960b8ff901c3b25aa8d0dea5da4b7b7de1d2848 Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 22 May 2023 13:39:06 +0300 Subject: [PATCH 97/98] Add in/out slot counters logging --- client/peerset/src/protocol_controller.rs | 60 ++++++++++++++++++----- 1 file changed, 49 insertions(+), 11 deletions(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index 20f42befcfcf5..ed0e6f9fcc8b3 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -301,21 +301,42 @@ impl ProtocolController { /// Send "accept" message to `Notifications`. fn accept_connection(&mut self, incoming_index: IncomingIndex) { - trace!(target: LOG_TARGET, "Accepting {incoming_index:?}."); + trace!( + target: LOG_TARGET, + "Accepting {:?} on {:?} ({}/{} num_in/max_in).", + incoming_index, + self.set_id, + self.num_in, + self.max_in, + ); let _ = self.to_notifications.unbounded_send(Message::Accept(incoming_index)); } /// Send "reject" message to `Notifications`. fn reject_connection(&mut self, incoming_index: IncomingIndex) { - trace!(target: LOG_TARGET, "Rejecting {incoming_index:?}."); + trace!( + target: LOG_TARGET, + "Rejecting {:?} on {:?} ({}/{} num_in/max_in).", + incoming_index, + self.set_id, + self.num_in, + self.max_in, + ); let _ = self.to_notifications.unbounded_send(Message::Reject(incoming_index)); } /// Send "connect" message to `Notifications`. fn start_connection(&mut self, peer_id: PeerId) { - trace!(target: LOG_TARGET, "Connecting to {peer_id}."); + trace!( + target: LOG_TARGET, + "Connecting to {} on {:?} ({}/{} num_out/max_out).", + peer_id, + self.set_id, + self.num_out, + self.max_out, + ); let _ = self .to_notifications @@ -324,7 +345,16 @@ impl ProtocolController { /// Send "drop" message to `Notifications`. fn drop_connection(&mut self, peer_id: PeerId) { - trace!(target: LOG_TARGET, "Dropping {peer_id}."); + trace!( + target: LOG_TARGET, + "Dropping {} on {:?} ({}/{} num_in/max_in, {}/{} num_out/max_out).", + peer_id, + self.set_id, + self.num_in, + self.max_in, + self.num_out, + self.max_out, + ); let _ = self .to_notifications @@ -404,7 +434,10 @@ impl ProtocolController { // Count connections as of regular node. trace!( target: LOG_TARGET, - "Making a connected reserved node {peer_id} ({direction:?}) a regular one.", + "Making a connected reserved node {} ({:?}) on {:?} a regular one.", + peer_id, + direction, + self.set_id, ); match direction { @@ -450,14 +483,19 @@ impl ProtocolController { // Disconnect all non-reserved peers. self.nodes - .keys() - .cloned() - .collect::>() .iter() - .for_each(|peer_id| self.drop_connection(*peer_id)); + .map(|(k, v)| (k.clone(), v.clone())) + .collect::>() + .iter() + .for_each(|(peer_id, direction)| { + // Update counters in the loop for `drop_connection` to report the correct number. + match direction { + Direction::Inbound => self.num_in -= 1, + Direction::Outbound => self.num_out -= 1, + } + self.drop_connection(*peer_id) + }); self.nodes.clear(); - self.num_in = 0; - self.num_out = 0; } /// Get the list of reserved peers. From a03b9a01e59c3d4ce4b6bb7f1f9ca150898b054c Mon Sep 17 00:00:00 2001 From: Dmitry Markin Date: Mon, 22 May 2023 13:50:58 +0300 Subject: [PATCH 98/98] Make clippy happy --- client/peerset/src/protocol_controller.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/peerset/src/protocol_controller.rs b/client/peerset/src/protocol_controller.rs index ed0e6f9fcc8b3..ce3961f130af0 100644 --- a/client/peerset/src/protocol_controller.rs +++ b/client/peerset/src/protocol_controller.rs @@ -484,7 +484,7 @@ impl ProtocolController { // Disconnect all non-reserved peers. self.nodes .iter() - .map(|(k, v)| (k.clone(), v.clone())) + .map(|(k, v)| (*k, *v)) .collect::>() .iter() .for_each(|(peer_id, direction)| {