From e7f1ef9fd7cae588a71dde425595d2fc829ad04d Mon Sep 17 00:00:00 2001 From: Quang Le Date: Wed, 17 Jun 2026 15:18:52 +0700 Subject: [PATCH 01/12] fix: public some ultilities --- crates/consensus/src/qbft/msg.rs | 4 ++-- crates/p2p/src/utils.rs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/consensus/src/qbft/msg.rs b/crates/consensus/src/qbft/msg.rs index 3fc6ae22..45f4f7f8 100644 --- a/crates/consensus/src/qbft/msg.rs +++ b/crates/consensus/src/qbft/msg.rs @@ -269,7 +269,7 @@ impl SomeMsg for Msg { /// The hash input is deterministic protobuf encoding, then SSZ `PutBytes` /// merkleization. `Any` is rejected because the consensus value hash must bind /// to the inner message bytes, not the transport envelope. -pub(crate) fn hash_proto(msg: &M) -> Result<[u8; 32]> +pub fn hash_proto(msg: &M) -> Result<[u8; 32]> where M: prost::Message + prost::Name, { @@ -288,7 +288,7 @@ where /// This helper hashes the bytes exactly as provided; it does not decode or /// canonicalize a protobuf envelope. Callers must pass bytes produced from the /// concrete inner message with deterministic field/map ordering. -pub(crate) fn hash_proto_bytes(encoded: &[u8]) -> Result<[u8; 32]> { +pub fn hash_proto_bytes(encoded: &[u8]) -> Result<[u8; 32]> { let mut hasher = Hasher::default(); let index = hasher.index(); hasher.put_bytes(encoded).map_err(Error::HashProto)?; diff --git a/crates/p2p/src/utils.rs b/crates/p2p/src/utils.rs index cb91d710..298056a2 100644 --- a/crates/p2p/src/utils.rs +++ b/crates/p2p/src/utils.rs @@ -129,7 +129,7 @@ pub(crate) fn default_swarm_config(cfg: libp2p::swarm::Config) -> libp2p::swarm: } /// Converts a secret key to a libp2p keypair. -pub(crate) fn keypair_from_secret_key(key: k256::SecretKey) -> crate::p2p::Result { +pub fn keypair_from_secret_key(key: k256::SecretKey) -> crate::p2p::Result { let mut der = key.to_sec1_der()?; let keypair = Keypair::secp256k1_from_der(&mut der)?; Ok(keypair) From a3836987b0a891c4cc29ba3090265bd3f67b8876 Mon Sep 17 00:00:00 2001 From: Quang Le Date: Wed, 17 Jun 2026 18:05:05 +0700 Subject: [PATCH 02/12] feat(core): implement prioritisier --- Cargo.lock | 26 + Cargo.toml | 1 + crates/core/src/deadline/calculator.rs | 10 + crates/priority/Cargo.toml | 35 + crates/priority/src/calculate.rs | 424 +++++++++++ crates/priority/src/component.rs | 584 +++++++++++++++ crates/priority/src/consensus.rs | 60 ++ crates/priority/src/error.rs | 141 ++++ crates/priority/src/lib.rs | 32 + crates/priority/src/p2p/behaviour.rs | 184 +++++ crates/priority/src/p2p/handler.rs | 215 ++++++ crates/priority/src/p2p/mod.rs | 364 +++++++++ crates/priority/src/p2p/protocol.rs | 197 +++++ crates/priority/src/prioritiser.rs | 864 ++++++++++++++++++++++ crates/priority/tests/prioritiser_test.rs | 383 ++++++++++ 15 files changed, 3520 insertions(+) create mode 100644 crates/priority/Cargo.toml create mode 100644 crates/priority/src/calculate.rs create mode 100644 crates/priority/src/component.rs create mode 100644 crates/priority/src/consensus.rs create mode 100644 crates/priority/src/error.rs create mode 100644 crates/priority/src/lib.rs create mode 100644 crates/priority/src/p2p/behaviour.rs create mode 100644 crates/priority/src/p2p/handler.rs create mode 100644 crates/priority/src/p2p/mod.rs create mode 100644 crates/priority/src/p2p/protocol.rs create mode 100644 crates/priority/src/prioritiser.rs create mode 100644 crates/priority/tests/prioritiser_test.rs diff --git a/Cargo.lock b/Cargo.lock index b65c3ffb..5485f7f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5929,6 +5929,32 @@ dependencies = [ "vise-exporter", ] +[[package]] +name = "pluto-priority" +version = "1.7.1" +dependencies = [ + "async-trait", + "chrono", + "ethereum_ssz", + "futures", + "k256", + "libp2p", + "pluto-consensus", + "pluto-core", + "pluto-k1util", + "pluto-p2p", + "pluto-ssz", + "pluto-testutil", + "prost 0.14.3", + "prost-types 0.14.3", + "rand 0.8.6", + "test-case", + "thiserror 2.0.18", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "pluto-relay-server" version = "1.7.1" diff --git a/Cargo.toml b/Cargo.toml index 3093cf40..15b3370e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ members = [ "crates/tracing", "crates/peerinfo", "crates/frost", + "crates/priority", ] resolver = "3" diff --git a/crates/core/src/deadline/calculator.rs b/crates/core/src/deadline/calculator.rs index 7ce188ab..3d75a76f 100644 --- a/crates/core/src/deadline/calculator.rs +++ b/crates/core/src/deadline/calculator.rs @@ -1,5 +1,7 @@ //! Deadline calculator trait and beacon-node-derived implementation. +use std::sync::Arc; + use chrono::{DateTime, Duration, Utc}; use pluto_eth2api::EthBeaconNodeApiClient; @@ -86,6 +88,14 @@ pub trait DeadlineCalculator: Send + Sync + 'static { fn deadline(&self, duty: &Duty) -> Result>>; } +/// Lets a shared (`Arc`-wrapped) calculator satisfy the trait, so a single +/// calculator instance can back both a deadliner task and other consumers. +impl DeadlineCalculator for Arc { + fn deadline(&self, duty: &Duty) -> Result>> { + (**self).deadline(duty) + } +} + /// Calculator that reports every duty as never expiring. Useful for /// scenarios that need to plug into the deadliner API but don't actually want /// any eviction (e.g. DKG, which is one-shot and outside the slot timeline). diff --git a/crates/priority/Cargo.toml b/crates/priority/Cargo.toml new file mode 100644 index 00000000..9c2d562a --- /dev/null +++ b/crates/priority/Cargo.toml @@ -0,0 +1,35 @@ +[package] +name = "pluto-priority" +version.workspace = true +edition.workspace = true +repository.workspace = true +license.workspace = true +publish.workspace = true + +[dependencies] +async-trait.workspace = true +chrono.workspace = true +futures.workspace = true +k256.workspace = true +libp2p.workspace = true +pluto-consensus.workspace = true +pluto-core.workspace = true +pluto-k1util.workspace = true +pluto-p2p.workspace = true +pluto-ssz.workspace = true +prost.workspace = true +prost-types.workspace = true +ssz.workspace = true +thiserror.workspace = true +tokio.workspace = true +tokio-util.workspace = true +tracing.workspace = true + +[dev-dependencies] +pluto-testutil.workspace = true +rand.workspace = true +test-case.workspace = true +tokio = { workspace = true, features = ["test-util"] } + +[lints] +workspace = true diff --git a/crates/priority/src/calculate.rs b/crates/priority/src/calculate.rs new file mode 100644 index 00000000..b2e9acd1 --- /dev/null +++ b/crates/priority/src/calculate.rs @@ -0,0 +1,424 @@ +//! Deterministic cluster-wide priority result calculation and message +//! validation for the priority protocol. + +use std::collections::{HashMap, HashSet}; + +use pluto_consensus::qbft::msg::hash_proto_bytes; +use pluto_core::corepb::v1::{ + core::Duty, + priority::{ + PriorityMsg, PriorityResult, PriorityScoredResult, PriorityTopicProposal, + PriorityTopicResult, + }, +}; +use prost::Message; +use prost_types::Any; + +use crate::error::{Error, Result}; + +/// Maximum number of priorities allowed per topic. +const MAX_PRIORITIES: usize = 1000; +/// Weight applied to peer count so it dominates relative priority ordering. +/// +/// Equals [`MAX_PRIORITIES`] so that one extra supporting peer always outweighs +/// any relative-priority difference (which is bounded by `MAX_PRIORITIES`). +/// `MAX_PRIORITIES` is a small compile-time constant that fits an `i64`. +#[allow(clippy::cast_possible_truncation, clippy::cast_possible_wrap)] +const COUNT_WEIGHT: i64 = MAX_PRIORITIES as i64; + +/// Returns the SSZ hash root of an `Any` envelope's deterministic protobuf +/// encoding. +/// +/// The priority protocol treats topics and priorities as opaque `Any` values +/// and binds equality to the encoded envelope (`type_url` + `value`), so the +/// envelope bytes are hashed directly rather than the inner concrete message. +fn hash_any(any: &Any) -> Result<[u8; 32]> { + let encoded = any.encode_to_vec(); + hash_proto_bytes(&encoded).map_err(Error::HashProto) +} + +/// Returns the cluster-wide priorities given the priorities of each peer. +/// +/// Priorities are included in the result if at least `min_required` peers +/// provided them and are ordered by number of peers, then by overall priority. +/// The output is deterministic regardless of input message order. +pub(crate) fn calculate_result(msgs: &[PriorityMsg], min_required: i64) -> Result { + validate_msgs(msgs)?; + + // Group all priority sets by topic. Grouping order is irrelevant: each + // topic is scored independently and `order_topic_results` sorts the final + // results by topic hash, so determinism rests on that final sort. + let mut proposals_by_topic: HashMap<[u8; 32], Vec<&PriorityTopicProposal>> = HashMap::new(); + + for msg in sort_input(msgs) { + for topic in &msg.topics { + let topic_hash = hash_any(topic_any(topic))?; + proposals_by_topic + .entry(topic_hash) + .or_default() + .push(topic); + } + } + + // Minimum required score: priorities supported by fewer peers are dropped. + let min_score = min_required.saturating_sub(1).saturating_mul(COUNT_WEIGHT); + + let mut topic_results: Vec = Vec::new(); + + for proposals in proposals_by_topic.values() { + // Accumulate overall score per priority, ordering by count then by + // relative priority. First-seen order is preserved for tie breaking. + let mut all_priorities: Vec<[u8; 32]> = Vec::new(); + let mut scores: HashMap<[u8; 32], i64> = HashMap::new(); + let mut priorities: HashMap<[u8; 32], Any> = HashMap::new(); + + for proposal in proposals { + for (order, prio) in proposal.priorities.iter().enumerate() { + let prio_hash = hash_any(prio)?; + + if !scores.contains_key(&prio_hash) { + all_priorities.push(prio_hash); + } + + // `order` is bounded below MAX_PRIORITIES by validate_msgs, so + // it fits in i64 and never exceeds COUNT_WEIGHT. + let weight = COUNT_WEIGHT.saturating_sub(i64::try_from(order).unwrap_or(i64::MAX)); + let score = scores.entry(prio_hash).or_insert(0); + *score = score.saturating_add(weight); + priorities.insert(prio_hash, prio.clone()); + } + } + + // Order by score decreasing. A stable sort preserves first-seen order + // for equal scores (input is pre-sorted by peer id), so the output is + // deterministic and internally consistent. + all_priorities.sort_by(|a, b| scores[b].cmp(&scores[a])); + + let mut result = PriorityTopicResult { + topic: proposals[0].topic.clone(), + priorities: Vec::new(), + }; + + for prio_hash in &all_priorities { + let score = scores[prio_hash]; + if score <= min_score { + continue; + } + + result.priorities.push(PriorityScoredResult { + priority: Some(priorities[prio_hash].clone()), + score, + }); + } + + topic_results.push(result); + } + + let ordered = order_topic_results(topic_results)?; + + Ok(PriorityResult { + msgs: msgs.to_vec(), + topics: ordered, + }) +} + +/// Returns topic results ordered by topic hash for deterministic output. +fn order_topic_results(values: Vec) -> Result> { + let mut tuples: Vec<([u8; 32], PriorityTopicResult)> = Vec::with_capacity(values.len()); + for value in values { + let hash = hash_any(topic_result_any(&value))?; + tuples.push((hash, value)); + } + + tuples.sort_by_key(|t| t.0); + + Ok(tuples.into_iter().map(|(_, value)| value).collect()) +} + +/// Returns a copy of the messages ordered by peer id. +fn sort_input(msgs: &[PriorityMsg]) -> Vec<&PriorityMsg> { + let mut resp: Vec<&PriorityMsg> = msgs.iter().collect(); + resp.sort_by(|a, b| a.peer_id.cmp(&b.peer_id)); + resp +} + +/// Validates the priority messages, rejecting: +/// - empty message sets, +/// - duplicate peers, +/// - mismatching duties, +/// - duplicate topics within a peer, +/// - more than 1000 priorities within a topic, +/// - duplicate priorities within a topic. +fn validate_msgs(msgs: &[PriorityMsg]) -> Result<()> { + if msgs.is_empty() { + return Err(Error::MessagesEmpty); + } + + let mut duty: Option = None; + let mut dedup_peers: HashSet = HashSet::new(); + + for msg in msgs { + // The reference duty is taken from the first message and stays unset + // while early messages carry no duty; once set, every subsequent duty + // must be proto-equal to the reference. + match duty { + None => duty = msg.duty, + Some(d) if Some(d) != msg.duty => { + return Err(Error::MismatchingDuties); + } + Some(_) => {} + } + + if !dedup_peers.insert(msg.peer_id.clone()) { + return Err(Error::DuplicatePeer); + } + + let mut dedup_topics: HashSet<[u8; 32]> = HashSet::new(); + + for topic in &msg.topics { + let topic_hash = hash_any(topic_any(topic))?; + + if !dedup_topics.insert(topic_hash) { + return Err(Error::DuplicateTopic); + } else if topic.priorities.len() >= MAX_PRIORITIES { + return Err(Error::MaxPriorityReached); + } + + let mut dedup_priority: HashSet<[u8; 32]> = HashSet::new(); + + for priority in &topic.priorities { + let prio_hash = hash_any(priority)?; + if !dedup_priority.insert(prio_hash) { + return Err(Error::DuplicatePriority); + } + } + } + } + + Ok(()) +} + +/// Returns the topic's `Any`, treating an absent topic as the empty `Any`. +/// +/// An unset topic and a default `Any` both encode to empty bytes, so they hash +/// identically; using the shared empty `Any` keeps that equivalence explicit. +fn topic_any(topic: &PriorityTopicProposal) -> &Any { + topic.topic.as_ref().unwrap_or(&EMPTY_ANY) +} + +/// See [`topic_any`]: yields the empty `Any` for an absent topic result topic. +fn topic_result_any(result: &PriorityTopicResult) -> &Any { + result.topic.as_ref().unwrap_or(&EMPTY_ANY) +} + +/// Shared empty `Any` used as the hash input for an unset topic. +static EMPTY_ANY: Any = Any { + type_url: String::new(), + value: Vec::new(), +}; + +#[cfg(test)] +mod tests { + use pluto_core::corepb::v1::core::ParSignedData; + use rand::seq::SliceRandom; + use test_case::test_case; + + use super::*; + + /// Quorum used by `TestCalculateResults` (not accurate, illustrative). + const Q: i64 = 3; + + /// Wraps a string as an `Any` of `ParSignedData{data: s}`. + fn to_any(s: &str) -> Any { + Any::from_msg(&ParSignedData { + data: s.as_bytes().to_vec().into(), + ..Default::default() + }) + .expect("pack ParSignedData") + } + + /// Wraps each string as an `Any` of `ParSignedData{data}`. + fn to_anys(ss: &[&str]) -> Vec { + ss.iter().map(|s| to_any(s)).collect() + } + + /// Extracts the string from an `Any` of `ParSignedData`. + fn from_any(a: &Any) -> String { + let psd: ParSignedData = a.to_msg().expect("unpack ParSignedData"); + String::from_utf8(psd.data.to_vec()).expect("utf8 data") + } + + /// Builds the priority messages for a calculate test case from a list of + /// per-peer priority sets and a slot. + fn build_msgs(priority_sets: &[&[&str]], slot: u64) -> Vec { + let topic = to_any("versions"); + let ignored = to_any("ignored"); + + priority_sets + .iter() + .enumerate() + .map(|(j, set)| PriorityMsg { + duty: Some(Duty { slot, r#type: 0 }), + topics: vec![ + PriorityTopicProposal { + topic: Some(topic.clone()), + priorities: to_anys(set), + }, + PriorityTopicProposal { + topic: Some(ignored.clone()), + priorities: Vec::new(), + }, + ], + peer_id: j.to_string(), + signature: Vec::new().into(), + }) + .collect() + } + + // Calculate-result cases. Each case is the priority sets per peer, the + // expected ordered result strings, and the expected scores (empty when the + // result is empty). + #[test_case(&[&["v1"]], &[], &[], 0; "1*v1")] + #[test_case(&[&["v1"], &["v1"]], &[], &[], 1; "Q-1*v1")] + #[test_case(&[&["v1"], &["v1"], &["v1"]], &["v1"], &[3000], 2; "Q*v1")] + #[test_case(&[&["v1"], &["v1"], &["v1"], &["v1"], &["v1"]], &["v1"], &[5000], 3; "N*v1")] + #[test_case(&[&["v1"], &["v1"], &["v1"], &["v1"], &["v2", "v1"]], &["v1"], &[4999], 4; "N-1*v1,1*v2")] + #[test_case(&[&["v1"], &["v1"], &["v2", "v1"], &["v2", "v1"], &["v2", "v1"]], &["v1", "v2"], &[4997, 3000], 5; "N-Q*v1,Q*v2")] + #[test_case(&[&["v2", "v1"], &["v2", "v1"], &["v2", "v1"], &["v2", "v1"], &["v2", "v1"]], &["v2", "v1"], &[5000, 4995], 6; "N*v2")] + #[test_case(&[&["v2", "v1"], &["v2", "v1"], &["v2", "v1"], &["v2", "v1"]], &["v2", "v1"], &[4000, 3996], 7; "N-1*v2,1*down")] + #[test_case(&[&["v2", "v1"], &["v2", "v1"]], &[], &[], 8; "Q-1*v2,3*down")] + #[test_case(&[&["v1"], &["v2", "v1"], &["v2", "v1"], &["v2", "v1"], &["v2", "v1"]], &["v1", "v2"], &[4996, 4000], 9; "1*v1,N-1*v2")] + #[test_case(&[&["v1"], &["v2", "v1"], &["v2", "v1"], &["v2", "v1"]], &["v1", "v2"], &[3997, 3000], 10; "1*v1,N-2*v2,1*down")] + #[test_case(&[&["v1"], &["v2", "v1"], &["v2", "v1"]], &["v1"], &[2998], 11; "1*v1,Q-1*v2,2*down")] + #[test_case(&[&["v1"], &["v2", "v1"], &["v2", "v1"], &["v2", "v1"], &["v3", "v2"]], &["v2", "v1"], &[3999, 3997], 12; "1*v1,N-2*v2,1*v3")] + #[test_case(&[&["v1"], &["v1"], &["v2", "v1"], &["v2", "v1"], &["v3", "v2"]], &["v1", "v2"], &[3998, 2999], 13; "2*v1,N-3*v2,1*v3")] + #[test_case(&[&["v1"], &["v2", "v1"], &["v3", "v2"], &["v3", "v2"], &["v3", "v2"]], &["v2", "v3"], &[3997, 3000], 14; "1*v1,1*v2,Q*v3")] + #[test_case(&[&["v1"], &["v1"], &["v3", "v2"], &["v3", "v2"], &["v3", "v2"]], &["v3", "v2"], &[3000, 2997], 15; "2*v1,Q*v3")] + #[test_case(&[&["x", "y"], &["x", "y"], &["y", "x"], &["y", "x"]], &["x", "y"], &[3998, 3998], 1; "deterministic ordering instance 1")] + #[test_case(&[&["x", "y"], &["x", "y"], &["y", "x"], &["y", "x"]], &["x", "y"], &[3998, 3998], 9; "deterministic ordering instance 9")] + fn calculate_results( + priority_sets: &[&[&str]], + expected_result: &[&str], + expected_scores: &[i64], + slot: u64, + ) { + let topic = to_any("versions"); + let mut msgs = build_msgs(priority_sets, slot); + + // Shuffle since the function must be deterministic. + msgs.shuffle(&mut rand::thread_rng()); + + let result = calculate_result(&msgs, Q).expect("calculate"); + assert_eq!(result.topics.len(), 2, "two topics (versions + ignored)"); + + let topic_result = result + .topics + .iter() + .find(|t| t.topic.as_ref() == Some(&topic)) + .expect("versions topic present"); + + if expected_result.is_empty() { + assert!( + topic_result.priorities.is_empty(), + "expected empty priorities, got {:?}", + topic_result.priorities + ); + return; + } + + let actual_result: Vec = topic_result + .priorities + .iter() + .map(|p| from_any(p.priority.as_ref().expect("priority any"))) + .collect(); + let actual_scores: Vec = topic_result.priorities.iter().map(|p| p.score).collect(); + + let expected_result: Vec = expected_result.iter().map(|s| s.to_string()).collect(); + assert_eq!(actual_result, expected_result, "result ordering"); + + if !expected_scores.is_empty() { + assert_eq!(actual_scores, expected_scores, "scores"); + } + } + + /// Helper to build a single valid message with one topic and given peer id. + fn msg(peer_id: &str, slot: u64, priorities: &[&str]) -> PriorityMsg { + PriorityMsg { + duty: Some(Duty { slot, r#type: 0 }), + topics: vec![PriorityTopicProposal { + topic: Some(to_any("versions")), + priorities: to_anys(priorities), + }], + peer_id: peer_id.to_string(), + signature: Vec::new().into(), + } + } + + #[test] + fn validate_empty() { + assert!(matches!( + calculate_result(&[], Q), + Err(Error::MessagesEmpty) + )); + } + + #[test] + fn validate_mismatching_duties() { + let msgs = vec![msg("0", 1, &["v1"]), msg("1", 2, &["v1"])]; + assert!(matches!( + calculate_result(&msgs, Q), + Err(Error::MismatchingDuties) + )); + } + + #[test] + fn validate_duplicate_peer() { + let msgs = vec![msg("0", 1, &["v1"]), msg("0", 1, &["v2"])]; + assert!(matches!( + calculate_result(&msgs, Q), + Err(Error::DuplicatePeer) + )); + } + + #[test] + fn validate_duplicate_topic() { + let mut m = msg("0", 1, &["v1"]); + m.topics.push(PriorityTopicProposal { + topic: Some(to_any("versions")), + priorities: to_anys(&["v2"]), + }); + assert!(matches!( + calculate_result(&[m], Q), + Err(Error::DuplicateTopic) + )); + } + + #[test] + fn validate_max_priority_reached() { + let priorities: Vec = (0..MAX_PRIORITIES) + .map(|i| to_any(&i.to_string())) + .collect(); + let m = PriorityMsg { + duty: Some(Duty { slot: 1, r#type: 0 }), + topics: vec![PriorityTopicProposal { + topic: Some(to_any("versions")), + priorities, + }], + peer_id: "0".to_string(), + signature: Vec::new().into(), + }; + assert!(matches!( + calculate_result(&[m], Q), + Err(Error::MaxPriorityReached) + )); + } + + #[test] + fn validate_duplicate_priority() { + let m = msg("0", 1, &["v1", "v1"]); + assert!(matches!( + calculate_result(&[m], Q), + Err(Error::DuplicatePriority) + )); + } +} diff --git a/crates/priority/src/component.rs b/crates/priority/src/component.rs new file mode 100644 index 00000000..395a8344 --- /dev/null +++ b/crates/priority/src/component.rs @@ -0,0 +1,584 @@ +//! Friendly priority API: domain types, message signing/verification, and +//! conversions between the domain types and their protobuf representations. +//! +//! Topics and priorities are arbitrary strings exposed to callers, wrapped on +//! the wire as `google.protobuf.Any`-packed structpb string values. + +use std::{collections::HashMap, sync::Arc, time::Duration}; + +use chrono::Utc; +use k256::{PublicKey, SecretKey}; +use libp2p::PeerId; +use pluto_consensus::qbft::msg::hash_proto; +use pluto_core::{ + corepb::v1::priority::{PriorityMsg, PriorityTopicProposal, PriorityTopicResult}, + deadline::{DeadlineCalculator, DeadlinerTask}, + types::Duty, +}; +use pluto_p2p::peer::peer_id_to_public_key; +use prost::Message; +use prost_types::{Any, Value, value::Kind}; +use tokio_util::sync::CancellationToken; + +use crate::{ + consensus::{Consensus, PrioritySubscriber}, + error::{Error, Result}, + p2p::Behaviour, + prioritiser::{Prioritiser, duty_to_proto}, +}; + +/// Protobuf `type_url` for `google.protobuf.Value`. +/// +/// `prost_types::Value` has no `prost::Name` impl, so the canonical type URL is +/// set explicitly when packing into `Any`. +const VALUE_TYPE_URL: &str = "type.googleapis.com/google.protobuf.Value"; + +/// Proposed priorities for a single prioritise topic. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TopicProposal { + /// Topic identifier. + pub topic: String, + /// Proposed priorities in decreasing preference. + pub priorities: Vec, +} + +/// Cluster-agreed resulting priorities for a single prioritise topic. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct TopicResult { + /// Topic identifier. + pub topic: String, + /// Resulting scored priorities in decreasing score. + pub priorities: Vec, +} + +impl TopicResult { + /// Returns the priorities without their scores. + pub fn priorities_only(&self) -> Vec { + self.priorities.iter().map(|p| p.priority.clone()).collect() + } +} + +/// A cluster-agreed priority including its score. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ScoredPriority { + /// Priority identifier. + pub priority: String, + /// Aggregate score across proposing peers. + pub score: i64, +} + +/// Validates a received priority message's signature against a known peer set. +/// +/// Returns the unknown-peer or invalid-signature error rather than a boolean, +/// so callers reject messages from unrecognised peers or with bad signatures. +pub type MsgVerifier = Box Result<()> + Send + Sync + 'static>; + +/// Returns a copy of the message signed by `privkey`. +/// +/// The signature field is cleared before hashing so the signature covers the +/// message content only. +pub fn sign_msg(msg: &PriorityMsg, privkey: &SecretKey) -> Result { + let mut clone = msg.clone(); + clone.signature = Default::default(); + + let hash = hash_proto(&clone).map_err(Error::HashProto)?; + let sig = pluto_k1util::sign(privkey, &hash).map_err(Error::Sign)?; + + clone.signature = sig.to_vec().into(); + + Ok(clone) +} + +/// Returns whether `msg` was signed by `pubkey`. +/// +/// Errors on an empty signature or on a recovery failure; a recovered key that +/// does not match `pubkey` returns `Ok(false)`. +pub(crate) fn verify_msg_sig(msg: &PriorityMsg, pubkey: &PublicKey) -> Result { + if msg.signature.is_empty() { + return Err(Error::EmptySignature); + } + + let mut clone = msg.clone(); + clone.signature = Default::default(); + + let hash = hash_proto(&clone).map_err(Error::HashProto)?; + let recovered = pluto_k1util::recover(&hash, &msg.signature).map_err(Error::Recover)?; + + Ok(&recovered == pubkey) +} + +/// Returns a verifier that checks message signatures against the public keys of +/// the provided peers. +/// +/// The verifier rejects messages with missing duty fields, from unknown peers, +/// or with invalid signatures. +pub(crate) fn new_msg_verifier(peers: &[PeerId]) -> Result { + let mut keys: HashMap = HashMap::with_capacity(peers.len()); + for peer in peers { + let pk = peer_id_to_public_key(peer).map_err(Error::PeerKey)?; + keys.insert(peer.to_string(), pk); + } + + Ok(Box::new(move |msg: &PriorityMsg| { + if msg.duty.is_none() { + return Err(Error::InvalidMsgProtoFields); + } + + let Some(key) = keys.get(&msg.peer_id) else { + return Err(Error::UnknownPeerId); + }; + + if verify_msg_sig(msg, key)? { + Ok(()) + } else { + Err(Error::InvalidSignature) + } + })) +} + +/// Packs a string as an `Any`-wrapped structpb string value. +fn string_to_any(s: &str) -> Any { + let value = Value { + kind: Some(Kind::StringValue(s.to_owned())), + }; + + Any { + type_url: VALUE_TYPE_URL.to_owned(), + value: value.encode_to_vec(), + } +} + +/// Returns the proto form of a topic proposal. +pub(crate) fn topic_proposal_to_proto(p: &TopicProposal) -> PriorityTopicProposal { + PriorityTopicProposal { + topic: Some(string_to_any(&p.topic)), + priorities: p.priorities.iter().map(|s| string_to_any(s)).collect(), + } +} + +impl TryFrom<&PriorityTopicResult> for TopicResult { + type Error = Error; + + /// Errors if an `Any` envelope is missing or carries the wrong message + /// type, or if any topic or priority value is not a structpb string. + fn try_from(p: &PriorityTopicResult) -> Result { + let topic_val = + unmarshal_value(p.topic.as_ref()).map_err(|e| Error::AnypbTopic(Box::new(e)))?; + let topic = value_string(topic_val)?; + + let mut priorities = Vec::with_capacity(p.priorities.len()); + for scored in &p.priorities { + let prio_val = unmarshal_value(scored.priority.as_ref()) + .map_err(|e| Error::AnypbPriority(Box::new(e)))?; + let prio = value_string(prio_val)?; + priorities.push(ScoredPriority { + priority: prio, + score: scored.score, + }); + } + + Ok(Self { topic, priorities }) + } +} + +/// Unpacks an optional `Any` envelope into a structpb [`Value`]. +/// +/// Rejects an absent envelope, or one whose `type_url` does not name +/// `google.protobuf.Value`, as a mismatched message type +/// ([`Error::MismatchedMessageType`]) before decoding. Only the path segment +/// after the last `/` of the `type_url` is significant, matching the identity +/// check applied when unpacking an `Any`. +fn unmarshal_value(any: Option<&Any>) -> Result { + let any = any.ok_or(Error::MismatchedMessageType)?; + + let type_name = any.type_url.rsplit('/').next().unwrap_or(&any.type_url); + if type_name != "google.protobuf.Value" { + return Err(Error::MismatchedMessageType); + } + + Value::decode(any.value.as_slice()).map_err(Error::DecodeAny) +} + +/// Extracts the string from a structpb [`Value`]. +/// +/// Returns [`Error::TopicValueNotString`] when the value is not a structpb +/// string. +fn value_string(value: Value) -> Result { + match value.kind { + Some(Kind::StringValue(s)) => Ok(s), + _ => Err(Error::TopicValueNotString), + } +} + +/// Friendly-API output subscriber invoked with each decided duty result. +/// +/// The boxed error is propagated back through the consensus subscription chain. +pub type ComponentSubscriber = Box< + dyn Fn(Duty, Vec) -> std::result::Result<(), crate::consensus::ConsensusError> + + Send + + Sync + + 'static, +>; + +/// Wraps a [`Prioritiser`] with the friendly string-based API and message +/// signing, hiding the underlying protobuf types. +pub struct Component { + peer_id: PeerId, + privkey: SecretKey, + prioritiser: Prioritiser, + calculator: Arc, + /// Drives the deadliner's expired-duty channel; consumed once by [`start`]. + /// + /// [`start`]: Component::start + expired: std::sync::Mutex>>, +} + +/// Constructs a priority [`Component`] and the libp2p [`Behaviour`] to register +/// with the swarm. +/// +/// Builds the message verifier from `peers`, spawns a deadliner driven by +/// `calculator`, and wires the prioritiser. The caller must register the +/// returned behaviour with its swarm and call [`Component::start`] exactly +/// once. +#[allow(clippy::too_many_arguments)] +pub fn new_component( + ctx: CancellationToken, + local_id: PeerId, + peers: Vec, + min_required: i64, + consensus: Arc, + exchange_timeout: Duration, + privkey: SecretKey, + calculator: impl DeadlineCalculator, +) -> Result<(Component, Behaviour)> { + let verifier = new_msg_verifier(&peers)?; + let calculator: Arc = Arc::new(calculator); + + let (deadliner, expired) = DeadlinerTask::start(ctx, "priority", calculator.clone()); + + let (prioritiser, behaviour) = Prioritiser::new_internal( + local_id, + peers, + min_required, + consensus, + verifier, + exchange_timeout, + deadliner, + ); + + let component = Component { + peer_id: local_id, + privkey, + prioritiser, + calculator, + expired: std::sync::Mutex::new(Some(expired)), + }; + + Ok((component, behaviour)) +} + +impl Component { + /// Starts the prioritiser's state-cleanup loop. Must be called exactly + /// once. + /// + /// Consumes the deadliner's expired-duty receiver, which can only be taken + /// once. + /// + /// # Panics + /// Panics if called more than once. + pub fn start(&self, ctx: CancellationToken) { + let expired = self + .expired + .lock() + .expect("expired receiver mutex poisoned") + .take() + .expect("Component::start called more than once"); + self.prioritiser.start(ctx, expired); + } + + /// Registers a friendly output subscriber. + /// + /// The subscriber receives the decided result as domain [`TopicResult`]s; + /// proto conversion errors short-circuit and propagate to consensus. + pub fn subscribe(&self, sub: ComponentSubscriber) { + let inner: PrioritySubscriber = Box::new(move |duty, result| { + let mut results = Vec::with_capacity(result.topics.len()); + for topic in &result.topics { + let r = TopicResult::try_from(topic) + .map_err(|e| -> crate::consensus::ConsensusError { Box::new(e) })?; + results.push(r); + } + sub(duty, results) + }); + self.prioritiser.subscribe(inner); + } + + /// Starts a prioritisation instance for `duty` with the given proposals. + /// + /// Returns [`Error::DutyAlreadyExpired`] if the duty has no future + /// deadline. Returns `Ok(())` when `ctx` is cancelled, otherwise + /// propagates a prioritiser error. + pub async fn prioritise( + &self, + ctx: CancellationToken, + duty: Duty, + proposals: &[TopicProposal], + ) -> Result<()> { + let topics = proposals.iter().map(topic_proposal_to_proto).collect(); + + // Derive a per-instance deadline context. A future deadline is + // required; absent or past means the duty already expired. + let deadline = self + .calculator + .deadline(&duty) + .ok() + .flatten() + .ok_or(Error::DutyAlreadyExpired)?; + + let msg = PriorityMsg { + duty: Some(duty_to_proto(&duty)), + peer_id: self.peer_id.to_string(), + topics, + signature: Default::default(), + }; + let msg = sign_msg(&msg, &self.privkey)?; + + // Bound the instance by the duty deadline. The token is cancelled (not + // merely dropped) on elapse so the prioritiser's detached consensus task, + // which holds a clone of it, also tears down. + let instance_ctx = ctx.child_token(); + let remaining = deadline + .signed_duration_since(Utc::now()) + .to_std() + .unwrap_or(Duration::ZERO); + + let res = tokio::select! { + res = self.prioritiser.prioritise(instance_ctx.clone(), msg) => res, + () = tokio::time::sleep(remaining) => { + instance_ctx.cancel(); + return Ok(()); + } + }; + + // A cancelled instance — parent context or deadline — is a graceful stop. + if instance_ctx.is_cancelled() { + return Ok(()); + } + // A non-cancelled failure carries the duty as context. + res.map_err(|e| Error::Prioritise { + duty, + source: Box::new(e), + }) + } +} + +#[cfg(test)] +mod tests { + use pluto_core::corepb::v1::{core::Duty, priority::PriorityScoredResult}; + + use super::*; + + /// Builds an unsigned message for the given peer id with one empty topic. + fn unsigned_msg(peer_id: &str) -> PriorityMsg { + PriorityMsg { + duty: Some(Duty { slot: 1, r#type: 0 }), + topics: vec![topic_proposal_to_proto(&TopicProposal { + topic: "versions".to_owned(), + priorities: vec!["v1".to_owned(), "v2".to_owned()], + })], + peer_id: peer_id.to_owned(), + signature: Default::default(), + } + } + + fn random_key() -> SecretKey { + SecretKey::random(&mut k256::elliptic_curve::rand_core::OsRng) + } + + #[test] + fn sign_verify_round_trip() { + let key = random_key(); + let peer_id = peer_id_from_secret(&key); + + let signed = sign_msg(&unsigned_msg(&peer_id.to_string()), &key).expect("sign"); + assert!(!signed.signature.is_empty(), "signature populated"); + + assert!(verify_msg_sig(&signed, &key.public_key()).expect("verify")); + } + + #[test] + fn verify_tampered_signature() { + let key = random_key(); + let peer_id = peer_id_from_secret(&key); + + let mut signed = sign_msg(&unsigned_msg(&peer_id.to_string()), &key).expect("sign"); + // Tamper with the content; the signature no longer covers it. + signed.peer_id = "tampered".to_owned(); + + // Recovered key differs from the signer's key. + assert!(!verify_msg_sig(&signed, &key.public_key()).expect("verify")); + } + + #[test] + fn verify_empty_signature() { + let key = random_key(); + assert!(matches!( + verify_msg_sig(&unsigned_msg("0"), &key.public_key()), + Err(Error::EmptySignature) + )); + } + + #[test] + fn msg_verifier_round_trip() { + let key = random_key(); + let peer_id = peer_id_from_secret(&key); + let verifier = new_msg_verifier(&[peer_id]).expect("verifier"); + + let signed = sign_msg(&unsigned_msg(&peer_id.to_string()), &key).expect("sign"); + verifier(&signed).expect("known peer + valid signature"); + } + + #[test] + fn msg_verifier_unknown_peer() { + let known = random_key(); + let other = random_key(); + let known_id = peer_id_from_secret(&known); + let verifier = new_msg_verifier(&[known_id]).expect("verifier"); + + let other_id = peer_id_from_secret(&other); + let signed = sign_msg(&unsigned_msg(&other_id.to_string()), &other).expect("sign"); + + assert!(matches!(verifier(&signed), Err(Error::UnknownPeerId))); + } + + #[test] + fn msg_verifier_invalid_signature() { + let key = random_key(); + let peer_id = peer_id_from_secret(&key); + let verifier = new_msg_verifier(&[peer_id]).expect("verifier"); + + // Signed by a different key but claiming the known peer id. + let attacker = random_key(); + let signed = sign_msg(&unsigned_msg(&peer_id.to_string()), &attacker).expect("sign"); + + assert!(matches!(verifier(&signed), Err(Error::InvalidSignature))); + } + + #[test] + fn msg_verifier_missing_duty() { + let key = random_key(); + let peer_id = peer_id_from_secret(&key); + let verifier = new_msg_verifier(&[peer_id]).expect("verifier"); + + let mut msg = sign_msg(&unsigned_msg(&peer_id.to_string()), &key).expect("sign"); + msg.duty = None; + + assert!(matches!(verifier(&msg), Err(Error::InvalidMsgProtoFields))); + } + + #[test] + fn structpb_round_trip() { + let proposal = TopicProposal { + topic: "versions".to_owned(), + priorities: vec!["v1".to_owned(), "v2".to_owned()], + }; + let proto = topic_proposal_to_proto(&proposal); + + // Build a topic result from the proposal to exercise unpacking. + let result_proto = PriorityTopicResult { + topic: proto.topic.clone(), + priorities: proto + .priorities + .iter() + .enumerate() + .map(|(i, any)| PriorityScoredResult { + priority: Some(any.clone()), + score: i64::try_from(i).expect("test index fits i64"), + }) + .collect(), + }; + + let result = TopicResult::try_from(&result_proto).expect("from proto"); + assert_eq!(result.topic, "versions"); + assert_eq!(result.priorities_only(), vec!["v1", "v2"]); + assert_eq!(result.priorities[0].score, 0); + assert_eq!(result.priorities[1].score, 1); + } + + #[test] + fn topic_result_from_proto_non_string() { + // Pack a non-string structpb value (number) into the topic Any. + let number = Value { + kind: Some(Kind::NumberValue(1.0)), + }; + let result_proto = PriorityTopicResult { + topic: Some(Any { + type_url: VALUE_TYPE_URL.to_owned(), + value: number.encode_to_vec(), + }), + priorities: Vec::new(), + }; + + assert!(matches!( + TopicResult::try_from(&result_proto), + Err(Error::TopicValueNotString) + )); + } + + #[test] + fn topic_result_from_proto_wrong_type_url() { + // Valid StringValue bytes but an envelope naming the wrong message type. + let value = Value { + kind: Some(Kind::StringValue("v1".to_owned())), + }; + let result_proto = PriorityTopicResult { + topic: Some(Any { + type_url: "type.googleapis.com/google.protobuf.Duration".to_owned(), + value: value.encode_to_vec(), + }), + priorities: Vec::new(), + }; + + assert!(matches!( + TopicResult::try_from(&result_proto), + Err(Error::AnypbTopic(_)) + )); + } + + #[test] + fn topic_result_from_proto_missing_topic() { + // Absent topic Any is rejected like a nil envelope. + let result_proto = PriorityTopicResult { + topic: None, + priorities: Vec::new(), + }; + + assert!(matches!( + TopicResult::try_from(&result_proto), + Err(Error::AnypbTopic(_)) + )); + } + + #[test] + fn topic_result_from_proto_missing_priority() { + // A present topic but an absent priority Any is rejected as priority. + let topic = string_to_any("versions"); + let result_proto = PriorityTopicResult { + topic: Some(topic), + priorities: vec![PriorityScoredResult { + priority: None, + score: 1, + }], + }; + + assert!(matches!( + TopicResult::try_from(&result_proto), + Err(Error::AnypbPriority(_)) + )); + } + + /// Derives a libp2p `PeerId` from a secp256k1 secret key. + fn peer_id_from_secret(key: &SecretKey) -> PeerId { + pluto_p2p::peer::peer_id_from_key(key.public_key()).expect("peer id from key") + } +} diff --git a/crates/priority/src/consensus.rs b/crates/priority/src/consensus.rs new file mode 100644 index 00000000..0cf55db7 --- /dev/null +++ b/crates/priority/src/consensus.rs @@ -0,0 +1,60 @@ +//! Consensus seam for the priority protocol. +//! +//! The prioritiser proposes each deterministically-computed [`PriorityResult`] +//! through cluster QBFT consensus and subscribes to the decided result. This +//! module defines the [`Consensus`] trait abstracting that interaction so the +//! prioritiser can be unit-tested against a mock, and implements it for the +//! QBFT component. + +use std::error::Error as StdError; + +use async_trait::async_trait; +use pluto_consensus::qbft::{self, SubscriberResult}; +use pluto_core::{corepb::v1::priority::PriorityResult, types::Duty}; +use tokio_util::sync::CancellationToken; + +/// Subscriber callback invoked with each decided priority consensus result. +pub type PrioritySubscriber = + Box SubscriberResult + Send + Sync + 'static>; + +/// Boxed error returned by [`Consensus::propose_priority`]. +pub type ConsensusError = Box; + +/// Cluster consensus over priority results. +/// +/// Implementors run a consensus instance per duty and notify subscribers when +/// agreement is reached. +#[async_trait] +pub trait Consensus: Send + Sync { + /// Proposes a priority result for the duty's consensus instance. + /// + /// `ct` is the instance's cancellation token, tied to the duty deadline, so + /// cancellation reaches the underlying consensus run. + async fn propose_priority( + &self, + duty: Duty, + result: PriorityResult, + ct: &CancellationToken, + ) -> Result<(), ConsensusError>; + + /// Registers a callback invoked with each decided priority result. + fn subscribe_priority(&self, callback: PrioritySubscriber); +} + +#[async_trait] +impl Consensus for qbft::Consensus { + async fn propose_priority( + &self, + duty: Duty, + result: PriorityResult, + ct: &CancellationToken, + ) -> Result<(), ConsensusError> { + qbft::Consensus::propose_priority(self, duty, result, ct) + .await + .map_err(|e| Box::new(e) as ConsensusError) + } + + fn subscribe_priority(&self, callback: PrioritySubscriber) { + qbft::Consensus::subscribe_priority(self, callback); + } +} diff --git a/crates/priority/src/error.rs b/crates/priority/src/error.rs new file mode 100644 index 00000000..7115ed3e --- /dev/null +++ b/crates/priority/src/error.rs @@ -0,0 +1,141 @@ +//! Error types for the priority protocol. + +use pluto_core::types::Duty; +use thiserror::Error; + +/// Result alias for the priority crate. +pub type Result = std::result::Result; + +/// Errors produced by the priority protocol. +#[derive(Debug, Error)] +pub enum Error { + /// No messages were provided to calculate a result. + #[error("messages empty")] + MessagesEmpty, + + /// Messages did not all carry the same duty. + #[error("mismatching duties")] + MismatchingDuties, + + /// Two messages claimed the same peer id. + #[error("duplicate peer")] + DuplicatePeer, + + /// A single peer proposed the same topic twice. + #[error("duplicate topic")] + DuplicateTopic, + + /// A topic proposed at least `maxPriorities` priorities. + #[error("max priority reached")] + MaxPriorityReached, + + /// A topic proposed the same priority twice. + #[error("duplicate priority")] + DuplicatePriority, + + /// Hashing a topic or priority protobuf failed. + #[error("hash proto: {0}")] + HashProto(#[source] pluto_consensus::qbft::msg::Error), + + /// A message carried no signature. + #[error("empty signature")] + EmptySignature, + + /// A message claimed a peer id not in the cluster. + #[error("unknown peer id")] + UnknownPeerId, + + /// A message signature did not match the claimed peer's public key. + #[error("invalid signature")] + InvalidSignature, + + /// A message was missing required proto fields (duty). + #[error("invalid priority msg proto fields")] + InvalidMsgProtoFields, + + /// An `Any`-wrapped topic or priority value was not a structpb string. + #[error("topic value not a string")] + TopicValueNotString, + + /// An `Any` envelope could not be decoded as a structpb value: it carried + /// the wrong `type_url` or undecodable bytes. + #[error("mismatched message type")] + MismatchedMessageType, + + /// Decoding a topic result's topic `Any` failed. + #[error("anypb topic: {0}")] + AnypbTopic(#[source] Box), + + /// Decoding a topic result's priority `Any` failed. + #[error("anypb priority: {0}")] + AnypbPriority(#[source] Box), + + /// Signing a message hash failed. + #[error("sign: {0}")] + Sign(#[source] pluto_k1util::K1UtilError), + + /// Recovering a public key from a signature failed. + #[error("sig to pub: {0}")] + Recover(#[source] pluto_k1util::K1UtilError), + + /// Deriving a peer's public key from its peer id failed. + #[error("peer id to key: {0}")] + PeerKey(#[source] pluto_p2p::peer::PeerError), + + /// Decoding an `Any`-wrapped structpb value failed. + #[error("anypb decode: {0}")] + DecodeAny(#[source] prost::DecodeError), + + /// A peer does not support the priority protocol. + #[error("priority protocol not supported")] + Unsupported, + + /// A libp2p stream or dial error occurred during an exchange. + #[error("priority transport: {0}")] + Transport(String), + + /// The prioritiser transport was shut down. + #[error("prioritiser shutdown")] + Shutdown, + + /// Calculating the deterministic priority result failed. + #[error("calculate priority protocol result: {0}")] + CalculateResult(#[source] Box), + + /// Enqueuing a received request timed out before the deadline. + #[error("timeout enqueuing request")] + TimeoutEnqueuing, + + /// Waiting for this node's proposed priorities timed out before the + /// deadline. + #[error("timeout waiting for proposed priorities")] + TimeoutWaiting, + + /// A received message's peer id did not match the connection's peer id. + #[error("invalid priority message peer id")] + InvalidPeerId, + + /// The duty for a received message had already expired. + #[error("duty expired")] + DutyExpired, + + /// The duty has no future deadline when a prioritise instance was started. + #[error("duty already expired")] + DutyAlreadyExpired, + + /// The prioritise instance's context was cancelled. + #[error("context canceled")] + Cancelled, + + /// A prioritise instance failed for a non-cancelled reason. + /// + /// Carries the `duty` as context and wraps the underlying failure. + #[error("prioritise: {source}")] + Prioritise { + /// The duty whose prioritisation failed. + duty: Duty, + /// The underlying failure. + #[source] + source: Box, + }, +} diff --git a/crates/priority/src/lib.rs b/crates/priority/src/lib.rs new file mode 100644 index 00000000..5fffe294 --- /dev/null +++ b/crates/priority/src/lib.rs @@ -0,0 +1,32 @@ +//! Priority protocol: deterministic cluster-wide priority resolution. +//! +//! Coordinates a cluster-wide priority result per duty by exchanging signed +//! priority messages between peers and computing a deterministic +//! [`calculate::calculate_result`]. + +/// Deterministic priority result calculation and message validation. +pub mod calculate; +/// Friendly priority API: domain types, signing, and proto conversions. +pub mod component; +/// Consensus seam for proposing and subscribing to priority results. +pub mod consensus; +/// Error types for the priority protocol. +pub mod error; +/// libp2p request/response transport for the priority protocol. +pub mod p2p; +/// Priority protocol engine: per-duty exchange and consensus orchestration. +pub mod prioritiser; + +pub use component::{ + Component, ComponentSubscriber, ScoredPriority, TopicProposal, TopicResult, new_component, +}; +pub use consensus::{Consensus, ConsensusError, PrioritySubscriber}; +pub use error::{Error, Result}; +pub use prioritiser::{PROTOCOL_ID, Prioritiser}; + +/// Returns the priority protocol identifiers this implementation supports. +/// +/// Used to register protocol support in a peer store. +pub fn protocols() -> Vec<&'static str> { + vec![PROTOCOL_ID] +} diff --git a/crates/priority/src/p2p/behaviour.rs b/crates/priority/src/p2p/behaviour.rs new file mode 100644 index 00000000..139d4ea8 --- /dev/null +++ b/crates/priority/src/p2p/behaviour.rs @@ -0,0 +1,184 @@ +//! Swarm behaviour backing the priority request/response protocol. +//! +//! The behaviour owns a registered inbound handler callback and routes +//! outbound [`SendReceive`](super::Command::SendReceive) commands to the +//! connection handler for the target peer, dialing first when no connection +//! exists. + +use std::{ + collections::{HashMap, VecDeque}, + task::{Context, Poll}, +}; + +use libp2p::{ + Multiaddr, PeerId, + swarm::{ + ConnectionDenied, ConnectionId, FromSwarm, NetworkBehaviour, NotifyHandler, THandler, + THandlerInEvent, THandlerOutEvent, ToSwarm, + dial_opts::{DialOpts, PeerCondition}, + }, +}; +use tokio::sync::mpsc; + +use super::{ + Command, InboundHandler, + handler::{FromBehaviour, Handler, OutboundRequest}, +}; + +/// Swarm behaviour for the priority protocol. +pub struct Behaviour { + inbound_handler: InboundHandler, + command_rx: mpsc::UnboundedReceiver, + /// Peers with at least one established connection. + connected: HashMap, + /// Outbound requests waiting for a connection to the target peer. + awaiting_connection: HashMap>, + pending_events: VecDeque>>, +} + +/// The priority behaviour emits no swarm-level events. +pub type Event = std::convert::Infallible; + +impl Behaviour { + pub(crate) fn new( + inbound_handler: InboundHandler, + command_rx: mpsc::UnboundedReceiver, + ) -> Self { + Self { + inbound_handler, + command_rx, + connected: HashMap::new(), + awaiting_connection: HashMap::new(), + pending_events: VecDeque::new(), + } + } + + fn handle_command(&mut self, command: Command) { + match command { + Command::SendReceive { peer, request } => self.send_receive(peer, request), + } + } + + fn send_receive(&mut self, peer: PeerId, request: OutboundRequest) { + if self.connected.contains_key(&peer) { + self.notify_handler(peer, request); + return; + } + + let first = self.awaiting_connection.entry(peer).or_default(); + let needs_dial = first.is_empty(); + first.push(request); + + if needs_dial { + self.pending_events.push_back(ToSwarm::Dial { + opts: DialOpts::peer_id(peer) + .condition(PeerCondition::DisconnectedAndNotDialing) + .build(), + }); + } + } + + fn notify_handler(&mut self, peer: PeerId, request: OutboundRequest) { + self.pending_events.push_back(ToSwarm::NotifyHandler { + peer_id: peer, + handler: NotifyHandler::Any, + event: FromBehaviour::SendReceive(request), + }); + } + + fn flush_awaiting(&mut self, peer: PeerId) { + if let Some(requests) = self.awaiting_connection.remove(&peer) { + for request in requests { + self.notify_handler(peer, request); + } + } + } + + fn fail_awaiting(&mut self, peer: PeerId, error: &crate::Error) { + if let Some(requests) = self.awaiting_connection.remove(&peer) { + for request in requests { + let _ = request.response.send(Err(clone_error(error))); + } + } + } +} + +impl NetworkBehaviour for Behaviour { + type ConnectionHandler = Handler; + type ToSwarm = Event; + + fn handle_established_inbound_connection( + &mut self, + _connection_id: ConnectionId, + peer: PeerId, + _local_addr: &Multiaddr, + _remote_addr: &Multiaddr, + ) -> Result, ConnectionDenied> { + Ok(Handler::new(peer, self.inbound_handler.clone())) + } + + fn handle_established_outbound_connection( + &mut self, + _connection_id: ConnectionId, + peer: PeerId, + _addr: &Multiaddr, + _role_override: libp2p::core::Endpoint, + _port_use: libp2p::core::transport::PortUse, + ) -> Result, ConnectionDenied> { + Ok(Handler::new(peer, self.inbound_handler.clone())) + } + + fn on_swarm_event(&mut self, event: FromSwarm) { + match event { + FromSwarm::ConnectionEstablished(event) => { + self.connected.insert(event.peer_id, ()); + self.flush_awaiting(event.peer_id); + } + FromSwarm::ConnectionClosed(event) if event.remaining_established == 0 => { + self.connected.remove(&event.peer_id); + } + FromSwarm::DialFailure(event) => { + if let Some(peer) = event.peer_id { + self.fail_awaiting(peer, &crate::Error::Transport(event.error.to_string())); + } + } + _ => {} + } + } + + fn on_connection_handler_event( + &mut self, + _peer_id: PeerId, + _connection_id: ConnectionId, + event: THandlerOutEvent, + ) { + match event {} + } + + fn poll( + &mut self, + cx: &mut Context<'_>, + ) -> Poll>> { + while let Poll::Ready(Some(command)) = self.command_rx.poll_recv(cx) { + self.handle_command(command); + } + + if let Some(event) = self.pending_events.pop_front() { + return Poll::Ready(event); + } + + Poll::Pending + } +} + +/// Clones an [`crate::Error`] for fan-out to multiple awaiting requests. +/// +/// Only transport/unsupported variants reach this path; both are cheaply +/// reconstructable from their displayed form without losing parity-relevant +/// detail. +fn clone_error(error: &crate::Error) -> crate::Error { + match error { + crate::Error::Unsupported => crate::Error::Unsupported, + other => crate::Error::Transport(other.to_string()), + } +} diff --git a/crates/priority/src/p2p/handler.rs b/crates/priority/src/p2p/handler.rs new file mode 100644 index 00000000..88150a14 --- /dev/null +++ b/crates/priority/src/p2p/handler.rs @@ -0,0 +1,215 @@ +//! Connection handler for the priority protocol. +//! +//! Each handler serves one libp2p connection. Inbound streams read a request, +//! invoke the registered handler callback, and write the response. Outbound +//! requests are delivered from the behaviour as [`FromBehaviour`] commands; +//! each opens its own substream, sends the request, reads the response, and +//! resolves the caller's oneshot. + +use std::{ + collections::VecDeque, + convert::Infallible, + task::{Context, Poll}, +}; + +use futures::{FutureExt, future::BoxFuture}; +use libp2p::{ + PeerId, Stream, + swarm::{ + ConnectionHandler, ConnectionHandlerEvent, StreamUpgradeError, SubstreamProtocol, + handler::{ + ConnectionEvent, DialUpgradeError, FullyNegotiatedInbound, FullyNegotiatedOutbound, + }, + }, +}; +use pluto_core::corepb::v1::priority::PriorityMsg; +use tokio::{sync::oneshot, time::timeout}; +use tracing::{debug, warn}; + +use super::{InboundHandler, protocol}; +use crate::error::Error; + +/// A single outbound request awaiting a fresh substream. +#[derive(Debug)] +pub struct OutboundRequest { + /// The request to send. + pub(crate) request: PriorityMsg, + /// Resolves with the peer's response or a transport error. + pub(crate) response: oneshot::Sender>, +} + +/// Command delivered from the behaviour to a connection handler. +#[derive(Debug)] +pub enum FromBehaviour { + /// Issue an outbound request/response exchange. + SendReceive(OutboundRequest), +} + +type InboundFuture = BoxFuture<'static, ()>; +type OutboundFuture = BoxFuture<'static, ()>; + +/// Per-connection priority protocol handler. +pub struct Handler { + peer_id: PeerId, + inbound_handler: InboundHandler, + /// In-flight inbound stream futures. + inbound: Vec, + /// In-flight outbound exchange futures. + outbound: Vec, + /// Outbound requests awaiting a substream, in arrival order. + pending: VecDeque, +} + +impl Handler { + pub(crate) fn new(peer_id: PeerId, inbound_handler: InboundHandler) -> Self { + Self { + peer_id, + inbound_handler, + inbound: Vec::new(), + outbound: Vec::new(), + pending: VecDeque::new(), + } + } +} + +impl ConnectionHandler for Handler { + type FromBehaviour = FromBehaviour; + type InboundOpenInfo = (); + type InboundProtocol = protocol::PriorityUpgrade; + // The originating request travels with the substream so a negotiated stream + // is paired with the request that opened it, never by negotiation order. + type OutboundOpenInfo = OutboundRequest; + type OutboundProtocol = protocol::PriorityUpgrade; + type ToBehaviour = Infallible; + + fn listen_protocol(&self) -> SubstreamProtocol { + SubstreamProtocol::new(protocol::upgrade(), ()) + } + + fn on_behaviour_event(&mut self, event: Self::FromBehaviour) { + match event { + FromBehaviour::SendReceive(request) => self.pending.push_back(request), + } + } + + fn poll( + &mut self, + cx: &mut Context<'_>, + ) -> Poll< + ConnectionHandlerEvent, + > { + self.inbound + .retain_mut(|fut| fut.poll_unpin(cx).is_pending()); + self.outbound + .retain_mut(|fut| fut.poll_unpin(cx).is_pending()); + + if let Some(request) = self.pending.pop_front() { + return Poll::Ready(ConnectionHandlerEvent::OutboundSubstreamRequest { + protocol: SubstreamProtocol::new(protocol::upgrade(), request), + }); + } + + Poll::Pending + } + + fn on_connection_event( + &mut self, + event: ConnectionEvent< + Self::InboundProtocol, + Self::OutboundProtocol, + Self::InboundOpenInfo, + Self::OutboundOpenInfo, + >, + ) { + match event { + ConnectionEvent::FullyNegotiatedInbound(FullyNegotiatedInbound { + protocol: stream, + .. + }) => { + self.inbound.push( + handle_inbound(self.peer_id, self.inbound_handler.clone(), stream).boxed(), + ); + } + ConnectionEvent::FullyNegotiatedOutbound(FullyNegotiatedOutbound { + protocol: stream, + info: request, + }) => { + self.outbound.push(run_outbound(request, stream).boxed()); + } + ConnectionEvent::DialUpgradeError(DialUpgradeError { + info: request, + error, + }) => { + let _ = request.response.send(Err(dial_error(error))); + } + _ => {} + } + } +} + +/// Serves a single inbound request: read, invoke handler, optionally respond. +/// +/// The request read is bounded so a peer that opens a stream but never writes +/// has its stream dropped rather than pinned for the connection's lifetime. +async fn handle_inbound(peer_id: PeerId, inbound_handler: InboundHandler, mut stream: Stream) { + let request = match timeout( + protocol::RECEIVE_TIMEOUT, + protocol::read_request(&mut stream), + ) + .await + { + Ok(Ok(request)) => request, + Ok(Err(error)) => { + debug!(peer = %peer_id, err = %error, "Error reading priority request"); + return; + } + Err(_) => { + debug!(peer = %peer_id, "Timed out reading priority request"); + return; + } + }; + + if !protocol::check_required_fields(&request) { + warn!(peer = %peer_id, "Received invalid priority message"); + return; + } + + let response = match inbound_handler(peer_id, request).await { + Ok(Some(response)) => response, + Ok(None) => return, + Err(error) => { + warn!(peer = %peer_id, err = %error, "Error handling priority request"); + return; + } + }; + + if let Err(error) = protocol::write_response(&mut stream, &response).await { + debug!(peer = %peer_id, err = %error, "Error writing priority response"); + } +} + +/// Runs a single outbound exchange and resolves the caller's oneshot. +/// +/// The whole write-and-read round-trip is bounded so an unresponsive peer fails +/// the exchange promptly instead of holding the substream open. +async fn run_outbound(request: OutboundRequest, mut stream: Stream) { + let result = match timeout( + protocol::SEND_TIMEOUT, + protocol::send_receive(&mut stream, &request.request), + ) + .await + { + Ok(result) => result.map_err(|error| Error::Transport(error.to_string())), + Err(_) => Err(Error::Transport("exchange timed out".to_owned())), + }; + let _ = request.response.send(result); +} + +fn dial_error(error: StreamUpgradeError) -> Error { + match error { + StreamUpgradeError::NegotiationFailed => Error::Unsupported, + StreamUpgradeError::Timeout => Error::Transport("negotiation timed out".to_owned()), + StreamUpgradeError::Apply(never) => match never {}, + StreamUpgradeError::Io(error) => Error::Transport(error.to_string()), + } +} diff --git a/crates/priority/src/p2p/mod.rs b/crates/priority/src/p2p/mod.rs new file mode 100644 index 00000000..35e2c8a0 --- /dev/null +++ b/crates/priority/src/p2p/mod.rs @@ -0,0 +1,364 @@ +//! libp2p request/response transport for the priority protocol. +//! +//! The transport is split into the user-facing [`Sender`] handle and the +//! libp2p-owned [`Behaviour`]/[`handler::Handler`] runtime objects. It performs +//! a single round-trip per exchange on the priority protocol: +//! +//! - Outbound: [`Sender::send_receive`] sends a [`PriorityMsg`] to a peer and +//! resolves with that peer's [`PriorityMsg`] response. +//! - Inbound: a negotiated stream reads a [`PriorityMsg`], invokes the +//! registered [`InboundHandler`] callback to produce a response, and writes +//! it back. A `None` response closes the stream without replying. +//! +//! [`new`] takes the inbound handler callback (the prioritiser's request +//! handler) and returns the [`Behaviour`] to register with the swarm plus a +//! cloneable [`Sender`] that the prioritiser uses to drive exchanges. + +mod behaviour; +mod handler; +pub(crate) mod protocol; + +use std::sync::Arc; + +use futures::future::BoxFuture; +use libp2p::PeerId; +use pluto_core::corepb::v1::priority::PriorityMsg; +use tokio::sync::{mpsc, oneshot}; + +pub use behaviour::{Behaviour, Event}; +pub use handler::{FromBehaviour, Handler, OutboundRequest}; +pub use protocol::PriorityProtocol; + +use crate::error::Error; + +/// Registered inbound request handler. +/// +/// Invoked with the remote peer id and the received request. Returns the +/// response to send (`Some`), no response (`None`, closing the stream), or an +/// error (logged, stream closed). +pub type InboundHandler = Arc< + dyn Fn(PeerId, PriorityMsg) -> BoxFuture<'static, crate::Result>> + + Send + + Sync + + 'static, +>; + +/// Command sent from a [`Sender`] to the [`Behaviour`]. +pub(crate) enum Command { + /// Send a request to a peer and resolve with its response. + SendReceive { + /// Target peer. + peer: PeerId, + /// Request payload and response channel. + request: OutboundRequest, + }, +} + +/// Cloneable handle used to initiate outbound priority exchanges. +#[derive(Clone)] +pub struct Sender { + command_tx: mpsc::UnboundedSender, +} + +impl Sender { + /// Sends `request` to `peer` and resolves with the peer's response. + /// + /// Errors with [`Error::Shutdown`] if the behaviour has been dropped, and + /// with [`Error::Transport`]/[`Error::Unsupported`] on dial or stream + /// failure. The caller is responsible for applying an exchange timeout. + pub fn send_receive( + &self, + peer: PeerId, + request: PriorityMsg, + ) -> BoxFuture<'static, crate::Result> { + let command_tx = self.command_tx.clone(); + Box::pin(async move { + let (response_tx, response_rx) = oneshot::channel(); + command_tx + .send(Command::SendReceive { + peer, + request: OutboundRequest { + request, + response: response_tx, + }, + }) + .map_err(|_| Error::Shutdown)?; + + response_rx.await.map_err(|_| Error::Shutdown)? + }) + } +} + +/// Creates the priority transport behaviour and an outbound [`Sender`]. +/// +/// `inbound_handler` is invoked for every received request on this protocol. +pub fn new(inbound_handler: InboundHandler) -> (Behaviour, Sender) { + let (command_tx, command_rx) = mpsc::unbounded_channel(); + let behaviour = Behaviour::new(inbound_handler, command_rx); + let sender = Sender { command_tx }; + (behaviour, sender) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use futures::{FutureExt, StreamExt}; + use libp2p::{ + Multiaddr, Swarm, + core::{Transport as _, transport::MemoryTransport, upgrade::Version}, + multiaddr::Protocol, + swarm::SwarmEvent, + }; + use pluto_core::corepb::v1::{core::Duty, priority::PriorityMsg}; + use pluto_p2p::{peer::peer_id_from_key, utils::keypair_from_secret_key}; + use pluto_testutil::random::generate_insecure_k1_key; + use tokio::time::timeout; + + use super::*; + + fn priority_msg(peer_id: &str) -> PriorityMsg { + PriorityMsg { + duty: Some(Duty { slot: 1, r#type: 0 }), + topics: Vec::new(), + peer_id: peer_id.to_owned(), + signature: Default::default(), + } + } + + /// In-process `/memory/` address, where `N` is derived from the seed + /// (non-zero so the kernel does not auto-assign a port). + fn memory_addr(seed: u8) -> Multiaddr { + Multiaddr::empty().with(Protocol::Memory(u64::from(seed) + 1)) + } + + struct TestNode { + swarm: Swarm, + sender: Sender, + addr: Multiaddr, + } + + /// Builds a swarm over an in-process [`MemoryTransport`] whose priority + /// behaviour responds to inbound requests with `responder(peer, request)`. + /// The libp2p identity is derived from the same secp256k1 key used for the + /// peer id, so the dialed peer id matches. + fn build_node(seed: u8, responder: F) -> TestNode + where + F: Fn(PeerId, PriorityMsg) -> Option + Send + Sync + 'static, + { + let key = generate_insecure_k1_key(seed); + let keypair = keypair_from_secret_key(key).expect("keypair"); + + let inbound: InboundHandler = Arc::new(move |peer, request| { + let response = responder(peer, request); + async move { Ok(response) }.boxed() + }); + let (behaviour, sender) = new(inbound); + + let swarm = libp2p::SwarmBuilder::with_existing_identity(keypair) + .with_tokio() + .with_other_transport(|key| { + MemoryTransport::default() + .upgrade(Version::V1) + .authenticate(libp2p::noise::Config::new(key).expect("noise config")) + .multiplex(libp2p::yamux::Config::default()) + }) + .expect("transport") + .with_behaviour(|_key| behaviour) + .expect("behaviour") + .build(); + + TestNode { + swarm, + sender, + addr: memory_addr(seed), + } + } + + #[tokio::test] + async fn send_receive_without_behaviour_returns_shutdown() { + let (_behaviour, sender) = new(Arc::new(|_, _| async { Ok(None) }.boxed())); + // Dropping the behaviour closes the command channel. + drop(_behaviour); + let peer = PeerId::random(); + let error = sender + .send_receive(peer, priority_msg("x")) + .await + .expect_err("send should fail without a running behaviour"); + assert!(matches!(error, Error::Shutdown)); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn round_trip_returns_peer_response() { + let peer_a = peer_id_from_key(generate_insecure_k1_key(0).public_key()).expect("peer a id"); + let peer_b = peer_id_from_key(generate_insecure_k1_key(1).public_key()).expect("peer b id"); + + // Node B echoes the request's peer id back inside its own response. + let responder_peer_b = peer_b.to_string(); + let mut node_b = build_node(1, move |_peer, request| { + Some(PriorityMsg { + peer_id: responder_peer_b.clone(), + ..request + }) + }); + let mut node_a = build_node(0, |_peer, _request| Some(priority_msg("unused"))); + + node_a + .swarm + .listen_on(node_a.addr.clone()) + .expect("listen a"); + node_b + .swarm + .listen_on(node_b.addr.clone()) + .expect("listen b"); + + // Wait for both nodes to start listening. + for swarm in [&mut node_a.swarm, &mut node_b.swarm] { + loop { + if matches!( + swarm.select_next_some().await, + SwarmEvent::NewListenAddr { .. } + ) { + break; + } + } + } + + node_a.swarm.dial(node_b.addr.clone()).expect("dial b"); + + // Drive node B in the background while node A waits for the dialed + // connection to establish. The behaviour only knows peer ids, not + // addresses, so the outbound exchange must reuse an existing connection + // rather than re-dialing by peer id (which has no known address). + let sender_a = node_a.sender.clone(); + let mut swarm_a = node_a.swarm; + let mut swarm_b = node_b.swarm; + let driver_b = tokio::spawn(async move { + loop { + let _ = swarm_b.select_next_some().await; + } + }); + loop { + if matches!( + swarm_a.select_next_some().await, + SwarmEvent::ConnectionEstablished { peer_id, .. } if peer_id == peer_b + ) { + break; + } + } + let driver_a = tokio::spawn(async move { + loop { + let _ = swarm_a.select_next_some().await; + } + }); + + let request = priority_msg(&peer_a.to_string()); + let response = timeout( + Duration::from_secs(10), + sender_a.send_receive(peer_b, request), + ) + .await + .expect("exchange should complete") + .expect("exchange should succeed"); + + assert_eq!(response.peer_id, peer_b.to_string()); + assert_eq!(response.duty, Some(Duty { slot: 1, r#type: 0 })); + + driver_a.abort(); + driver_b.abort(); + } + + /// Concurrent exchanges to the same peer resolve each caller's oneshot with + /// its own request's response, never by stream-negotiation order. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_same_peer_exchanges_route_by_identity() { + // Distinct seeds from the round-trip test so the in-process memory + // addresses do not collide when tests run in parallel. + let peer_b = peer_id_from_key(generate_insecure_k1_key(3).public_key()).expect("peer b id"); + + // Node B echoes the request's duty (its slot distinguishes requests) and + // stamps its own peer id on the response. + let responder_peer_b = peer_b.to_string(); + let mut node_b = build_node(3, move |_peer, request| { + Some(PriorityMsg { + peer_id: responder_peer_b.clone(), + duty: request.duty, + ..request + }) + }); + let mut node_a = build_node(2, |_peer, _request| Some(priority_msg("unused"))); + + node_a + .swarm + .listen_on(node_a.addr.clone()) + .expect("listen a"); + node_b + .swarm + .listen_on(node_b.addr.clone()) + .expect("listen b"); + + for swarm in [&mut node_a.swarm, &mut node_b.swarm] { + loop { + if matches!( + swarm.select_next_some().await, + SwarmEvent::NewListenAddr { .. } + ) { + break; + } + } + } + + node_a.swarm.dial(node_b.addr.clone()).expect("dial b"); + + let sender_a = node_a.sender.clone(); + let mut swarm_a = node_a.swarm; + let mut swarm_b = node_b.swarm; + let driver_b = tokio::spawn(async move { + loop { + let _ = swarm_b.select_next_some().await; + } + }); + loop { + if matches!( + swarm_a.select_next_some().await, + SwarmEvent::ConnectionEstablished { peer_id, .. } if peer_id == peer_b + ) { + break; + } + } + let driver_a = tokio::spawn(async move { + loop { + let _ = swarm_a.select_next_some().await; + } + }); + + // Issue many concurrent exchanges to the same peer, each carrying a + // distinct slot. Each response must echo the slot of its own request. + let slots: Vec = (100..110).collect(); + let mut requests = Vec::new(); + for &slot in &slots { + let req = PriorityMsg { + duty: Some(Duty { slot, r#type: 0 }), + ..priority_msg("x") + }; + requests.push(sender_a.send_receive(peer_b, req)); + } + + let responses = timeout(Duration::from_secs(10), futures::future::join_all(requests)) + .await + .expect("all exchanges complete"); + + for (slot, response) in slots.iter().zip(responses) { + let response = response.expect("exchange should succeed"); + assert_eq!(response.peer_id, peer_b.to_string()); + assert_eq!( + response.duty.expect("duty echoed").slot, + *slot, + "response must match its own request slot" + ); + } + + driver_a.abort(); + driver_b.abort(); + } +} diff --git a/crates/priority/src/p2p/protocol.rs b/crates/priority/src/p2p/protocol.rs new file mode 100644 index 00000000..aaf166f2 --- /dev/null +++ b/crates/priority/src/p2p/protocol.rs @@ -0,0 +1,197 @@ +//! Wire protocol for the priority request/response protocol. +//! +//! A single round-trip exchanges one [`PriorityMsg`] request for one +//! [`PriorityMsg`] response, length-delimited on the wire as +//! `[unsigned varint length][protobuf bytes]`. + +use std::time::Duration; + +use libp2p::{core::upgrade::ReadyUpgrade, swarm::Stream}; +use pluto_core::corepb::v1::priority::PriorityMsg; + +use crate::PROTOCOL_ID; + +/// Wire token negotiated for the priority protocol. +/// +/// The canonical protocol identifier is the slug-less [`PROTOCOL_ID`]. The +/// negotiated token derives from it directly, so [`PROTOCOL_ID`] is the single +/// source of truth. +/// +/// `libp2p`'s multistream-select requires every negotiated protocol token to +/// begin with `/` and rejects any other form before it reaches the wire, so the +/// token offered for negotiation is [`PROTOCOL_ID`] with a leading `/`. This is +/// the one place where the wire form differs from the canonical identifier. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PriorityProtocol; + +/// Negotiated wire token: the canonical identifier with the leading `/` that +/// multistream-select mandates. Tied to [`PROTOCOL_ID`] by a compile-time +/// assertion below, keeping the canonical identifier the single source of +/// truth. +const WIRE_TOKEN: &str = "/charon/priority/2.0.0"; + +/// The wire token must be the canonical identifier prefixed with `/`. +const _: () = { + let id = PROTOCOL_ID.as_bytes(); + let wire = WIRE_TOKEN.as_bytes(); + assert!( + wire.len() == id.len() + 1, + "wire token must be /" + ); + assert!(wire[0] == b'/', "wire token must start with /"); + let mut i = 0; + while i < id.len() { + assert!(wire[i + 1] == id[i], "wire token must equal /"); + i += 1; + } +}; + +impl AsRef for PriorityProtocol { + fn as_ref(&self) -> &str { + WIRE_TOKEN + } +} + +/// Upgrade negotiating the priority protocol on inbound and outbound streams. +pub(crate) type PriorityUpgrade = ReadyUpgrade; + +/// Returns the upgrade used to negotiate the priority protocol. +pub(crate) fn upgrade() -> PriorityUpgrade { + ReadyUpgrade::new(PriorityProtocol) +} + +/// Maximum protobuf message size (128MB). +pub(crate) const MAX_MESSAGE_SIZE: usize = 128 << 20; + +/// Maximum time a peer is given to deliver an inbound request. +/// +/// A peer that opens a stream but does not write within this window has its +/// stream dropped. +pub(crate) const RECEIVE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Maximum time for a full outbound exchange (open, write, read). +/// +/// Exceeds [`RECEIVE_TIMEOUT`] by the round-trip hop allowance, matching the +/// send deadline applied to the whole request/response round-trip. +pub(crate) const SEND_TIMEOUT: Duration = Duration::from_secs(7); + +/// Sends a request and reads the peer's response on a fresh outbound stream. +pub(crate) async fn send_receive( + stream: &mut Stream, + request: &PriorityMsg, +) -> std::io::Result { + pluto_p2p::proto::write_protobuf(stream, request).await?; + pluto_p2p::proto::read_protobuf_with_max_size(stream, MAX_MESSAGE_SIZE).await +} + +/// Reads an inbound request from a stream. +pub(crate) async fn read_request(stream: &mut Stream) -> std::io::Result { + pluto_p2p::proto::read_protobuf_with_max_size(stream, MAX_MESSAGE_SIZE).await +} + +/// Rejects a decoded request that omits a required message field. +/// +/// Applies the pre-handler proto validation to received messages: any +/// non-optional nested message field that is absent makes the whole message +/// invalid. For [`PriorityMsg`] the absent-field cases reachable from the wire +/// are the `duty` field and the `topic` of any proposed topic; an empty +/// `topics` or `priorities` list is valid. +pub(crate) fn check_required_fields(msg: &PriorityMsg) -> bool { + if msg.duty.is_none() { + return false; + } + + msg.topics.iter().all(|proposal| proposal.topic.is_some()) +} + +/// Writes a response to a stream. +pub(crate) async fn write_response( + stream: &mut Stream, + response: &PriorityMsg, +) -> std::io::Result<()> { + pluto_p2p::proto::write_protobuf(stream, response).await +} + +#[cfg(test)] +mod tests { + use pluto_core::corepb::v1::{ + core::Duty, + priority::{PriorityMsg, PriorityTopicProposal}, + }; + use prost_types::Any; + + use super::*; + + /// The canonical protocol identifier carries no leading slug, exactly as + /// the reference implementation registers it. + #[test] + fn canonical_protocol_id_has_no_leading_slash() { + assert_eq!(PROTOCOL_ID, "charon/priority/2.0.0"); + assert!(!PROTOCOL_ID.starts_with('/')); + } + + /// The token offered for negotiation is the canonical identifier with the + /// leading `/` that multistream-select mandates; no other divergence. + #[test] + fn wire_token_is_canonical_id_with_leading_slash() { + assert_eq!(PriorityProtocol.as_ref(), "/charon/priority/2.0.0"); + assert_eq!(PriorityProtocol.as_ref(), format!("/{PROTOCOL_ID}")); + } + + fn any() -> Any { + Any { + type_url: "type.googleapis.com/google.protobuf.Value".to_owned(), + value: Vec::new(), + } + } + + #[test] + fn required_fields_accepts_present_fields() { + let msg = PriorityMsg { + duty: Some(Duty { slot: 1, r#type: 0 }), + topics: vec![PriorityTopicProposal { + topic: Some(any()), + priorities: vec![any()], + }], + peer_id: "p".to_owned(), + signature: Default::default(), + }; + assert!(check_required_fields(&msg)); + } + + #[test] + fn required_fields_rejects_missing_duty() { + let msg = PriorityMsg { + duty: None, + topics: Vec::new(), + peer_id: "p".to_owned(), + signature: Default::default(), + }; + assert!(!check_required_fields(&msg)); + } + + #[test] + fn required_fields_rejects_missing_topic_any() { + let msg = PriorityMsg { + duty: Some(Duty { slot: 1, r#type: 0 }), + topics: vec![PriorityTopicProposal { + topic: None, + priorities: Vec::new(), + }], + peer_id: "p".to_owned(), + signature: Default::default(), + }; + assert!(!check_required_fields(&msg)); + } + + #[test] + fn required_fields_accepts_empty_topics() { + let msg = PriorityMsg { + duty: Some(Duty { slot: 1, r#type: 0 }), + topics: Vec::new(), + peer_id: "p".to_owned(), + signature: Default::default(), + }; + assert!(check_required_fields(&msg)); + } +} diff --git a/crates/priority/src/prioritiser.rs b/crates/priority/src/prioritiser.rs new file mode 100644 index 00000000..c79ab891 --- /dev/null +++ b/crates/priority/src/prioritiser.rs @@ -0,0 +1,864 @@ +//! Priority protocol engine: per-duty exchange and consensus orchestration. +//! +//! [`Prioritiser`] resolves cluster-wide priorities for a duty in two steps: +//! first it exchanges its own signed [`PriorityMsg`] with all peers and +//! collects their responses (until all received or the exchange timeout +//! elapses), then it deterministically computes a [`PriorityResult`] and +//! proposes it through cluster [`Consensus`]. +//! +//! The engine is built on tokio primitives: a per-duty request buffer +//! ([`mpsc`]) feeds the [`run_instance`] select loop, +//! peer exchanges run as spawned tasks writing into a shared responses channel, +//! request responses travel over [`oneshot`] channels, and shutdown is +//! signalled by a [`CancellationToken`]. + +use std::{ + collections::HashMap, + sync::{Arc, Mutex, OnceLock}, + time::Duration, +}; + +use futures::FutureExt; +use libp2p::PeerId; +use pluto_core::{ + corepb::v1::{core::Duty as ProtoDuty, priority::PriorityMsg}, + deadline::{AddOutcome, DeadlinerHandle}, + types::{Duty, DutyType, SlotNumber}, +}; +use tokio::sync::{mpsc, oneshot}; +use tokio_util::sync::CancellationToken; + +use crate::{ + calculate::calculate_result, + component::MsgVerifier, + consensus::{Consensus, PrioritySubscriber}, + error::{Error, Result}, + p2p::{self, Behaviour, InboundHandler, Sender, protocol::RECEIVE_TIMEOUT}, +}; + +/// Supported priority protocol identifier. +pub const PROTOCOL_ID: &str = "charon/priority/2.0.0"; + +/// A received peer request paired with a channel to deliver this peer's reply. +struct Request { + /// The peer's priority message. + msg: PriorityMsg, + /// Channel on which to send our own message back to the peer. + response: oneshot::Sender, +} + +/// Output subscriber callback invoked with each decided priority result. +type Subscriber = PrioritySubscriber; + +/// Per-duty request buffer: a cloneable sender shared with peer handlers and a +/// receiver consumed once by the duty's run loop. +struct BufferEntry { + /// Sender clones handed to inbound request handlers. + tx: mpsc::Sender, + /// Receiver, taken by the single run loop for the duty. + rx: Option>, +} + +/// Returns a [`Duty`] from its proto form, tolerating any encoded duty type. +/// +/// The conversion is infallible and preserves the raw type integer; +/// out-of-range values map to [`DutyType::Unknown`] so engine bookkeeping never +/// rejects a message that passed validation. +fn duty_from_proto(duty: &ProtoDuty) -> Duty { + let duty_type = DutyType::try_from(duty.r#type).unwrap_or(DutyType::Unknown); + Duty::new(SlotNumber::new(duty.slot), duty_type) +} + +/// Returns the proto form of a [`Duty`], preserving the type integer. +/// +/// The conversion is infallible; a duty type with no proto integer maps to the +/// unknown encoding (`0`), the inverse of [`duty_from_proto`]'s tolerance, so a +/// proto value round-trips through both without rejecting the duty. +pub(crate) fn duty_to_proto(duty: &Duty) -> ProtoDuty { + ProtoDuty { + slot: duty.slot.inner(), + r#type: i32::try_from(&duty.duty_type).unwrap_or(0), + } +} + +/// Shared, immutable engine state referenced by both the public API and the +/// inbound request handler. +struct Inner { + /// Local peer id; excluded when exchanging with peers. + local_id: PeerId, + /// Outbound request/response transport (the p2p send-receive seam). + sender: Sender, + /// Minimum number of peers that must propose a priority to include it. + min_required: i64, + /// Per-instance exchange timeout before falling back to consensus. + exchange_timeout: Duration, + /// Cluster peers participating in the protocol. + peers: Vec, + /// Cluster consensus over priority results. + consensus: Arc, + /// Validates received messages (peer membership + signature). + msg_validator: Arc, + /// Cancelled when the engine shuts down, signalling instances to stop. + quit: CancellationToken, + /// Deadline scheduler; expired duties drop their request buffers. + deadliner: DeadlinerHandle, + /// Per-duty request buffers feeding each instance's run loop. + req_buffers: Mutex>, +} + +impl Inner { + /// Request buffer capacity: `2 * peers` so neither peer requests nor our + /// responses block the run loop. + fn buffer_capacity(&self) -> usize { + self.peers.len().max(1).saturating_mul(2) + } + + /// Returns the request sender for a duty, creating the buffer on first use. + fn get_req_buffer(&self, duty: Duty) -> mpsc::Sender { + let cap = self.buffer_capacity(); + let mut buffers = self.req_buffers.lock().expect("req_buffers mutex poisoned"); + buffers + .entry(duty) + .or_insert_with(|| { + let (tx, rx) = mpsc::channel(cap); + BufferEntry { tx, rx: Some(rx) } + }) + .tx + .clone() + } + + /// Takes the receiver for a duty's run loop, creating the buffer if absent. + /// + /// Returns `None` if a run loop already took it (one instance per duty). + fn take_req_receiver(&self, duty: Duty) -> Option> { + let cap = self.buffer_capacity(); + let mut buffers = self.req_buffers.lock().expect("req_buffers mutex poisoned"); + buffers + .entry(duty) + .or_insert_with(|| { + let (tx, rx) = mpsc::channel(cap); + BufferEntry { tx, rx: Some(rx) } + }) + .rx + .take() + } + + /// Drops the request buffer for an expired duty. + fn delete_recv_buffer(&self, duty: Duty) { + self.req_buffers + .lock() + .expect("req_buffers mutex poisoned") + .remove(&duty); + } + + /// Handles a priority message exchange initiated by a peer. + /// + /// Validates the message, enqueues it for the duty's run loop, and awaits + /// this node's own message to return as the response. Returns + /// [`Error::Shutdown`] if the engine stops while waiting. + async fn handle_request(&self, peer: PeerId, msg: PriorityMsg) -> Result { + if peer.to_string() != msg.peer_id { + return Err(Error::InvalidPeerId); + } + + (self.msg_validator)(&msg)?; // Arc> auto-derefs for call. + + let proto_duty = msg.duty.as_ref().ok_or(Error::InvalidMsgProtoFields)?; + let duty = duty_from_proto(proto_duty); + + if self.deadliner.add(duty.clone()).await != AddOutcome::Scheduled { + return Err(Error::DutyExpired); + } + + let buffer = self.get_req_buffer(duty); + let (response_tx, response_rx) = oneshot::channel(); + let req = Request { + msg, + response: response_tx, + }; + + // The enqueue and response-wait phases share a single receive-timeout + // deadline: a peer that opens a stream cannot pin the handler past it. + let deadline = tokio::time::sleep(RECEIVE_TIMEOUT); + tokio::pin!(deadline); + + tokio::select! { + send_res = buffer.send(req) => { + send_res.map_err(|_| Error::Shutdown)?; + } + () = &mut deadline => return Err(Error::TimeoutEnqueuing), + () = self.quit.cancelled() => return Err(Error::Shutdown), + } + + tokio::select! { + resp = response_rx => resp.map_err(|_| Error::Shutdown), + () = &mut deadline => Err(Error::TimeoutWaiting), + () = self.quit.cancelled() => Err(Error::Shutdown), + } + } +} + +/// Resolves cluster-wide priorities for duties. +#[derive(Clone)] +pub struct Prioritiser { + inner: Arc, + /// Output subscribers; appended via [`Prioritiser::subscribe`] before + /// [`Prioritiser::start`]. Wrapped so the consensus subscription can read + /// the current set without sharing ownership at construction time. + subs: Arc>>>, +} + +impl Prioritiser { + /// Constructs a prioritiser and its libp2p transport behaviour. + /// + /// Returns the [`Prioritiser`] plus the [`Behaviour`] the caller must + /// register with the swarm. The behaviour's inbound handler dispatches into + /// [`Inner::handle_request`]. + #[allow(clippy::too_many_arguments)] + pub fn new_internal( + local_id: PeerId, + peers: Vec, + min_required: i64, + consensus: Arc, + msg_validator: MsgVerifier, + exchange_timeout: Duration, + deadliner: DeadlinerHandle, + ) -> (Self, Behaviour) { + // The transport's inbound handler needs the engine state, which in turn + // needs the transport's outbound `Sender`. Break the cycle with a slot + // the handler reads lazily; it is filled before the behaviour can + // receive any request (the swarm is not yet driving it). + let inner_slot: Arc>> = Arc::new(OnceLock::new()); + let handler_slot = inner_slot.clone(); + + let inbound: InboundHandler = Arc::new(move |peer, msg| { + let slot = handler_slot.clone(); + async move { + let inner = slot + .get() + .expect("inbound handler invoked before engine initialised"); + inner.handle_request(peer, msg).await.map(Some) + } + .boxed() + }); + + let (behaviour, sender) = p2p::new(inbound); + + let inner = Arc::new(Inner { + local_id, + sender, + min_required, + exchange_timeout, + peers, + consensus: consensus.clone(), + msg_validator: Arc::new(msg_validator), + quit: CancellationToken::new(), + deadliner, + req_buffers: Mutex::new(HashMap::new()), + }); + inner_slot + .set(inner.clone()) + .unwrap_or_else(|_| unreachable!("inner slot set once")); + + let subs: Arc>>> = Arc::new(Mutex::new(Vec::new())); + + // Wire consensus output to our subscribers: invoke each in order, + // returning on the first error. + let subs_for_consensus = subs.clone(); + consensus.subscribe_priority(Box::new(move |duty, result| { + let subs = subs_for_consensus + .lock() + .expect("subscribers mutex poisoned") + .clone(); + for sub in &subs { + sub(duty.clone(), result.clone())?; + } + Ok(()) + })); + + (Self { inner, subs }, behaviour) + } + + /// Starts the background loop that drops request buffers for expired + /// duties. + /// + /// Must be called exactly once. The returned task runs until `ctx` is + /// cancelled, then signals engine shutdown via the internal quit token. + /// `expired` is the deadliner's expired-duty receiver. + pub fn start(&self, ctx: CancellationToken, mut expired: mpsc::Receiver) { + let inner = self.inner.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + () = ctx.cancelled() => break, + maybe = expired.recv() => match maybe { + Some(duty) => inner.delete_recv_buffer(duty), + None => break, + }, + } + } + // Cancelling `quit` on exit unblocks any in-flight `handle_request`. + inner.quit.cancel(); + }); + } + + /// Registers an output subscriber invoked with each decided result. + /// + /// Not thread safe relative to a running instance; call before any + /// [`Prioritiser::prioritise`]. + pub fn subscribe(&self, sub: Subscriber) { + self.subs + .lock() + .expect("subscribers mutex poisoned") + .push(Arc::new(sub)); + } + + /// Starts a new prioritisation instance for `msg` and runs it to + /// completion. + /// + /// Drops the instance silently (returns `Ok`) if the duty has already + /// expired. Otherwise blocks until consensus is proposed and `ctx` is + /// cancelled, returning [`Error::Cancelled`] on cancellation. + pub async fn prioritise(&self, ctx: CancellationToken, msg: PriorityMsg) -> Result<()> { + let proto_duty = msg.duty.as_ref().ok_or(Error::InvalidMsgProtoFields)?; + let duty = duty_from_proto(proto_duty); + + if self.inner.deadliner.add(duty.clone()).await != AddOutcome::Scheduled { + tracing::warn!(%duty, "Dropping priority protocol instance for expired duty"); + return Ok(()); + } + + let requests = self + .inner + .take_req_receiver(duty.clone()) + .expect("one prioritise instance per duty"); + run_instance(ctx, &self.inner, duty, msg, requests).await + } +} + +/// Runs a single priority instance: exchange messages, respond to peer +/// requests, and start consensus once all messages are collected or the +/// exchange timeout elapses. +/// +/// Blocks until `ctx` is cancelled (returning [`Error::Cancelled`]) or a +/// consensus calculation fails. +async fn run_instance( + ctx: CancellationToken, + inner: &Inner, + duty: Duty, + own: PriorityMsg, + mut request_rx: mpsc::Receiver, +) -> Result<()> { + tracing::debug!(%duty, "Priority protocol instance started"); + + // Seed `msgs` with our own message but leave the dedup set empty, so a + // (rejected-by-validation) duplicate of our peer id surfaces in + // `calculate_result` as a duplicate-peer error rather than being silently + // swallowed. + let mut msgs: Vec = vec![own.clone()]; + let mut dedup_peers: HashMap = HashMap::new(); + + let mut cons_started = false; + + let (responses_tx, mut responses_rx) = mpsc::channel::(inner.peers.len().max(1)); + + let exchange_timeout = tokio::time::sleep(inner.exchange_timeout); + tokio::pin!(exchange_timeout); + + exchange(&ctx, inner, responses_tx, own.clone()); + + loop { + let mut should_start_consensus = false; + + tokio::select! { + () = ctx.cancelled() => return Err(Error::Cancelled), + // Matching-pattern arms: when every sender drops the channel + // closes, recv resolves to None, and the arm is disabled rather + // than completing the select and re-looping. A disabled arm behaves + // like a channel that simply never becomes ready again. + Some(req) = request_rx.recv() => { + add_msg(&mut msgs, &mut dedup_peers, req.msg); + // Respond with our own message; the buffer guarantees the + // receiver never blocks, so a dropped peer is harmless. + let _ = req.response.send(own.clone()); + } + Some(msg) = responses_rx.recv() => { + add_msg(&mut msgs, &mut dedup_peers, msg); + } + () = &mut exchange_timeout, if !cons_started => { + tracing::debug!(%duty, "Priority protocol instance exchange timeout, starting consensus"); + should_start_consensus = true; + } + } + + if should_start_consensus { + cons_started = true; + start_consensus(&ctx, inner, &duty, &msgs)?; + } + + if !cons_started && msgs.len() == inner.peers.len() { + tracing::debug!(%duty, "Priority protocol instance messages exchanged, starting consensus"); + cons_started = true; + start_consensus(&ctx, inner, &duty, &msgs)?; + } + } +} + +/// Adds the first message seen from each peer to `msgs`. +fn add_msg(msgs: &mut Vec, dedup: &mut HashMap, msg: PriorityMsg) { + if dedup.contains_key(&msg.peer_id) { + return; + } + dedup.insert(msg.peer_id.clone(), ()); + msgs.push(msg); +} + +/// Initiates a priority message exchange with every peer except self. +/// +/// Each peer is handled in its own task: send our message, validate the +/// response's peer id and signature, then forward it to the run loop. Failures +/// are dropped silently (the transport logs them). +fn exchange( + ctx: &CancellationToken, + inner: &Inner, + responses: mpsc::Sender, + own: PriorityMsg, +) { + for &peer in &inner.peers { + if peer == inner.local_id { + continue; + } + + let ctx = ctx.clone(); + let sender = inner.sender.clone(); + let validator = inner.msg_validator.clone(); + let responses = responses.clone(); + let own = own.clone(); + + tokio::spawn(async move { + let send = sender.send_receive(peer, own); + let response = tokio::select! { + () = ctx.cancelled() => return, + res = send => match res { + Ok(resp) => resp, + Err(_) => return, // Transport already logged. + }, + }; + + if peer.to_string() != response.peer_id { + tracing::warn!(%peer, "Invalid priority message peer id"); + return; + } + + if let Err(err) = validator(&response) { + tracing::warn!(%peer, %err, "Invalid priority message from peer"); + return; + } + + tokio::select! { + () = ctx.cancelled() => {} + _ = responses.send(response) => {} + } + }); + } +} + +/// Calculates the deterministic result and proposes it through consensus. +/// +/// Consensus runs in a spawned task because it blocks until agreement while the +/// instance must keep servicing peer requests. +fn start_consensus( + ctx: &CancellationToken, + inner: &Inner, + duty: &Duty, + msgs: &[PriorityMsg], +) -> Result<()> { + let result = calculate_result(msgs, inner.min_required) + .map_err(|e| Error::CalculateResult(Box::new(e)))?; + + let consensus = inner.consensus.clone(); + let duty = duty.clone(); + let ctx = ctx.clone(); + tokio::spawn(async move { + // Fire-and-forget so the instance keeps servicing peer requests while + // consensus runs. The instance token reaches consensus, so cancellation + // tears the proposal down; a propose failure is unexpected. + if let Err(err) = consensus.propose_priority(duty, result, &ctx).await { + tracing::warn!(%err, "Priority protocol consensus"); + } + }); + + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex as StdMutex; + + use chrono::{Duration as ChronoDuration, Utc}; + use pluto_core::{ + corepb::v1::priority::PriorityResult, + deadline::{DeadlineCalculator, DeadlinerTask}, + }; + + use super::*; + use crate::{ + component::{TopicProposal, new_msg_verifier, sign_msg, topic_proposal_to_proto}, + consensus::ConsensusError, + }; + + /// Calculator reporting every duty as expiring one hour from now, so the + /// deadliner schedules (does not drop) duties under test. + struct FutureCalculator; + + impl DeadlineCalculator for FutureCalculator { + fn deadline( + &self, + _duty: &Duty, + ) -> pluto_core::deadline::Result>> { + Ok(Some( + Utc::now() + .checked_add_signed(ChronoDuration::hours(1)) + .expect("deadline in range"), + )) + } + } + + /// Mock consensus that decides on the first proposal by invoking + /// subscribers and records every proposed result for assertions. + #[derive(Default)] + struct MockConsensus { + subs: StdMutex>, + proposed: Arc>>, + } + + #[async_trait::async_trait] + impl Consensus for MockConsensus { + async fn propose_priority( + &self, + duty: Duty, + result: PriorityResult, + _ct: &CancellationToken, + ) -> std::result::Result<(), ConsensusError> { + let first = { + let mut proposed = self.proposed.lock().expect("proposed mutex"); + let first = proposed.is_empty(); + proposed.push((duty.clone(), result.clone())); + first + }; + if first { + let subs = self.subs.lock().expect("subs mutex"); + for sub in subs.iter() { + sub(duty.clone(), result.clone()).map_err(|e| -> ConsensusError { e })?; + } + } + Ok(()) + } + + fn subscribe_priority(&self, callback: PrioritySubscriber) { + self.subs.lock().expect("subs mutex").push(callback); + } + } + + fn key_and_peer(seed: u8) -> (k256::SecretKey, PeerId) { + let key = pluto_testutil::random::generate_insecure_k1_key(seed); + let peer = pluto_p2p::peer::peer_id_from_key(key.public_key()).expect("peer id"); + (key, peer) + } + + fn build_msg(key: &k256::SecretKey, peer: PeerId, prio: &str) -> PriorityMsg { + let msg = PriorityMsg { + duty: Some(ProtoDuty { + slot: 97, + r#type: 0, + }), + topics: vec![topic_proposal_to_proto(&TopicProposal { + topic: "topic".to_owned(), + priorities: vec![prio.to_owned()], + })], + peer_id: peer.to_string(), + signature: Default::default(), + }; + sign_msg(&msg, key).expect("sign") + } + + /// Single-node instance reaches consensus on the exchange timeout and + /// delivers the decided result to the prioritiser's subscriber. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn single_node_prioritise_decides_and_notifies() { + let (key, peer) = key_and_peer(0); + let peers = vec![peer]; + let validator = new_msg_verifier(&peers).expect("verifier"); + + let consensus = Arc::new(MockConsensus::default()); + let proposed = consensus.proposed.clone(); + + let ct = CancellationToken::new(); + let (deadliner, expired) = DeadlinerTask::start(ct.clone(), "test", FutureCalculator); + + // A single-node instance has no peers to exchange with, so it starts + // consensus on the exchange timeout (matching the reference, which + // never short-circuits the empty exchange). Keep the timeout short so + // the test decides promptly. + let (prio, _behaviour) = Prioritiser::new_internal( + peer, + peers, + 1, + consensus, + validator, + Duration::from_millis(100), + deadliner, + ); + + let (result_tx, mut result_rx) = mpsc::unbounded_channel(); + prio.subscribe(Box::new(move |duty, result| { + let _ = result_tx.send((duty, result)); + Ok(()) + })); + prio.start(ct.clone(), expired); + + let msg = build_msg(&key, peer, "v1"); + let run_ct = ct.clone(); + let handle = tokio::spawn(async move { prio.prioritise(run_ct, msg).await }); + + let (duty, result) = tokio::time::timeout(Duration::from_secs(5), result_rx.recv()) + .await + .expect("subscriber notified") + .expect("result delivered"); + assert_eq!(duty.slot, SlotNumber::new(97)); + assert_eq!(result.topics.len(), 1); + + // The mock recorded exactly one proposal. + assert_eq!(proposed.lock().expect("proposed").len(), 1); + + // Cancelling returns the cancellation error from the run loop. + ct.cancel(); + let res = tokio::time::timeout(Duration::from_secs(5), handle) + .await + .expect("join") + .expect("task"); + assert!(matches!(res, Err(Error::Cancelled))); + } + + /// The run loop makes progress only on real events, never by busy-spinning, + /// and cancellation completes promptly. + /// + /// Uses a paused clock: tokio auto-advances virtual time only when the + /// runtime is otherwise idle (every task parked). The assertions rely on + /// that idleness — they would not resolve if the loop spun. Concretely the + /// empty single-peer exchange does not start consensus until the exchange + /// timeout fires (event-driven progress), the loop then parks (it does not + /// complete on its own), and cancellation ends it at once. + #[tokio::test(start_paused = true)] + async fn run_instance_makes_progress_only_on_events() { + let (key, peer) = key_and_peer(0); + let peers = vec![peer]; + let validator = new_msg_verifier(&peers).expect("verifier"); + let consensus = Arc::new(MockConsensus::default()); + let proposed = consensus.proposed.clone(); + + let ct = CancellationToken::new(); + let (deadliner, expired) = DeadlinerTask::start(ct.clone(), "test", FutureCalculator); + let exchange_timeout = Duration::from_secs(2); + let (prio, _behaviour) = Prioritiser::new_internal( + peer, + peers, + 1, + consensus, + validator, + exchange_timeout, + deadliner, + ); + prio.start(ct.clone(), expired); + + let msg = build_msg(&key, peer, "v1"); + let run_ct = ct.clone(); + let handle = tokio::spawn(async move { prio.prioritise(run_ct, msg).await }); + + // Before the exchange timeout the empty exchange yields no event, so no + // consensus is proposed. The clock only advances here because the loop + // is parked rather than spinning. + tokio::time::sleep(exchange_timeout / 2).await; + assert_eq!(proposed.lock().expect("proposed").len(), 0); + + // After the exchange timeout the loop wakes on that single event and + // proposes exactly once, then parks again. + tokio::time::sleep(exchange_timeout).await; + assert_eq!(proposed.lock().expect("proposed").len(), 1); + assert!(!handle.is_finished()); + + ct.cancel(); + let res = handle.await.expect("task joins"); + assert!(matches!(res, Err(Error::Cancelled))); + } + + /// `handle_request` does not pin a stream indefinitely: with no run loop to + /// fulfil the response, it returns the receive-timeout error after the + /// deadline elapses. + #[tokio::test(start_paused = true)] + async fn handle_request_times_out_waiting_for_response() { + let (key, peer) = key_and_peer(0); + let peers = vec![peer]; + let validator = new_msg_verifier(&peers).expect("verifier"); + let consensus = Arc::new(MockConsensus::default()); + + let ct = CancellationToken::new(); + let (deadliner, _expired) = DeadlinerTask::start(ct.clone(), "test", FutureCalculator); + let (prio, _behaviour) = Prioritiser::new_internal( + peer, + peers, + 1, + consensus, + validator, + Duration::from_secs(3600), + deadliner, + ); + + // No prioritise instance runs for this duty, so the buffered request's + // response oneshot is never fulfilled; the wait must time out. + let msg = build_msg(&key, peer, "v1"); + let err = tokio::time::timeout( + RECEIVE_TIMEOUT + Duration::from_secs(1), + prio.inner.handle_request(peer, msg), + ) + .await + .expect("handle_request returns within the receive timeout") + .expect_err("response wait times out"); + assert!(matches!(err, Error::TimeoutWaiting)); + assert_eq!(err.to_string(), "timeout waiting for proposed priorities"); + } + + /// `handle_request` rejects a message whose peer id differs from the + /// connection's peer id. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn handle_request_invalid_peer_id() { + let (key, peer) = key_and_peer(0); + let peers = vec![peer]; + let validator = new_msg_verifier(&peers).expect("verifier"); + let consensus = Arc::new(MockConsensus::default()); + + let ct = CancellationToken::new(); + let (deadliner, _expired) = DeadlinerTask::start(ct.clone(), "test", FutureCalculator); + let (prio, _behaviour) = Prioritiser::new_internal( + peer, + peers, + 1, + consensus, + validator, + Duration::from_secs(3600), + deadliner, + ); + + let msg = build_msg(&key, peer, "v1"); + // Use a different connection peer id than the message claims. + let other = PeerId::random(); + let err = prio + .inner + .handle_request(other, msg) + .await + .expect_err("peer id mismatch"); + assert!(matches!(err, Error::InvalidPeerId)); + assert_eq!(err.to_string(), "invalid priority message peer id"); + } + + /// `handle_request` rejects a message for a duty the deadliner cannot + /// schedule (already expired). + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn handle_request_duty_expired() { + struct ExpiredCalculator; + impl DeadlineCalculator for ExpiredCalculator { + fn deadline( + &self, + _duty: &Duty, + ) -> pluto_core::deadline::Result>> { + Ok(Some( + Utc::now() + .checked_sub_signed(ChronoDuration::hours(1)) + .expect("deadline in range"), + )) + } + } + + let (key, peer) = key_and_peer(0); + let peers = vec![peer]; + let validator = new_msg_verifier(&peers).expect("verifier"); + let consensus = Arc::new(MockConsensus::default()); + + let ct = CancellationToken::new(); + let (deadliner, _expired) = DeadlinerTask::start(ct.clone(), "test", ExpiredCalculator); + let (prio, _behaviour) = Prioritiser::new_internal( + peer, + peers, + 1, + consensus, + validator, + Duration::from_secs(3600), + deadliner, + ); + + let msg = build_msg(&key, peer, "v1"); + let err = prio + .inner + .handle_request(peer, msg) + .await + .expect_err("duty expired"); + assert!(matches!(err, Error::DutyExpired)); + assert_eq!(err.to_string(), "duty expired"); + } + + /// The `Start` cleanup loop drops the duty's request buffer when the + /// deadliner reports it expired. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn start_cleans_expired_buffer() { + let (_key, peer) = key_and_peer(0); + let peers = vec![peer]; + let validator = new_msg_verifier(&peers).expect("verifier"); + let consensus = Arc::new(MockConsensus::default()); + + let ct = CancellationToken::new(); + // The cleanup loop consumes whatever the deadliner emits; drive it with + // a hand-built expired-duty channel to assert deletion deterministically. + let (expired_tx, expired_rx) = mpsc::channel(1); + let (deadliner, _real_expired) = DeadlinerTask::start(ct.clone(), "test", FutureCalculator); + let (prio, _behaviour) = Prioritiser::new_internal( + peer, + peers, + 1, + consensus, + validator, + Duration::from_secs(3600), + deadliner, + ); + prio.start(ct.clone(), expired_rx); + + let duty = Duty::new(SlotNumber::new(42), DutyType::Unknown); + let _ = prio.inner.get_req_buffer(duty.clone()); + assert!( + prio.inner + .req_buffers + .lock() + .expect("lock") + .contains_key(&duty) + ); + + expired_tx.send(duty.clone()).await.expect("send expired"); + + // Poll until the cleanup loop removes the entry. + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if !prio + .inner + .req_buffers + .lock() + .expect("lock") + .contains_key(&duty) + { + return; + } + tokio::task::yield_now().await; + } + }) + .await + .expect("buffer cleaned"); + } +} diff --git a/crates/priority/tests/prioritiser_test.rs b/crates/priority/tests/prioritiser_test.rs new file mode 100644 index 00000000..9587ffac --- /dev/null +++ b/crates/priority/tests/prioritiser_test.rs @@ -0,0 +1,383 @@ +//! Three-host integration test for the priority protocol. +//! +//! Three in-process libp2p hosts run the priority exchange against a mock +//! consensus that "decides" on the first proposal and asserts every proposal is +//! identical. Each host proposes a different number of priorities (`0:[0]`, +//! `1:[0,1]`, `2:[0,1,2]`); the cluster-agreed result keeps only priority `0` +//! (proposed by all three) with score `n*1000`. + +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::Duration, +}; + +use futures::StreamExt as _; +use libp2p::{ + Multiaddr, PeerId, Swarm, + core::{Transport as _, transport::MemoryTransport, upgrade::Version}, + multiaddr::Protocol, + swarm::SwarmEvent, +}; +use pluto_core::{ + corepb::v1::{ + core::Duty as ProtoDuty, + priority::{PriorityMsg, PriorityResult, PriorityTopicProposal}, + }, + deadline::{DeadlineCalculator, DeadlineError, DeadlinerHandle, DeadlinerTask}, + types::{Duty, DutyType, SlotNumber}, +}; +use pluto_priority::{Consensus, ConsensusError, Prioritiser, PrioritySubscriber}; +use prost_types::Any; +use tokio::{sync::mpsc, time::timeout}; +use tokio_util::sync::CancellationToken; + +use pluto_p2p::{peer::peer_id_from_key, utils::keypair_from_secret_key}; +use pluto_priority::p2p::Behaviour; +use pluto_testutil::random::generate_insecure_k1_key; + +/// Calculator that schedules every duty one hour out. +struct FutureCalculator; + +impl DeadlineCalculator for FutureCalculator { + fn deadline( + &self, + _duty: &Duty, + ) -> Result>, DeadlineError> { + Ok(Some( + chrono::Utc::now() + .checked_add_signed(chrono::Duration::hours(1)) + .expect("deadline in range"), + )) + } +} + +/// Mock consensus that decides on the first proposal per duty by invoking its +/// subscribers, and asserts every subsequent proposal for that duty is +/// identical. +#[derive(Default)] +struct TestConsensus { + subs: Mutex>, + proposed: Mutex>, +} + +#[async_trait::async_trait] +impl Consensus for TestConsensus { + async fn propose_priority( + &self, + duty: Duty, + result: PriorityResult, + _ct: &CancellationToken, + ) -> Result<(), ConsensusError> { + let slot = duty.slot.inner(); + + // Decide-once: if already proposed, assert identical and return. + { + let proposed = self.proposed.lock().expect("proposed mutex"); + if let Some(prev) = proposed.get(&slot) { + assert_eq!( + prev.topics, result.topics, + "all proposals for a duty must be identical" + ); + return Ok(()); + } + } + + let subs = self.subs.lock().expect("subs mutex"); + for sub in subs.iter() { + sub(duty.clone(), result.clone())?; + } + drop(subs); + + self.proposed + .lock() + .expect("proposed mutex") + .insert(slot, result); + Ok(()) + } + + fn subscribe_priority(&self, callback: PrioritySubscriber) { + self.subs.lock().expect("subs mutex").push(callback); + } +} + +/// Wraps priority `prio` as an `Any` of `Duty{slot: prio}`. +fn prio_to_any(prio: u64) -> Any { + Any::from_msg(&ProtoDuty { + slot: prio, + r#type: 0, + }) + .expect("pack Duty") +} + +/// A built node: its swarm, the prioritiser, and its listen address. +struct Host { + swarm: Swarm, + prioritiser: Prioritiser, + addr: Multiaddr, + peer_id: PeerId, +} + +/// In-process `/memory/` address, where `N` is derived from the seed +/// (non-zero so the kernel does not auto-assign a port). +fn memory_addr(seed: u8) -> Multiaddr { + Multiaddr::empty().with(Protocol::Memory(u64::from(seed) + 1)) +} + +/// Builds one host wired to the shared `consensus` and `deadliner`, running its +/// priority behaviour over an in-process [`MemoryTransport`]. The libp2p +/// identity is derived from the same secp256k1 key used for the peer id. +fn build_host( + seed: u8, + peers: Vec, + consensus: Arc, + deadliner: DeadlinerHandle, +) -> Host { + let key = generate_insecure_k1_key(seed); + let peer_id = peer_id_from_key(key.public_key()).expect("peer id"); + let keypair = keypair_from_secret_key(key).expect("keypair"); + + // A permissive verifier returning Ok for every message. + let validator = Box::new(|_: &PriorityMsg| Ok(())); + + let (prioritiser, behaviour) = Prioritiser::new_internal( + peer_id, + peers.clone(), + i64::try_from(peers.len()).expect("peer count fits i64"), + consensus, + validator, + Duration::from_secs(3600), + deadliner, + ); + + let swarm = libp2p::SwarmBuilder::with_existing_identity(keypair) + .with_tokio() + .with_other_transport(|key| { + MemoryTransport::default() + .upgrade(Version::V1) + .authenticate(libp2p::noise::Config::new(key).expect("noise config")) + .multiplex(libp2p::yamux::Config::default()) + }) + .expect("transport") + .with_behaviour(|_key| behaviour) + .expect("behaviour") + .build(); + + Host { + swarm, + prioritiser, + addr: memory_addr(seed), + peer_id, + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn three_host_prioritiser() { + const N: usize = 3; + const SEEDS: [u8; N] = [0, 1, 2]; + + let duties = [ + Duty::new(SlotNumber::new(97), DutyType::Unknown), + Duty::new(SlotNumber::new(98), DutyType::Unknown), + Duty::new(SlotNumber::new(99), DutyType::Unknown), + ]; + + // Derive the peer set deterministically from the per-host seeds. + let keys: Vec<_> = SEEDS.into_iter().map(generate_insecure_k1_key).collect(); + let peers: Vec = keys + .iter() + .map(|k| peer_id_from_key(k.public_key()).expect("peer id")) + .collect(); + + let ct = CancellationToken::new(); + let consensus = Arc::new(TestConsensus::default()); + + // One deadliner shared across all hosts. + let (deadliner, expired) = DeadlinerTask::start(ct.clone(), "test", FutureCalculator); + + // The deadliner emits expired duties to a single consumer, and `start` + // consumes the receiver once. Hand it to the first prioritiser; the others + // get a never-emitting channel (their buffers are cleaned via cancellation + // on `ct`). + let mut expired_opt = Some(expired); + + // Build hosts. + let mut hosts: Vec = Vec::with_capacity(N); + for &seed in &SEEDS { + hosts.push(build_host( + seed, + peers.clone(), + consensus.clone(), + deadliner.clone(), + )); + } + + // Collect all decided priority lists across subscribers. + let (results_tx, mut results_rx) = + mpsc::unbounded_channel::>(); + + // Wire one subscriber per prioritiser. Each asserts the topic shape and + // forwards the priorities. + let topic_any = Any::from_msg(&pluto_core::corepb::v1::core::ParSignedData { + data: b"test topic".to_vec().into(), + ..Default::default() + }) + .expect("pack topic"); + + for host in &hosts { + let tx = results_tx.clone(); + let expected_topic = topic_any.clone(); + host.prioritiser.subscribe(Box::new(move |duty, result| { + let slot = duty.slot.inner(); + assert!( + [97, 98, 99].contains(&slot), + "decided duty slot must be one of the proposed duties, got {slot}" + ); + assert_eq!(result.topics.len(), 1, "exactly one topic"); + let topic = &result.topics[0]; + assert_eq!( + topic.topic.as_ref().expect("topic any"), + &expected_topic, + "topic round-trips" + ); + let _ = tx.send(topic.priorities.clone()); + Ok(()) + })); + } + drop(results_tx); + + // Start cleanup loops. The shared deadliner has a single expired receiver, + // so only the first prioritiser drives real cleanup; the others get an + // open (never-emitting) channel. The senders are kept alive for the test so + // those cleanup loops park on `recv()` rather than exiting and cancelling + // their quit tokens (which would break inbound exchange handling). + let mut keepalive_senders: Vec> = Vec::new(); + for host in &hosts { + let rx = expired_opt.take().unwrap_or_else(|| { + let (tx, rx) = mpsc::channel::(1); + keepalive_senders.push(tx); + rx + }); + host.prioritiser.start(ct.clone(), rx); + } + + // Begin listening, then full-mesh dial. + for host in &mut hosts { + host.swarm.listen_on(host.addr.clone()).expect("listen"); + } + for host in &mut hosts { + loop { + if matches!( + host.swarm.select_next_some().await, + SwarmEvent::NewListenAddr { .. } + ) { + break; + } + } + } + + // Dial every other host from each host. + let addrs: Vec = hosts.iter().map(|h| h.addr.clone()).collect(); + for (i, host) in hosts.iter_mut().enumerate() { + for (j, addr) in addrs.iter().enumerate() { + if i != j { + host.swarm.dial(addr.clone()).expect("dial"); + } + } + } + + // Extract per-host prioritisers (with their key/peer id) and drive each + // swarm in the background. Host `i` proposes priorities `0..=i`. + let mut launchers = Vec::with_capacity(N); + let mut drivers = Vec::with_capacity(N); + for (i, host) in hosts.into_iter().enumerate() { + let count = u64::try_from(i).expect("host index fits u64"); + launchers.push((host.prioritiser, keys[i].clone(), host.peer_id, count)); + let mut swarm = host.swarm; + drivers.push(tokio::spawn(async move { + loop { + let _ = swarm.select_next_some().await; + } + })); + } + + // Launch prioritise across all (host, duty) pairs. + let (err_tx, mut err_rx) = mpsc::unbounded_channel(); + let mut prioritise_tasks = Vec::new(); + for (prio, key, peer_id, max_prio) in &launchers { + // Propose 0:[0], 1:[0,1], 2:[0,1,2]. + let priorities: Vec = (0..=*max_prio).map(prio_to_any).collect(); + + for duty in &duties { + let proto_duty = ProtoDuty { + slot: duty.slot.inner(), + r#type: 0, + }; + let msg = sign( + key, + PriorityMsg { + duty: Some(proto_duty), + topics: vec![PriorityTopicProposal { + topic: Some(topic_any.clone()), + priorities: priorities.clone(), + }], + peer_id: peer_id.to_string(), + signature: Default::default(), + }, + ); + + let prio = prio.clone(); + let ct = ct.clone(); + let err_tx = err_tx.clone(); + prioritise_tasks.push(tokio::spawn(async move { + let res = prio.prioritise(ct, msg).await; + let _ = err_tx.send(res); + })); + } + } + drop(err_tx); + + // Expect N * len(duties) decided priority lists, each [prio 0] @ score N*1000. + let expected_results = N * duties.len(); + let expected_score = i64::try_from(N).expect("N fits i64") * 1000; + let zero_any = prio_to_any(0); + + for _ in 0..expected_results { + let res = timeout(Duration::from_secs(30), results_rx.recv()) + .await + .expect("result within timeout") + .expect("result delivered"); + assert_eq!(res.len(), 1, "exactly one priority survives"); + assert_eq!(res[0].score, expected_score, "score is n*1000"); + assert_eq!( + res[0].priority.as_ref().expect("priority any"), + &zero_any, + "the surviving priority is 0" + ); + } + + // Cancel: every prioritise instance returns the cancellation error. + ct.cancel(); + for _ in 0..expected_results { + let res = timeout(Duration::from_secs(10), err_rx.recv()) + .await + .expect("error within timeout") + .expect("error delivered"); + assert!( + matches!(res, Err(pluto_priority::Error::Cancelled)), + "cancelled prioritise returns context-canceled, got {res:?}" + ); + } + + for d in drivers { + d.abort(); + } + for t in prioritise_tasks { + t.abort(); + } +} + +/// Signs a priority message with the secp256k1 key. +fn sign(key: &k256::SecretKey, msg: PriorityMsg) -> PriorityMsg { + pluto_priority::component::sign_msg(&msg, key).expect("sign") +} From df1fb27bd4a9465c9e7f9f57bfb39385444b1cc6 Mon Sep 17 00:00:00 2001 From: Quang Le Date: Wed, 17 Jun 2026 18:16:36 +0700 Subject: [PATCH 03/12] fix: machete --- Cargo.lock | 2 -- crates/priority/Cargo.toml | 2 -- 2 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5485f7f7..6eeb583c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5935,7 +5935,6 @@ version = "1.7.1" dependencies = [ "async-trait", "chrono", - "ethereum_ssz", "futures", "k256", "libp2p", @@ -5943,7 +5942,6 @@ dependencies = [ "pluto-core", "pluto-k1util", "pluto-p2p", - "pluto-ssz", "pluto-testutil", "prost 0.14.3", "prost-types 0.14.3", diff --git a/crates/priority/Cargo.toml b/crates/priority/Cargo.toml index 9c2d562a..d04b3156 100644 --- a/crates/priority/Cargo.toml +++ b/crates/priority/Cargo.toml @@ -16,10 +16,8 @@ pluto-consensus.workspace = true pluto-core.workspace = true pluto-k1util.workspace = true pluto-p2p.workspace = true -pluto-ssz.workspace = true prost.workspace = true prost-types.workspace = true -ssz.workspace = true thiserror.workspace = true tokio.workspace = true tokio-util.workspace = true From 4d049e61f83472810a06fd60228703f48b11556c Mon Sep 17 00:00:00 2001 From: Quang Le Date: Thu, 18 Jun 2026 14:07:24 +0700 Subject: [PATCH 04/12] fix: don't panic when duplicated duty --- crates/priority/src/error.rs | 4 ++++ crates/priority/src/prioritiser.rs | 36 +++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/crates/priority/src/error.rs b/crates/priority/src/error.rs index 7115ed3e..a2c1b9ac 100644 --- a/crates/priority/src/error.rs +++ b/crates/priority/src/error.rs @@ -33,6 +33,10 @@ pub enum Error { #[error("duplicate priority")] DuplicatePriority, + /// A prioritise instance is already running for this duty. + #[error("duplicate priority instance for duty {0}")] + DuplicateInstance(Duty), + /// Hashing a topic or priority protobuf failed. #[error("hash proto: {0}")] HashProto(#[source] pluto_consensus::qbft::msg::Error), diff --git a/crates/priority/src/prioritiser.rs b/crates/priority/src/prioritiser.rs index c79ab891..f2395e3f 100644 --- a/crates/priority/src/prioritiser.rs +++ b/crates/priority/src/prioritiser.rs @@ -331,7 +331,7 @@ impl Prioritiser { let requests = self .inner .take_req_receiver(duty.clone()) - .expect("one prioritise instance per duty"); + .ok_or_else(|| Error::DuplicateInstance(duty.clone()))?; run_instance(ctx, &self.inner, duty, msg, requests).await } } @@ -582,6 +582,40 @@ mod tests { sign_msg(&msg, key).expect("sign") } + /// A second `prioritise` for a duty whose instance already holds the + /// receiver surfaces [`Error::DuplicateInstance`] rather than panicking. + #[tokio::test] + async fn duplicate_instance_returns_error() { + let (key, peer) = key_and_peer(0); + let peers = vec![peer]; + let validator = new_msg_verifier(&peers).expect("verifier"); + let consensus = Arc::new(MockConsensus::default()); + let ct = CancellationToken::new(); + let (deadliner, _expired) = DeadlinerTask::start(ct.clone(), "test", FutureCalculator); + let (prio, _behaviour) = Prioritiser::new_internal( + peer, + peers, + 1, + consensus, + validator, + Duration::from_secs(3600), + deadliner, + ); + + let msg = build_msg(&key, peer, "v1"); + let duty = duty_from_proto(msg.duty.as_ref().expect("duty")); + + // Simulate a running instance that already took the duty's receiver. + let _rx = prio + .inner + .take_req_receiver(duty.clone()) + .expect("first take"); + + // The duplicate is rejected after the (passing) deadliner gate. + let res = prio.prioritise(ct, msg).await; + assert!(matches!(res, Err(Error::DuplicateInstance(d)) if d == duty)); + } + /// Single-node instance reaches consensus on the exchange timeout and /// delivers the decided result to the prioritiser's subscriber. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] From bded87184b94a4ca44a6c9487c73c4c7ab23f8c7 Mon Sep 17 00:00:00 2001 From: Quang Le Date: Thu, 18 Jun 2026 14:13:48 +0700 Subject: [PATCH 05/12] fix: don't flatten Deadline Error --- crates/priority/src/component.rs | 3 +-- crates/priority/src/error.rs | 6 +++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/priority/src/component.rs b/crates/priority/src/component.rs index 395a8344..ddd64427 100644 --- a/crates/priority/src/component.rs +++ b/crates/priority/src/component.rs @@ -331,8 +331,7 @@ impl Component { let deadline = self .calculator .deadline(&duty) - .ok() - .flatten() + .map_err(Error::Deadline)? .ok_or(Error::DutyAlreadyExpired)?; let msg = PriorityMsg { diff --git a/crates/priority/src/error.rs b/crates/priority/src/error.rs index a2c1b9ac..490570d2 100644 --- a/crates/priority/src/error.rs +++ b/crates/priority/src/error.rs @@ -1,6 +1,6 @@ //! Error types for the priority protocol. -use pluto_core::types::Duty; +use pluto_core::{deadline::DeadlineError, types::Duty}; use thiserror::Error; /// Result alias for the priority crate. @@ -127,6 +127,10 @@ pub enum Error { #[error("duty already expired")] DutyAlreadyExpired, + /// Computing the duty's deadline failed. + #[error("compute deadline: {0}")] + Deadline(#[source] DeadlineError), + /// The prioritise instance's context was cancelled. #[error("context canceled")] Cancelled, From 333c3dbef6d861f77740230adcec4c6216a94978 Mon Sep 17 00:00:00 2001 From: Quang Le Date: Thu, 18 Jun 2026 14:25:26 +0700 Subject: [PATCH 06/12] fix: derive local_id from priv key --- crates/priority/src/component.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/crates/priority/src/component.rs b/crates/priority/src/component.rs index ddd64427..fc9e9f8c 100644 --- a/crates/priority/src/component.rs +++ b/crates/priority/src/component.rs @@ -15,7 +15,7 @@ use pluto_core::{ deadline::{DeadlineCalculator, DeadlinerTask}, types::Duty, }; -use pluto_p2p::peer::peer_id_to_public_key; +use pluto_p2p::peer::{peer_id_from_key, peer_id_to_public_key}; use prost::Message; use prost_types::{Any, Value, value::Kind}; use tokio_util::sync::CancellationToken; @@ -236,14 +236,17 @@ pub struct Component { /// Constructs a priority [`Component`] and the libp2p [`Behaviour`] to register /// with the swarm. /// +/// The local peer id is derived from `privkey`, so the `peer_id` carried in +/// outgoing messages and the signature over them cannot diverge. `privkey` must +/// be the same key the caller builds its libp2p swarm from, so the on-wire peer +/// id matches the message peer id. +/// /// Builds the message verifier from `peers`, spawns a deadliner driven by /// `calculator`, and wires the prioritiser. The caller must register the /// returned behaviour with its swarm and call [`Component::start`] exactly /// once. -#[allow(clippy::too_many_arguments)] pub fn new_component( ctx: CancellationToken, - local_id: PeerId, peers: Vec, min_required: i64, consensus: Arc, @@ -251,6 +254,10 @@ pub fn new_component( privkey: SecretKey, calculator: impl DeadlineCalculator, ) -> Result<(Component, Behaviour)> { + // Derive the local peer id from the signing key so the message `peer_id` + // and its signature always agree (peers verify the two against each other). + let local_id = peer_id_from_key(privkey.public_key()).map_err(Error::PeerKey)?; + let verifier = new_msg_verifier(&peers)?; let calculator: Arc = Arc::new(calculator); From 750d779044a622dae3fc15bca2c1540c15224f40 Mon Sep 17 00:00:00 2001 From: Quang Le Date: Thu, 18 Jun 2026 14:40:21 +0700 Subject: [PATCH 07/12] fix: protocol_id start with slash --- crates/priority/src/p2p/mod.rs | 1 - crates/priority/src/p2p/protocol.rs | 68 +++++------------------------ crates/priority/src/prioritiser.rs | 9 +++- 3 files changed, 18 insertions(+), 60 deletions(-) diff --git a/crates/priority/src/p2p/mod.rs b/crates/priority/src/p2p/mod.rs index 35e2c8a0..b386c8b3 100644 --- a/crates/priority/src/p2p/mod.rs +++ b/crates/priority/src/p2p/mod.rs @@ -27,7 +27,6 @@ use tokio::sync::{mpsc, oneshot}; pub use behaviour::{Behaviour, Event}; pub use handler::{FromBehaviour, Handler, OutboundRequest}; -pub use protocol::PriorityProtocol; use crate::error::Error; diff --git a/crates/priority/src/p2p/protocol.rs b/crates/priority/src/p2p/protocol.rs index aaf166f2..969671ca 100644 --- a/crates/priority/src/p2p/protocol.rs +++ b/crates/priority/src/p2p/protocol.rs @@ -6,58 +6,20 @@ use std::time::Duration; -use libp2p::{core::upgrade::ReadyUpgrade, swarm::Stream}; +use libp2p::{ + core::upgrade::ReadyUpgrade, + swarm::{Stream, StreamProtocol}, +}; use pluto_core::corepb::v1::priority::PriorityMsg; use crate::PROTOCOL_ID; -/// Wire token negotiated for the priority protocol. -/// -/// The canonical protocol identifier is the slug-less [`PROTOCOL_ID`]. The -/// negotiated token derives from it directly, so [`PROTOCOL_ID`] is the single -/// source of truth. -/// -/// `libp2p`'s multistream-select requires every negotiated protocol token to -/// begin with `/` and rejects any other form before it reaches the wire, so the -/// token offered for negotiation is [`PROTOCOL_ID`] with a leading `/`. This is -/// the one place where the wire form differs from the canonical identifier. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct PriorityProtocol; - -/// Negotiated wire token: the canonical identifier with the leading `/` that -/// multistream-select mandates. Tied to [`PROTOCOL_ID`] by a compile-time -/// assertion below, keeping the canonical identifier the single source of -/// truth. -const WIRE_TOKEN: &str = "/charon/priority/2.0.0"; - -/// The wire token must be the canonical identifier prefixed with `/`. -const _: () = { - let id = PROTOCOL_ID.as_bytes(); - let wire = WIRE_TOKEN.as_bytes(); - assert!( - wire.len() == id.len() + 1, - "wire token must be /" - ); - assert!(wire[0] == b'/', "wire token must start with /"); - let mut i = 0; - while i < id.len() { - assert!(wire[i + 1] == id[i], "wire token must equal /"); - i += 1; - } -}; - -impl AsRef for PriorityProtocol { - fn as_ref(&self) -> &str { - WIRE_TOKEN - } -} - /// Upgrade negotiating the priority protocol on inbound and outbound streams. -pub(crate) type PriorityUpgrade = ReadyUpgrade; +pub(crate) type PriorityUpgrade = ReadyUpgrade; /// Returns the upgrade used to negotiate the priority protocol. pub(crate) fn upgrade() -> PriorityUpgrade { - ReadyUpgrade::new(PriorityProtocol) + ReadyUpgrade::new(StreamProtocol::new(PROTOCOL_ID)) } /// Maximum protobuf message size (128MB). @@ -122,20 +84,12 @@ mod tests { use super::*; - /// The canonical protocol identifier carries no leading slug, exactly as - /// the reference implementation registers it. - #[test] - fn canonical_protocol_id_has_no_leading_slash() { - assert_eq!(PROTOCOL_ID, "charon/priority/2.0.0"); - assert!(!PROTOCOL_ID.starts_with('/')); - } - - /// The token offered for negotiation is the canonical identifier with the - /// leading `/` that multistream-select mandates; no other divergence. + /// The protocol identifier is the libp2p-negotiated wire token, which + /// multistream-select requires to begin with `/`. #[test] - fn wire_token_is_canonical_id_with_leading_slash() { - assert_eq!(PriorityProtocol.as_ref(), "/charon/priority/2.0.0"); - assert_eq!(PriorityProtocol.as_ref(), format!("/{PROTOCOL_ID}")); + fn protocol_id_is_wire_token() { + assert_eq!(PROTOCOL_ID, "/charon/priority/2.0.0"); + assert!(PROTOCOL_ID.starts_with('/')); } fn any() -> Any { diff --git a/crates/priority/src/prioritiser.rs b/crates/priority/src/prioritiser.rs index f2395e3f..6825d454 100644 --- a/crates/priority/src/prioritiser.rs +++ b/crates/priority/src/prioritiser.rs @@ -36,8 +36,13 @@ use crate::{ p2p::{self, Behaviour, InboundHandler, Sender, protocol::RECEIVE_TIMEOUT}, }; -/// Supported priority protocol identifier. -pub const PROTOCOL_ID: &str = "charon/priority/2.0.0"; +/// Supported priority protocol identifier (the libp2p-negotiated wire token). +/// +/// The leading `/` is mandatory: multistream-select rejects slash-less tokens. +/// The reference implementation negotiates the slash-less +/// `charon/priority/2.0.0`, so this does NOT interop cross-implementation — the +/// wire tokens differ by that byte. +pub const PROTOCOL_ID: &str = "/charon/priority/2.0.0"; /// A received peer request paired with a channel to deliver this peer's reply. struct Request { From bdbdf9573ef78788729f1474e1d82b0a1ec1e4f1 Mon Sep 17 00:00:00 2001 From: Quang Le Date: Thu, 18 Jun 2026 15:07:44 +0700 Subject: [PATCH 08/12] fix: surface deadline compute failed error --- crates/priority/src/error.rs | 5 +++++ crates/priority/src/prioritiser.rs | 18 +++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/crates/priority/src/error.rs b/crates/priority/src/error.rs index 490570d2..eb945a84 100644 --- a/crates/priority/src/error.rs +++ b/crates/priority/src/error.rs @@ -131,6 +131,11 @@ pub enum Error { #[error("compute deadline: {0}")] Deadline(#[source] DeadlineError), + /// The deadliner could not compute the duty's deadline (computation error + /// or shutdown), as distinct from the duty being expired. + #[error("deadline computation failed")] + DeadlineComputeFailed, + /// The prioritise instance's context was cancelled. #[error("context canceled")] Cancelled, diff --git a/crates/priority/src/prioritiser.rs b/crates/priority/src/prioritiser.rs index 6825d454..c4a0d8d8 100644 --- a/crates/priority/src/prioritiser.rs +++ b/crates/priority/src/prioritiser.rs @@ -171,8 +171,12 @@ impl Inner { let proto_duty = msg.duty.as_ref().ok_or(Error::InvalidMsgProtoFields)?; let duty = duty_from_proto(proto_duty); - if self.deadliner.add(duty.clone()).await != AddOutcome::Scheduled { - return Err(Error::DutyExpired); + match self.deadliner.add(duty.clone()).await { + AddOutcome::Scheduled => {} + AddOutcome::FailedToCompute => return Err(Error::DeadlineComputeFailed), + AddOutcome::AlreadyExpired | AddOutcome::NoDeadline => { + return Err(Error::DutyExpired); + } } let buffer = self.get_req_buffer(duty); @@ -328,9 +332,13 @@ impl Prioritiser { let proto_duty = msg.duty.as_ref().ok_or(Error::InvalidMsgProtoFields)?; let duty = duty_from_proto(proto_duty); - if self.inner.deadliner.add(duty.clone()).await != AddOutcome::Scheduled { - tracing::warn!(%duty, "Dropping priority protocol instance for expired duty"); - return Ok(()); + match self.inner.deadliner.add(duty.clone()).await { + AddOutcome::Scheduled => {} + AddOutcome::FailedToCompute => return Err(Error::DeadlineComputeFailed), + AddOutcome::AlreadyExpired | AddOutcome::NoDeadline => { + tracing::warn!(%duty, "Dropping priority protocol instance for expired duty"); + return Ok(()); + } } let requests = self From d2123ccdba0e077018b50b4655356ffbc0a99235 Mon Sep 17 00:00:00 2001 From: Quang Le Date: Thu, 18 Jun 2026 16:07:55 +0700 Subject: [PATCH 09/12] fix: using HashSet to simplify code --- crates/priority/src/p2p/behaviour.rs | 10 +++++----- crates/priority/src/prioritiser.rs | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/crates/priority/src/p2p/behaviour.rs b/crates/priority/src/p2p/behaviour.rs index 139d4ea8..f758de2c 100644 --- a/crates/priority/src/p2p/behaviour.rs +++ b/crates/priority/src/p2p/behaviour.rs @@ -6,7 +6,7 @@ //! exists. use std::{ - collections::{HashMap, VecDeque}, + collections::{HashMap, HashSet, VecDeque}, task::{Context, Poll}, }; @@ -30,7 +30,7 @@ pub struct Behaviour { inbound_handler: InboundHandler, command_rx: mpsc::UnboundedReceiver, /// Peers with at least one established connection. - connected: HashMap, + connected: HashSet, /// Outbound requests waiting for a connection to the target peer. awaiting_connection: HashMap>, pending_events: VecDeque>>, @@ -47,7 +47,7 @@ impl Behaviour { Self { inbound_handler, command_rx, - connected: HashMap::new(), + connected: HashSet::new(), awaiting_connection: HashMap::new(), pending_events: VecDeque::new(), } @@ -60,7 +60,7 @@ impl Behaviour { } fn send_receive(&mut self, peer: PeerId, request: OutboundRequest) { - if self.connected.contains_key(&peer) { + if self.connected.contains(&peer) { self.notify_handler(peer, request); return; } @@ -131,7 +131,7 @@ impl NetworkBehaviour for Behaviour { fn on_swarm_event(&mut self, event: FromSwarm) { match event { FromSwarm::ConnectionEstablished(event) => { - self.connected.insert(event.peer_id, ()); + self.connected.insert(event.peer_id); self.flush_awaiting(event.peer_id); } FromSwarm::ConnectionClosed(event) if event.remaining_established == 0 => { diff --git a/crates/priority/src/prioritiser.rs b/crates/priority/src/prioritiser.rs index c4a0d8d8..565ba8cc 100644 --- a/crates/priority/src/prioritiser.rs +++ b/crates/priority/src/prioritiser.rs @@ -13,7 +13,7 @@ //! signalled by a [`CancellationToken`]. use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, sync::{Arc, Mutex, OnceLock}, time::Duration, }; @@ -369,7 +369,7 @@ async fn run_instance( // `calculate_result` as a duplicate-peer error rather than being silently // swallowed. let mut msgs: Vec = vec![own.clone()]; - let mut dedup_peers: HashMap = HashMap::new(); + let mut dedup_peers: HashSet = HashSet::new(); let mut cons_started = false; @@ -418,11 +418,11 @@ async fn run_instance( } /// Adds the first message seen from each peer to `msgs`. -fn add_msg(msgs: &mut Vec, dedup: &mut HashMap, msg: PriorityMsg) { - if dedup.contains_key(&msg.peer_id) { +fn add_msg(msgs: &mut Vec, dedup: &mut HashSet, msg: PriorityMsg) { + if dedup.contains(&msg.peer_id) { return; } - dedup.insert(msg.peer_id.clone(), ()); + dedup.insert(msg.peer_id.clone()); msgs.push(msg); } From ae5b3c6d0c3c15a54415c6b90234aa394b40574d Mon Sep 17 00:00:00 2001 From: Quang Le Date: Thu, 18 Jun 2026 16:11:37 +0700 Subject: [PATCH 10/12] fix: add debug on unexpected error --- crates/priority/src/p2p/behaviour.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/priority/src/p2p/behaviour.rs b/crates/priority/src/p2p/behaviour.rs index f758de2c..a99d6c92 100644 --- a/crates/priority/src/p2p/behaviour.rs +++ b/crates/priority/src/p2p/behaviour.rs @@ -176,9 +176,20 @@ impl NetworkBehaviour for Behaviour { /// Only transport/unsupported variants reach this path; both are cheaply /// reconstructable from their displayed form without losing parity-relevant /// detail. +/// Clones the subset of [`crate::Error`] that can reach the awaiting-connection +/// path (dial/negotiation outcomes), which is not `Clone` as a whole. +/// +/// Only `Unsupported` and `Transport` are expected here; any other variant is a +/// bug (caught in debug) and is flattened to `Transport` rather than re-wrapped +/// — re-wrapping a `Transport` via `to_string()` would duplicate its Display +/// prefix. fn clone_error(error: &crate::Error) -> crate::Error { match error { crate::Error::Unsupported => crate::Error::Unsupported, - other => crate::Error::Transport(other.to_string()), + crate::Error::Transport(msg) => crate::Error::Transport(msg.clone()), + other => { + debug_assert!(false, "unexpected error on awaiting path: {other}"); + crate::Error::Transport(other.to_string()) + } } } From e1189dd8b1d51ef25ec067a3871459aec46fbe7e Mon Sep 17 00:00:00 2001 From: Quang Le Date: Thu, 18 Jun 2026 18:06:01 +0700 Subject: [PATCH 11/12] fix: protocol_id not start with slash --- Cargo.lock | 8 +- Cargo.toml | 11 + crates/priority/Cargo.toml | 6 +- crates/priority/src/p2p/protocol.rs | 25 +- crates/priority/src/prioritiser.rs | 10 +- crates/priority/tests/interop_protocol_id.rs | 68 ++ .../multistream-select/.cargo_vcs_info.json | 6 + third_party/multistream-select/CHANGELOG.md | 101 ++ third_party/multistream-select/Cargo.lock | 860 ++++++++++++++++++ third_party/multistream-select/Cargo.toml | 74 ++ .../multistream-select/Cargo.toml.orig | 34 + third_party/multistream-select/PATCHES.md | 54 ++ .../multistream-select/src/dialer_select.rs | 203 +++++ .../src/length_delimited.rs | 489 ++++++++++ third_party/multistream-select/src/lib.rs | 144 +++ .../multistream-select/src/listener_select.rs | 308 +++++++ .../multistream-select/src/negotiated.rs | 390 ++++++++ .../multistream-select/src/protocol.rs | 529 +++++++++++ .../multistream-select/tests/dialer_select.rs | 202 ++++ 19 files changed, 3501 insertions(+), 21 deletions(-) create mode 100644 crates/priority/tests/interop_protocol_id.rs create mode 100644 third_party/multistream-select/.cargo_vcs_info.json create mode 100644 third_party/multistream-select/CHANGELOG.md create mode 100644 third_party/multistream-select/Cargo.lock create mode 100644 third_party/multistream-select/Cargo.toml create mode 100644 third_party/multistream-select/Cargo.toml.orig create mode 100644 third_party/multistream-select/PATCHES.md create mode 100644 third_party/multistream-select/src/dialer_select.rs create mode 100644 third_party/multistream-select/src/length_delimited.rs create mode 100644 third_party/multistream-select/src/lib.rs create mode 100644 third_party/multistream-select/src/listener_select.rs create mode 100644 third_party/multistream-select/src/negotiated.rs create mode 100644 third_party/multistream-select/src/protocol.rs create mode 100644 third_party/multistream-select/tests/dialer_select.rs diff --git a/Cargo.lock b/Cargo.lock index 7f75a812..531f5abb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2208,7 +2208,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccc2776f0c61eca1ca32528f85548abd1a4be8fb53d1b21c013e4f18da1e7090" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.117", ] [[package]] @@ -3438,7 +3438,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.57.0", + "windows-core 0.62.2", ] [[package]] @@ -4834,8 +4834,6 @@ checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" [[package]] name = "multistream-select" version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea0df8e5eec2298a62b326ee4f0d7fe1a6b90a09dfcf9df37b38f947a8c42f19" dependencies = [ "bytes", "futures", @@ -5941,6 +5939,7 @@ dependencies = [ "futures", "k256", "libp2p", + "multistream-select", "pluto-consensus", "pluto-core", "pluto-k1util", @@ -7966,6 +7965,7 @@ checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" dependencies = [ "bytes", "futures-core", + "futures-io", "futures-sink", "pin-project-lite", "tokio", diff --git a/Cargo.toml b/Cargo.toml index 15b3370e..211505cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,10 @@ members = [ "crates/frost", "crates/priority", ] +# Vendored fork consumed only via [patch.crates-io]; excluded so it builds/tests +# standalone (its upstream code isn't written to this workspace's lints) without +# cargo treating it as a member. +exclude = ["third_party/multistream-select"] resolver = "3" [workspace.package] @@ -163,3 +167,10 @@ opt-level = 3 [profile.test.package.aes] opt-level = 3 + +# Forked multistream-select that accepts slash-less protocol names, required for +# the priority protocol to interoperate with Charon's "charon/priority/2.0.0" +# (no leading slash). Upstream rejects such names; the only delta is relaxing +# that check. See third_party/multistream-select/PATCHES.md. +[patch.crates-io] +multistream-select = { path = "third_party/multistream-select" } diff --git a/crates/priority/Cargo.toml b/crates/priority/Cargo.toml index d04b3156..5a6ddf3b 100644 --- a/crates/priority/Cargo.toml +++ b/crates/priority/Cargo.toml @@ -27,7 +27,11 @@ tracing.workspace = true pluto-testutil.workspace = true rand.workspace = true test-case.workspace = true -tokio = { workspace = true, features = ["test-util"] } +tokio = { workspace = true, features = ["test-util", "rt-multi-thread", "macros", "net", "time", "io-util"] } +# Interop proof: drive the patched multistream-select negotiation over real TCP. +multistream-select = "0.13" +tokio-util = { workspace = true, features = ["compat"] } +futures.workspace = true [lints] workspace = true diff --git a/crates/priority/src/p2p/protocol.rs b/crates/priority/src/p2p/protocol.rs index 969671ca..8d9eb474 100644 --- a/crates/priority/src/p2p/protocol.rs +++ b/crates/priority/src/p2p/protocol.rs @@ -6,20 +6,23 @@ use std::time::Duration; -use libp2p::{ - core::upgrade::ReadyUpgrade, - swarm::{Stream, StreamProtocol}, -}; +use libp2p::{core::upgrade::ReadyUpgrade, swarm::Stream}; use pluto_core::corepb::v1::priority::PriorityMsg; use crate::PROTOCOL_ID; /// Upgrade negotiating the priority protocol on inbound and outbound streams. -pub(crate) type PriorityUpgrade = ReadyUpgrade; +/// +/// Uses `&'static str` rather than `StreamProtocol`: the latter is sealed and +/// rejects [`PROTOCOL_ID`]'s slash-less token, while `ReadyUpgrade` only +/// requires `AsRef + Clone`. Negotiation of the slash-less token is +/// enabled by the patched multistream-select (see +/// third_party/multistream-select). +pub(crate) type PriorityUpgrade = ReadyUpgrade<&'static str>; /// Returns the upgrade used to negotiate the priority protocol. pub(crate) fn upgrade() -> PriorityUpgrade { - ReadyUpgrade::new(StreamProtocol::new(PROTOCOL_ID)) + ReadyUpgrade::new(PROTOCOL_ID) } /// Maximum protobuf message size (128MB). @@ -84,12 +87,12 @@ mod tests { use super::*; - /// The protocol identifier is the libp2p-negotiated wire token, which - /// multistream-select requires to begin with `/`. + /// The protocol identifier is the slash-less wire token, matching the + /// reference implementation exactly for cross-implementation interop. #[test] - fn protocol_id_is_wire_token() { - assert_eq!(PROTOCOL_ID, "/charon/priority/2.0.0"); - assert!(PROTOCOL_ID.starts_with('/')); + fn protocol_id_matches_reference_wire_token() { + assert_eq!(PROTOCOL_ID, "charon/priority/2.0.0"); + assert!(!PROTOCOL_ID.starts_with('/')); } fn any() -> Any { diff --git a/crates/priority/src/prioritiser.rs b/crates/priority/src/prioritiser.rs index 565ba8cc..45d7ecbe 100644 --- a/crates/priority/src/prioritiser.rs +++ b/crates/priority/src/prioritiser.rs @@ -38,11 +38,11 @@ use crate::{ /// Supported priority protocol identifier (the libp2p-negotiated wire token). /// -/// The leading `/` is mandatory: multistream-select rejects slash-less tokens. -/// The reference implementation negotiates the slash-less -/// `charon/priority/2.0.0`, so this does NOT interop cross-implementation — the -/// wire tokens differ by that byte. -pub const PROTOCOL_ID: &str = "/charon/priority/2.0.0"; +/// Slash-less, to match the reference implementation's wire token exactly for +/// cross-implementation interop. Stock rust-libp2p multistream-select rejects +/// slash-less names; the workspace patches it (see +/// third_party/multistream-select) so this exact token negotiates. +pub const PROTOCOL_ID: &str = "charon/priority/2.0.0"; /// A received peer request paired with a channel to deliver this peer's reply. struct Request { diff --git a/crates/priority/tests/interop_protocol_id.rs b/crates/priority/tests/interop_protocol_id.rs new file mode 100644 index 00000000..c5663d06 --- /dev/null +++ b/crates/priority/tests/interop_protocol_id.rs @@ -0,0 +1,68 @@ +//! Real-TCP proof that the priority protocol negotiates the exact slash-less +//! wire token `charon/priority/2.0.0` (matching the reference implementation), +//! exercising the patched multistream-select end-to-end. +//! +//! This is the negotiation that runs over every transport (TCP here); a stock +//! multistream-select would reject the slash-less token at propose/advertise/ +//! decode, so a successful handshake proves the patch works on a real socket. + +use std::time::Duration; + +use futures::future; +use multistream_select::{Version, dialer_select_proto, listener_select_proto}; +use tokio::{ + net::{TcpListener, TcpStream}, + time::timeout, +}; +use tokio_util::compat::TokioAsyncReadCompatExt; + +/// The token both nodes negotiate — Pluto's priority protocol id. +const PROTO: &str = pluto_priority::PROTOCOL_ID; + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn negotiates_slashless_priority_protocol_over_tcp() { + // The id must carry NO leading slash, byte-identical to Charon's wire token. + assert!( + !PROTO.starts_with('/'), + "priority protocol id must be slash-less, got {PROTO:?}" + ); + println!( + "priority PROTOCOL_ID = {PROTO:?} (leading '/': {})", + PROTO.starts_with('/') + ); + + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind"); + let addr = listener.local_addr().expect("local addr"); + + // Listener node: accept one connection and negotiate the priority protocol. + let server = tokio::spawn(async move { + let (sock, _peer) = listener.accept().await.expect("accept"); + let (proto, _io) = listener_select_proto(sock.compat(), std::iter::once(PROTO)) + .await + .expect("listener negotiation"); + proto + }); + + // Dialer node: connect and propose the priority protocol. + let client = tokio::spawn(async move { + let sock = TcpStream::connect(addr).await.expect("connect"); + let (proto, _io) = dialer_select_proto(sock.compat(), std::iter::once(PROTO), Version::V1) + .await + .expect("dialer negotiation"); + proto + }); + + let (server_res, client_res) = timeout(Duration::from_secs(10), future::join(server, client)) + .await + .expect("negotiation completed within timeout"); + let server_proto = server_res.expect("server task"); + let client_proto = client_res.expect("client task"); + + println!("negotiated over TCP — listener: {server_proto:?}, dialer: {client_proto:?}"); + + // Both ends agreed on the exact slash-less token over the real socket. + assert_eq!(server_proto, PROTO); + assert_eq!(client_proto, PROTO); + assert_eq!(server_proto, "charon/priority/2.0.0"); + assert!(!server_proto.starts_with('/')); +} diff --git a/third_party/multistream-select/.cargo_vcs_info.json b/third_party/multistream-select/.cargo_vcs_info.json new file mode 100644 index 00000000..95e27ffb --- /dev/null +++ b/third_party/multistream-select/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "42566869b2ab3d1903d164c84da3308e7e890b8b" + }, + "path_in_vcs": "misc/multistream-select" +} \ No newline at end of file diff --git a/third_party/multistream-select/CHANGELOG.md b/third_party/multistream-select/CHANGELOG.md new file mode 100644 index 00000000..b34eff7b --- /dev/null +++ b/third_party/multistream-select/CHANGELOG.md @@ -0,0 +1,101 @@ +## 0.13.0 + +- Don't wait for negotiation on `::poll_close`. + This can save one round-trip for protocols that use stream closing as an operation in ones protocol, e.g. using stream closing to signal the end of a request. + See [PR 4019] for details. + +- Raise MSRV to 1.65. + See [PR 3715]. + +[PR 4019]: https://github.com/libp2p/rust-libp2p/pull/4019 +[PR 3715]: https://github.com/libp2p/rust-libp2p/pull/3715 + +## 0.12.1 + +- Update `rust-version` to reflect the actual MSRV: 1.60.0. See [PR 3090]. + +[PR 3090]: https://github.com/libp2p/rust-libp2p/pull/3090 + +## 0.12.0 + +- Remove parallel dialing optimization, to avoid requiring the use of the `ls` command. See [PR 2934]. + +[PR 2934]: https://github.com/libp2p/rust-libp2p/pull/2934 + +## 0.11.0 [2022-01-27] + +- Migrate to Rust edition 2021 (see [PR 2339]). + +[PR 2339]: https://github.com/libp2p/rust-libp2p/pull/2339 + +## 0.10.4 [2021-11-01] + +- Implement `From for ProtocolError` instead of `Into`. + [PR 2169](https://github.com/libp2p/rust-libp2p/pull/2169) + +## 0.10.3 [2021-03-17] + +- Update dependencies. + +## 0.10.2 [2021-03-01] + +- Re-enable "parallel negotiation" if the dialer has 3 or more + alternative protocols. + [PR 1934](https://github.com/libp2p/rust-libp2p/pull/1934) + +## 0.10.1 [2021-02-15] + +- Update dependencies. + +## 0.10.0 [2021-01-12] + +- Update dependencies. + +## 0.9.1 [2020-12-02] + +- Ensure uniform outcomes for failed negotiations with both + `V1` and `V1Lazy`. + [PR 1871](https://github.com/libp2p/rust-libp2p/pull/1871) + +## 0.9.0 [2020-11-25] + +- Make the `V1Lazy` upgrade strategy more interoperable with `V1`. Specifically, + the listener now behaves identically with `V1` and `V1Lazy`. Furthermore, the + multistream-select protocol header is now also identical, making `V1` and `V1Lazy` + indistinguishable on the wire. The remaining central effect of `V1Lazy` is that the dialer, + if it only supports a single protocol in a negotiation, optimistically settles on that + protocol without immediately flushing the negotiation data (i.e. protocol proposal) + and without waiting for the corresponding confirmation before it is able to start + sending application data, expecting the used protocol to be confirmed with + the response. + +- Fix the encoding and decoding of `ls` responses to + be spec-compliant and interoperable with other implementations. + For a clean upgrade, `0.8.4` must already be deployed. + +## 0.8.5 [2020-11-09] + +- During negotiation do not interpret EOF error as an IO error, but instead as a + negotiation error. See https://github.com/libp2p/rust-libp2p/pull/1823. + +## 0.8.4 [2020-10-20] + +- Temporarily disable the internal selection of "parallel" protocol + negotiation for the dialer to later change the response format of the "ls" + message for spec compliance. See https://github.com/libp2p/rust-libp2p/issues/1795. + +## 0.8.3 [2020-10-16] + +- Fix a regression resulting in a panic with the `V1Lazy` protocol. + [PR 1783](https://github.com/libp2p/rust-libp2p/pull/1783). + +- Fix a potential deadlock during protocol negotiation due + to a missing flush, potentially resulting in sporadic protocol + upgrade timeouts. + [PR 1781](https://github.com/libp2p/rust-libp2p/pull/1781). + +- Update dependencies. + +## 0.8.2 [2020-06-22] + +- Updated dependencies. diff --git a/third_party/multistream-select/Cargo.lock b/third_party/multistream-select/Cargo.lock new file mode 100644 index 00000000..aecbf032 --- /dev/null +++ b/third_party/multistream-select/Cargo.lock @@ -0,0 +1,860 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "async-attributes" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3203e79f4dd9bdda415ed03cf14dae5a2bf775c683a00f94e9cd1faf0f596e5" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "async-channel" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81953c529336010edd6d8e358f886d9581267795c61b19475b71314bffa46d35" +dependencies = [ + "concurrent-queue", + "event-listener 2.5.3", + "futures-core", +] + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-executor" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96bf972d85afc50bf5ab8fe2d54d1586b4e0b46c97c50a0c9e71e2f7bcd812a" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "pin-project-lite", + "slab", +] + +[[package]] +name = "async-global-executor" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" +dependencies = [ + "async-channel 2.5.0", + "async-executor", + "async-io", + "async-lock", + "blocking", + "futures-lite", + "once_cell", +] + +[[package]] +name = "async-io" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "456b8a8feb6f42d237746d4b3e9a178494627745c3c56c6ea55d92ba50d026fc" +dependencies = [ + "autocfg", + "cfg-if", + "concurrent-queue", + "futures-io", + "futures-lite", + "parking", + "polling", + "rustix", + "slab", + "windows-sys", +] + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener 5.4.1", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "async-std" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c8e079a4ab67ae52b7403632e4618815d6db36d2a010cfe41b02c1b1578f93b" +dependencies = [ + "async-attributes", + "async-channel 1.9.0", + "async-global-executor", + "async-io", + "async-lock", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "blocking" +version = "1.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e83f8d02be6967315521be875afa792a316e28d57b5a2d401897e2a7921b7f21" +dependencies = [ + "async-channel 2.5.0", + "async-task", + "futures-io", + "futures-lite", + "piper", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytes" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ae3f5d315924270530207e2a68396c3cc547f6dca3fbdca317cfb1a51edb593" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "env_logger" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" +dependencies = [ + "humantime", + "is-terminal", + "log", + "regex", + "termcolor", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "event-listener" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" + +[[package]] +name = "event-listener" +version = "5.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener 5.4.1", + "pin-project-lite", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "futures" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-lite" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f78e10609fe0e0b3f4157ffab1876319b5b0db102a2c60dc4626306dc46b44ad" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "parking", + "pin-project-lite", +] + +[[package]] +name = "futures-macro" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "futures_ringbuf" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6628abb6eb1fc74beaeb20cd0670c43d158b0150f7689b38c3eaf663f99bdec7" +dependencies = [ + "futures", + "log", + "ringbuf", + "rustc_version", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + +[[package]] +name = "is-terminal" +version = "0.4.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" +dependencies = [ + "hermit-abi", + "libc", + "windows-sys", +] + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "log" +version = "0.4.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953f07c43838f8e6f9758cab68bf5bed85465e7587ebe0b823f1bcd81978ad3a" +dependencies = [ + "value-bag", +] + +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + +[[package]] +name = "multistream-select" +version = "0.13.0" +dependencies = [ + "async-std", + "bytes", + "env_logger", + "futures", + "futures_ringbuf", + "log", + "pin-project", + "rand", + "rw-stream-sink", + "smallvec", + "unsigned-varint", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "pin-project" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2466b2336ed02bcdca6b294417127b90ec92038d1d5c4fbeac971a922e0e0924" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c96395f0a926bc13b1c17622aaddda1ecb55d49c8f1bf9777e4d877800a43f8b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "piper" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c835479a4443ded371d6c535cbfd8d31ad92c5d23ae9770a61bc155e4992a3c1" +dependencies = [ + "atomic-waker", + "fastrand", + "futures-io", +] + +[[package]] +name = "polling" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d0e4f59085d47d8241c88ead0f274e8a0cb551f3625263c05eb8dd897c34218" +dependencies = [ + "cfg-if", + "concurrent-queue", + "hermit-abi", + "pin-project-lite", + "rustix", + "windows-sys", +] + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41f2619966050689382d2b44f664f4bc593e129785a36d6ee376ddf37259b924" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "regex" +version = "1.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1292b7759ae1cb9ec195452d1390a074f0cd8541ab7a5a8c31cd6db45d4a6ba" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e1dd4122fc1595e8162618945476892eefca7b88c52820e74af6262213cae8f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "ringbuf" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79abed428d1fd2a128201cec72c5f6938e2da607c6f3745f769fabea399d950a" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "rw-stream-sink" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8c9026ff5d2f23da5e45bbc283f156383001bfb09c4e44256d02c1a685fe9a1" +dependencies = [ + "futures", + "pin-project", + "static_assertions", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unsigned-varint" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6889a77d49f1f013504cec6bf97a2c730394adedaeb1deb5ea08949a50541105" + +[[package]] +name = "value-bag" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ba6f5989077681266825251a52748b8c1d8a4ad098cc37e440103d0ea717fc0" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.118", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zerocopy" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce1022995ff5ff5d841ad7d994facc23098cd40152f2c1d11cd607c6f530653f" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.52" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ae7f38b72ec2a254e2b87ef277cf2cd4fb97cbebf944faa6f33354da0867930" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] diff --git a/third_party/multistream-select/Cargo.toml b/third_party/multistream-select/Cargo.toml new file mode 100644 index 00000000..57cbbd2c --- /dev/null +++ b/third_party/multistream-select/Cargo.toml @@ -0,0 +1,74 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.65.0" +name = "multistream-select" +version = "0.13.0" +authors = ["Parity Technologies "] +description = "Multistream-select negotiation protocol for libp2p" +keywords = [ + "peer-to-peer", + "libp2p", + "networking", +] +categories = [ + "network-programming", + "asynchronous", +] +license = "MIT" +repository = "https://github.com/libp2p/rust-libp2p" + +[package.metadata.docs.rs] +all-features = true +rustc-args = [ + "--cfg", + "docsrs", +] +rustdoc-args = [ + "--cfg", + "docsrs", +] + +[dependencies.bytes] +version = "1" + +[dependencies.futures] +version = "0.3" + +[dependencies.log] +version = "0.4" + +[dependencies.pin-project] +version = "1.1.0" + +[dependencies.smallvec] +version = "1.6.1" + +[dependencies.unsigned-varint] +version = "0.7" + +[dev-dependencies.async-std] +version = "1.6.2" +features = ["attributes"] + +[dev-dependencies.env_logger] +version = "0.10" + +[dev-dependencies.futures_ringbuf] +version = "0.4.0" + +[dev-dependencies.rand] +version = "0.8" + +[dev-dependencies.rw-stream-sink] +version = "0.4.0" diff --git a/third_party/multistream-select/Cargo.toml.orig b/third_party/multistream-select/Cargo.toml.orig new file mode 100644 index 00000000..f75852ba --- /dev/null +++ b/third_party/multistream-select/Cargo.toml.orig @@ -0,0 +1,34 @@ +[package] +name = "multistream-select" +edition = "2021" +rust-version = { workspace = true } +description = "Multistream-select negotiation protocol for libp2p" +version = "0.13.0" +authors = ["Parity Technologies "] +license = "MIT" +repository = "https://github.com/libp2p/rust-libp2p" +keywords = ["peer-to-peer", "libp2p", "networking"] +categories = ["network-programming", "asynchronous"] + +[dependencies] +bytes = "1" +futures = "0.3" +log = "0.4" +pin-project = "1.1.0" +smallvec = "1.6.1" +unsigned-varint = "0.7" + +[dev-dependencies] +async-std = { version = "1.6.2", features = ["attributes"] } +env_logger = "0.10" +futures_ringbuf = "0.4.0" +quickcheck = { workspace = true } +rand = "0.8" +rw-stream-sink = { workspace = true } + +# Passing arguments to the docsrs builder in order to properly document cfg's. +# More information: https://docs.rs/about/builds#cross-compiling +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] +rustc-args = ["--cfg", "docsrs"] diff --git a/third_party/multistream-select/PATCHES.md b/third_party/multistream-select/PATCHES.md new file mode 100644 index 00000000..75023152 --- /dev/null +++ b/third_party/multistream-select/PATCHES.md @@ -0,0 +1,54 @@ +# multistream-select (vendored fork of 0.13.0) + +Vendored from crates.io `multistream-select v0.13.0` and consumed via +`[patch.crates-io]` in the workspace `Cargo.toml`. + +## Why + +rust-libp2p's multistream-select requires every protocol token to begin with +`/` and rejects anything else at negotiation time (emit, advertise, and +decode). Charon advertises and dials the priority protocol as the slash-less +token `charon/priority/2.0.0` (the only Charon protocol without a leading +slash). To interoperate with unmodified Charon nodes, Pluto must speak that +exact token, which stock multistream-select cannot do. + +## Delta vs upstream 0.13.0 + +All changes are in `src/protocol.rs`, guarded by `// PLUTO PATCH:` comments. +The leading-`/` requirement is relaxed, while the empty-name and message +classification invariants the slash check previously also enforced are +preserved explicitly: + +1. `impl TryFrom<&str> for Protocol` — dropped the `!starts_with('/')` + rejection (outbound propose + listener advertise path); still rejects an + empty name. +2. `impl TryFrom for Protocol` — dropped the `!starts_with(b"/")` + rejection (inbound decode path); still rejects an empty name. +3. `Message::decode` single-protocol classification — dropped the + `msg.first() == Some(&b'/')` condition and replaced its bare-`"\n"` + exclusion with an explicit `msg.len() > 1` guard. A single protocol line is + distinguished from an `ls` response by having exactly one (trailing) `\n` + *and* a non-empty name; the empty `ls` response `Protocols([])` (which + encodes to `"\n"`) still parses as a zero-entry list, not an empty protocol. + +Net effect: acceptance is *widened* only to slash-less non-empty names; +slash-prefixed tokens and the empty-`ls` round-trip behave exactly as upstream, +so all other Pluto p2p protocols are unaffected. No other files differ from +upstream. + +## Tests + +The upstream quickcheck round-trip property test in `src/protocol.rs` (and its +`Arbitrary` impls) was removed: it depended on rust-libp2p's unpublished +workspace crate `quickcheck-ext`, which is unavailable from crates.io. It is +replaced by deterministic unit tests covering the patched behaviour — +slash-less single-protocol round-trip, empty-name rejection, and the empty +`Protocols([])` round-trip (the case the property test would have caught). The +upstream `tests/` integration tests (slash-prefixed negotiation) are unchanged. + +## Maintenance + +On a libp2p bump, re-vendor the matching multistream-select version and +re-apply these three edits. The `[patch]` requires the version to stay +semver-compatible with what libp2p depends on (`^0.13.0`), else cargo silently +ignores the patch. diff --git a/third_party/multistream-select/src/dialer_select.rs b/third_party/multistream-select/src/dialer_select.rs new file mode 100644 index 00000000..af9f79d8 --- /dev/null +++ b/third_party/multistream-select/src/dialer_select.rs @@ -0,0 +1,203 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//! Protocol negotiation strategies for the peer acting as the dialer. + +use crate::protocol::{HeaderLine, Message, MessageIO, Protocol, ProtocolError}; +use crate::{Negotiated, NegotiationError, Version}; + +use futures::prelude::*; +use std::{ + convert::TryFrom as _, + iter, mem, + pin::Pin, + task::{Context, Poll}, +}; + +/// Returns a `Future` that negotiates a protocol on the given I/O stream +/// for a peer acting as the _dialer_ (or _initiator_). +/// +/// This function is given an I/O stream and a list of protocols and returns a +/// computation that performs the protocol negotiation with the remote. The +/// returned `Future` resolves with the name of the negotiated protocol and +/// a [`Negotiated`] I/O stream. +/// +/// Within the scope of this library, a dialer always commits to a specific +/// multistream-select [`Version`], whereas a listener always supports +/// all versions supported by this library. Frictionless multistream-select +/// protocol upgrades may thus proceed by deployments with updated listeners, +/// eventually followed by deployments of dialers choosing the newer protocol. +pub fn dialer_select_proto( + inner: R, + protocols: I, + version: Version, +) -> DialerSelectFuture +where + R: AsyncRead + AsyncWrite, + I: IntoIterator, + I::Item: AsRef, +{ + let protocols = protocols.into_iter().peekable(); + DialerSelectFuture { + version, + protocols, + state: State::SendHeader { + io: MessageIO::new(inner), + }, + } +} + +/// A `Future` returned by [`dialer_select_proto`] which negotiates +/// a protocol iteratively by considering one protocol after the other. +#[pin_project::pin_project] +pub struct DialerSelectFuture { + // TODO: It would be nice if eventually N = I::Item = Protocol. + protocols: iter::Peekable, + state: State, + version: Version, +} + +enum State { + SendHeader { io: MessageIO }, + SendProtocol { io: MessageIO, protocol: N }, + FlushProtocol { io: MessageIO, protocol: N }, + AwaitProtocol { io: MessageIO, protocol: N }, + Done, +} + +impl Future for DialerSelectFuture +where + // The Unpin bound here is required because we produce a `Negotiated` as the output. + // It also makes the implementation considerably easier to write. + R: AsyncRead + AsyncWrite + Unpin, + I: Iterator, + I::Item: AsRef, +{ + type Output = Result<(I::Item, Negotiated), NegotiationError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + loop { + match mem::replace(this.state, State::Done) { + State::SendHeader { mut io } => { + match Pin::new(&mut io).poll_ready(cx)? { + Poll::Ready(()) => {} + Poll::Pending => { + *this.state = State::SendHeader { io }; + return Poll::Pending; + } + } + + let h = HeaderLine::from(*this.version); + if let Err(err) = Pin::new(&mut io).start_send(Message::Header(h)) { + return Poll::Ready(Err(From::from(err))); + } + + let protocol = this.protocols.next().ok_or(NegotiationError::Failed)?; + + // The dialer always sends the header and the first protocol + // proposal in one go for efficiency. + *this.state = State::SendProtocol { io, protocol }; + } + + State::SendProtocol { mut io, protocol } => { + match Pin::new(&mut io).poll_ready(cx)? { + Poll::Ready(()) => {} + Poll::Pending => { + *this.state = State::SendProtocol { io, protocol }; + return Poll::Pending; + } + } + + let p = Protocol::try_from(protocol.as_ref())?; + if let Err(err) = Pin::new(&mut io).start_send(Message::Protocol(p.clone())) { + return Poll::Ready(Err(From::from(err))); + } + log::debug!("Dialer: Proposed protocol: {}", p); + + if this.protocols.peek().is_some() { + *this.state = State::FlushProtocol { io, protocol } + } else { + match this.version { + Version::V1 => *this.state = State::FlushProtocol { io, protocol }, + // This is the only effect that `V1Lazy` has compared to `V1`: + // Optimistically settling on the only protocol that + // the dialer supports for this negotiation. Notably, + // the dialer expects a regular `V1` response. + Version::V1Lazy => { + log::debug!("Dialer: Expecting proposed protocol: {}", p); + let hl = HeaderLine::from(Version::V1Lazy); + let io = Negotiated::expecting(io.into_reader(), p, Some(hl)); + return Poll::Ready(Ok((protocol, io))); + } + } + } + } + + State::FlushProtocol { mut io, protocol } => { + match Pin::new(&mut io).poll_flush(cx)? { + Poll::Ready(()) => *this.state = State::AwaitProtocol { io, protocol }, + Poll::Pending => { + *this.state = State::FlushProtocol { io, protocol }; + return Poll::Pending; + } + } + } + + State::AwaitProtocol { mut io, protocol } => { + let msg = match Pin::new(&mut io).poll_next(cx)? { + Poll::Ready(Some(msg)) => msg, + Poll::Pending => { + *this.state = State::AwaitProtocol { io, protocol }; + return Poll::Pending; + } + // Treat EOF error as [`NegotiationError::Failed`], not as + // [`NegotiationError::ProtocolError`], allowing dropping or closing an I/O + // stream as a permissible way to "gracefully" fail a negotiation. + Poll::Ready(None) => return Poll::Ready(Err(NegotiationError::Failed)), + }; + + match msg { + Message::Header(v) if v == HeaderLine::from(*this.version) => { + *this.state = State::AwaitProtocol { io, protocol }; + } + Message::Protocol(ref p) if p.as_ref() == protocol.as_ref() => { + log::debug!("Dialer: Received confirmation for protocol: {}", p); + let io = Negotiated::completed(io.into_inner()); + return Poll::Ready(Ok((protocol, io))); + } + Message::NotAvailable => { + log::debug!( + "Dialer: Received rejection of protocol: {}", + protocol.as_ref() + ); + let protocol = this.protocols.next().ok_or(NegotiationError::Failed)?; + *this.state = State::SendProtocol { io, protocol } + } + _ => return Poll::Ready(Err(ProtocolError::InvalidMessage.into())), + } + } + + State::Done => panic!("State::poll called after completion"), + } + } + } +} diff --git a/third_party/multistream-select/src/length_delimited.rs b/third_party/multistream-select/src/length_delimited.rs new file mode 100644 index 00000000..9c95a6bf --- /dev/null +++ b/third_party/multistream-select/src/length_delimited.rs @@ -0,0 +1,489 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +use bytes::{Buf as _, BufMut as _, Bytes, BytesMut}; +use futures::{io::IoSlice, prelude::*}; +use std::{ + convert::TryFrom as _, + io, + pin::Pin, + task::{Context, Poll}, + u16, +}; + +const MAX_LEN_BYTES: u16 = 2; +const MAX_FRAME_SIZE: u16 = (1 << (MAX_LEN_BYTES * 8 - MAX_LEN_BYTES)) - 1; +const DEFAULT_BUFFER_SIZE: usize = 64; + +/// A `Stream` and `Sink` for unsigned-varint length-delimited frames, +/// wrapping an underlying `AsyncRead + AsyncWrite` I/O resource. +/// +/// We purposely only support a frame sizes up to 16KiB (2 bytes unsigned varint +/// frame length). Frames mostly consist in a short protocol name, which is highly +/// unlikely to be more than 16KiB long. +#[pin_project::pin_project] +#[derive(Debug)] +pub(crate) struct LengthDelimited { + /// The inner I/O resource. + #[pin] + inner: R, + /// Read buffer for a single incoming unsigned-varint length-delimited frame. + read_buffer: BytesMut, + /// Write buffer for outgoing unsigned-varint length-delimited frames. + write_buffer: BytesMut, + /// The current read state, alternating between reading a frame + /// length and reading a frame payload. + read_state: ReadState, +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +enum ReadState { + /// We are currently reading the length of the next frame of data. + ReadLength { + buf: [u8; MAX_LEN_BYTES as usize], + pos: usize, + }, + /// We are currently reading the frame of data itself. + ReadData { len: u16, pos: usize }, +} + +impl Default for ReadState { + fn default() -> Self { + ReadState::ReadLength { + buf: [0; MAX_LEN_BYTES as usize], + pos: 0, + } + } +} + +impl LengthDelimited { + /// Creates a new I/O resource for reading and writing unsigned-varint + /// length delimited frames. + pub(crate) fn new(inner: R) -> LengthDelimited { + LengthDelimited { + inner, + read_state: ReadState::default(), + read_buffer: BytesMut::with_capacity(DEFAULT_BUFFER_SIZE), + write_buffer: BytesMut::with_capacity(DEFAULT_BUFFER_SIZE + MAX_LEN_BYTES as usize), + } + } + + /// Drops the [`LengthDelimited`] resource, yielding the underlying I/O stream. + /// + /// # Panic + /// + /// Will panic if called while there is data in the read or write buffer. + /// The read buffer is guaranteed to be empty whenever `Stream::poll` yields + /// a new `Bytes` frame. The write buffer is guaranteed to be empty after + /// flushing. + pub(crate) fn into_inner(self) -> R { + assert!(self.read_buffer.is_empty()); + assert!(self.write_buffer.is_empty()); + self.inner + } + + /// Converts the [`LengthDelimited`] into a [`LengthDelimitedReader`], dropping the + /// uvi-framed `Sink` in favour of direct `AsyncWrite` access to the underlying + /// I/O stream. + /// + /// This is typically done if further uvi-framed messages are expected to be + /// received but no more such messages are written, allowing the writing of + /// follow-up protocol data to commence. + pub(crate) fn into_reader(self) -> LengthDelimitedReader { + LengthDelimitedReader { inner: self } + } + + /// Writes all buffered frame data to the underlying I/O stream, + /// _without flushing it_. + /// + /// After this method returns `Poll::Ready`, the write buffer of frames + /// submitted to the `Sink` is guaranteed to be empty. + fn poll_write_buffer(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> + where + R: AsyncWrite, + { + let mut this = self.project(); + + while !this.write_buffer.is_empty() { + match this.inner.as_mut().poll_write(cx, this.write_buffer) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Ok(0)) => { + return Poll::Ready(Err(io::Error::new( + io::ErrorKind::WriteZero, + "Failed to write buffered frame.", + ))) + } + Poll::Ready(Ok(n)) => this.write_buffer.advance(n), + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + } + } + + Poll::Ready(Ok(())) + } +} + +impl Stream for LengthDelimited +where + R: AsyncRead, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let mut this = self.project(); + + loop { + match this.read_state { + ReadState::ReadLength { buf, pos } => { + match this.inner.as_mut().poll_read(cx, &mut buf[*pos..*pos + 1]) { + Poll::Ready(Ok(0)) => { + if *pos == 0 { + return Poll::Ready(None); + } else { + return Poll::Ready(Some(Err(io::ErrorKind::UnexpectedEof.into()))); + } + } + Poll::Ready(Ok(n)) => { + debug_assert_eq!(n, 1); + *pos += n; + } + Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))), + Poll::Pending => return Poll::Pending, + }; + + if (buf[*pos - 1] & 0x80) == 0 { + // MSB is not set, indicating the end of the length prefix. + let (len, _) = unsigned_varint::decode::u16(buf).map_err(|e| { + log::debug!("invalid length prefix: {}", e); + io::Error::new(io::ErrorKind::InvalidData, "invalid length prefix") + })?; + + if len >= 1 { + *this.read_state = ReadState::ReadData { len, pos: 0 }; + this.read_buffer.resize(len as usize, 0); + } else { + debug_assert_eq!(len, 0); + *this.read_state = ReadState::default(); + return Poll::Ready(Some(Ok(Bytes::new()))); + } + } else if *pos == MAX_LEN_BYTES as usize { + // MSB signals more length bytes but we have already read the maximum. + // See the module documentation about the max frame len. + return Poll::Ready(Some(Err(io::Error::new( + io::ErrorKind::InvalidData, + "Maximum frame length exceeded", + )))); + } + } + ReadState::ReadData { len, pos } => { + match this + .inner + .as_mut() + .poll_read(cx, &mut this.read_buffer[*pos..]) + { + Poll::Ready(Ok(0)) => { + return Poll::Ready(Some(Err(io::ErrorKind::UnexpectedEof.into()))) + } + Poll::Ready(Ok(n)) => *pos += n, + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(err)) => return Poll::Ready(Some(Err(err))), + }; + + if *pos == *len as usize { + // Finished reading the frame. + let frame = this.read_buffer.split_off(0).freeze(); + *this.read_state = ReadState::default(); + return Poll::Ready(Some(Ok(frame))); + } + } + } + } + } +} + +impl Sink for LengthDelimited +where + R: AsyncWrite, +{ + type Error = io::Error; + + fn poll_ready(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // Use the maximum frame length also as a (soft) upper limit + // for the entire write buffer. The actual (hard) limit is thus + // implied to be roughly 2 * MAX_FRAME_SIZE. + if self.as_mut().project().write_buffer.len() >= MAX_FRAME_SIZE as usize { + match self.as_mut().poll_write_buffer(cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Pending => return Poll::Pending, + } + + debug_assert!(self.as_mut().project().write_buffer.is_empty()); + } + + Poll::Ready(Ok(())) + } + + fn start_send(self: Pin<&mut Self>, item: Bytes) -> Result<(), Self::Error> { + let this = self.project(); + + let len = match u16::try_from(item.len()) { + Ok(len) if len <= MAX_FRAME_SIZE => len, + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "Maximum frame size exceeded.", + )) + } + }; + + let mut uvi_buf = unsigned_varint::encode::u16_buffer(); + let uvi_len = unsigned_varint::encode::u16(len, &mut uvi_buf); + this.write_buffer.reserve(len as usize + uvi_len.len()); + this.write_buffer.put(uvi_len); + this.write_buffer.put(item); + + Ok(()) + } + + fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // Write all buffered frame data to the underlying I/O stream. + match LengthDelimited::poll_write_buffer(self.as_mut(), cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Pending => return Poll::Pending, + } + + let this = self.project(); + debug_assert!(this.write_buffer.is_empty()); + + // Flush the underlying I/O stream. + this.inner.poll_flush(cx) + } + + fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // Write all buffered frame data to the underlying I/O stream. + match LengthDelimited::poll_write_buffer(self.as_mut(), cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Pending => return Poll::Pending, + } + + let this = self.project(); + debug_assert!(this.write_buffer.is_empty()); + + // Close the underlying I/O stream. + this.inner.poll_close(cx) + } +} + +/// A `LengthDelimitedReader` implements a `Stream` of uvi-length-delimited +/// frames on an underlying I/O resource combined with direct `AsyncWrite` access. +#[pin_project::pin_project] +#[derive(Debug)] +pub(crate) struct LengthDelimitedReader { + #[pin] + inner: LengthDelimited, +} + +impl LengthDelimitedReader { + /// Destroys the `LengthDelimitedReader` and returns the underlying I/O stream. + /// + /// This method is guaranteed not to drop any data read from or not yet + /// submitted to the underlying I/O stream. + /// + /// # Panic + /// + /// Will panic if called while there is data in the read or write buffer. + /// The read buffer is guaranteed to be empty whenever [`Stream::poll_next`] + /// yield a new `Message`. The write buffer is guaranteed to be empty whenever + /// [`LengthDelimited::poll_write_buffer`] yields [`Poll::Ready`] or after + /// the [`Sink`] has been completely flushed via [`Sink::poll_flush`]. + pub(crate) fn into_inner(self) -> R { + self.inner.into_inner() + } +} + +impl Stream for LengthDelimitedReader +where + R: AsyncRead, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_next(cx) + } +} + +impl AsyncWrite for LengthDelimitedReader +where + R: AsyncWrite, +{ + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + // `this` here designates the `LengthDelimited`. + let mut this = self.project().inner; + + // We need to flush any data previously written with the `LengthDelimited`. + match LengthDelimited::poll_write_buffer(this.as_mut(), cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Pending => return Poll::Pending, + } + debug_assert!(this.write_buffer.is_empty()); + + this.project().inner.poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_flush(cx) + } + + fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_close(cx) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[IoSlice<'_>], + ) -> Poll> { + // `this` here designates the `LengthDelimited`. + let mut this = self.project().inner; + + // We need to flush any data previously written with the `LengthDelimited`. + match LengthDelimited::poll_write_buffer(this.as_mut(), cx) { + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => return Poll::Ready(Err(err)), + Poll::Pending => return Poll::Pending, + } + debug_assert!(this.write_buffer.is_empty()); + + this.project().inner.poll_write_vectored(cx, bufs) + } +} + +#[cfg(test)] +mod tests { + use crate::length_delimited::LengthDelimited; + use futures::{io::Cursor, prelude::*}; + use std::io::ErrorKind; + + #[test] + fn basic_read() { + let data = vec![6, 9, 8, 7, 6, 5, 4]; + let framed = LengthDelimited::new(Cursor::new(data)); + let recved = futures::executor::block_on(framed.try_collect::>()).unwrap(); + assert_eq!(recved, vec![vec![9, 8, 7, 6, 5, 4]]); + } + + #[test] + fn basic_read_two() { + let data = vec![6, 9, 8, 7, 6, 5, 4, 3, 9, 8, 7]; + let framed = LengthDelimited::new(Cursor::new(data)); + let recved = futures::executor::block_on(framed.try_collect::>()).unwrap(); + assert_eq!(recved, vec![vec![9, 8, 7, 6, 5, 4], vec![9, 8, 7]]); + } + + #[test] + fn two_bytes_long_packet() { + let len = 5000u16; + assert!(len < (1 << 15)); + let frame = (0..len).map(|n| (n & 0xff) as u8).collect::>(); + let mut data = vec![(len & 0x7f) as u8 | 0x80, (len >> 7) as u8]; + data.extend(frame.clone().into_iter()); + let mut framed = LengthDelimited::new(Cursor::new(data)); + let recved = futures::executor::block_on(async move { framed.next().await }).unwrap(); + assert_eq!(recved.unwrap(), frame); + } + + #[test] + fn packet_len_too_long() { + let mut data = vec![0x81, 0x81, 0x1]; + data.extend((0..16513).map(|_| 0)); + let mut framed = LengthDelimited::new(Cursor::new(data)); + let recved = futures::executor::block_on(async move { framed.next().await.unwrap() }); + + if let Err(io_err) = recved { + assert_eq!(io_err.kind(), ErrorKind::InvalidData) + } else { + panic!() + } + } + + #[test] + fn empty_frames() { + let data = vec![0, 0, 6, 9, 8, 7, 6, 5, 4, 0, 3, 9, 8, 7]; + let framed = LengthDelimited::new(Cursor::new(data)); + let recved = futures::executor::block_on(framed.try_collect::>()).unwrap(); + assert_eq!( + recved, + vec![ + vec![], + vec![], + vec![9, 8, 7, 6, 5, 4], + vec![], + vec![9, 8, 7], + ] + ); + } + + #[test] + fn unexpected_eof_in_len() { + let data = vec![0x89]; + let framed = LengthDelimited::new(Cursor::new(data)); + let recved = futures::executor::block_on(framed.try_collect::>()); + if let Err(io_err) = recved { + assert_eq!(io_err.kind(), ErrorKind::UnexpectedEof) + } else { + panic!() + } + } + + #[test] + fn unexpected_eof_in_data() { + let data = vec![5]; + let framed = LengthDelimited::new(Cursor::new(data)); + let recved = futures::executor::block_on(framed.try_collect::>()); + if let Err(io_err) = recved { + assert_eq!(io_err.kind(), ErrorKind::UnexpectedEof) + } else { + panic!() + } + } + + #[test] + fn unexpected_eof_in_data2() { + let data = vec![5, 9, 8, 7]; + let framed = LengthDelimited::new(Cursor::new(data)); + let recved = futures::executor::block_on(framed.try_collect::>()); + if let Err(io_err) = recved { + assert_eq!(io_err.kind(), ErrorKind::UnexpectedEof) + } else { + panic!() + } + } + + // PLUTO: removed the upstream `writing_reading` quickcheck property test — + // it depended on rust-libp2p's unpublished `quickcheck-ext`. The + // deterministic read/write tests above retain length-delimited coverage. +} diff --git a/third_party/multistream-select/src/lib.rs b/third_party/multistream-select/src/lib.rs new file mode 100644 index 00000000..ec62023a --- /dev/null +++ b/third_party/multistream-select/src/lib.rs @@ -0,0 +1,144 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//! # Multistream-select Protocol Negotiation +//! +//! This crate implements the `multistream-select` protocol, which is the protocol +//! used by libp2p to negotiate which application-layer protocol to use with the +//! remote on a connection or substream. +//! +//! > **Note**: This crate is used primarily by core components of *libp2p* and it +//! > is usually not used directly on its own. +//! +//! ## Roles +//! +//! Two peers using the multistream-select negotiation protocol on an I/O stream +//! are distinguished by their role as a _dialer_ (or _initiator_) or as a _listener_ +//! (or _responder_). Thereby the dialer plays the active part, driving the protocol, +//! whereas the listener reacts to the messages received. +//! +//! The dialer has two options: it can either pick a protocol from the complete list +//! of protocols that the listener supports, or it can directly suggest a protocol. +//! Either way, a selected protocol is sent to the listener who can either accept (by +//! echoing the same protocol) or reject (by responding with a message stating +//! "not available"). If a suggested protocol is not available, the dialer may +//! suggest another protocol. This process continues until a protocol is agreed upon, +//! yielding a [`Negotiated`](self::Negotiated) stream, or the dialer has run out of +//! alternatives. +//! +//! See [`dialer_select_proto`](self::dialer_select_proto) and +//! [`listener_select_proto`](self::listener_select_proto). +//! +//! ## [`Negotiated`](self::Negotiated) +//! +//! A `Negotiated` represents an I/O stream that has settled on a protocol +//! to use. By default, with [`Version::V1`], protocol negotiation is always +//! at least one dedicated round-trip message exchange, before application +//! data for the negotiated protocol can be sent by the dialer. There is +//! a variant [`Version::V1Lazy`] that permits 0-RTT negotiation if the +//! dialer only supports a single protocol. In that case, when a dialer +//! settles on a protocol to use, the [`DialerSelectFuture`] yields a +//! [`Negotiated`](self::Negotiated) I/O stream before the negotiation +//! data has been flushed. It is then expecting confirmation for that protocol +//! as the first messages read from the stream. This behaviour allows the dialer +//! to immediately send data relating to the negotiated protocol together with the +//! remaining negotiation message(s). Note, however, that a dialer that performs +//! multiple 0-RTT negotiations in sequence for different protocols layered on +//! top of each other may trigger undesirable behaviour for a listener not +//! supporting one of the intermediate protocols. See +//! [`dialer_select_proto`](self::dialer_select_proto) and the documentation +//! of [`Version::V1Lazy`] for further details. +//! +//! ## Examples +//! +//! For a dialer: +//! +//! ```no_run +//! use async_std::net::TcpStream; +//! use multistream_select::{dialer_select_proto, Version}; +//! use futures::prelude::*; +//! +//! async_std::task::block_on(async move { +//! let socket = TcpStream::connect("127.0.0.1:10333").await.unwrap(); +//! +//! let protos = vec!["/echo/1.0.0", "/echo/2.5.0"]; +//! let (protocol, _io) = dialer_select_proto(socket, protos, Version::V1).await.unwrap(); +//! +//! println!("Negotiated protocol: {:?}", protocol); +//! // You can now use `_io` to communicate with the remote. +//! }); +//! ``` +//! + +#![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] + +mod dialer_select; +mod length_delimited; +mod listener_select; +mod negotiated; +mod protocol; + +pub use self::dialer_select::{dialer_select_proto, DialerSelectFuture}; +pub use self::listener_select::{listener_select_proto, ListenerSelectFuture}; +pub use self::negotiated::{Negotiated, NegotiatedComplete, NegotiationError}; +pub use self::protocol::ProtocolError; + +/// Supported multistream-select versions. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum Version { + /// Version 1 of the multistream-select protocol. See [1] and [2]. + /// + /// [1]: https://github.com/libp2p/specs/blob/master/connections/README.md#protocol-negotiation + /// [2]: https://github.com/multiformats/multistream-select + #[default] + V1, + /// A "lazy" variant of version 1 that is identical on the wire but whereby + /// the dialer delays flushing protocol negotiation data in order to combine + /// it with initial application data, thus performing 0-RTT negotiation. + /// + /// This strategy is only applicable for the node with the role of "dialer" + /// in the negotiation and only if the dialer supports just a single + /// application protocol. In that case the dialer immedidately "settles" + /// on that protocol, buffering the negotiation messages to be sent + /// with the first round of application protocol data (or an attempt + /// is made to read from the `Negotiated` I/O stream). + /// + /// A listener will behave identically to `V1`. This ensures interoperability with `V1`. + /// Notably, it will immediately send the multistream header as well as the protocol + /// confirmation, resulting in multiple frames being sent on the underlying transport. + /// Nevertheless, if the listener supports the protocol that the dialer optimistically + /// settled on, it can be a 0-RTT negotiation. + /// + /// > **Note**: `V1Lazy` is specific to `rust-libp2p`. The wire protocol is identical to `V1` + /// > and generally interoperable with peers only supporting `V1`. Nevertheless, there is a + /// > pitfall that is rarely encountered: When nesting multiple protocol negotiations, the + /// > listener should either be known to support all of the dialer's optimistically chosen + /// > protocols or there is must be no intermediate protocol without a payload and none of + /// > the protocol payloads must have the potential for being mistaken for a multistream-select + /// > protocol message. This avoids rare edge-cases whereby the listener may not recognize + /// > upgrade boundaries and erroneously process a request despite not supporting one of + /// > the intermediate protocols that the dialer committed to. See [1] and [2]. + /// + /// [1]: https://github.com/multiformats/go-multistream/issues/20 + /// [2]: https://github.com/libp2p/rust-libp2p/pull/1212 + V1Lazy, + // Draft: https://github.com/libp2p/specs/pull/95 + // V2, +} diff --git a/third_party/multistream-select/src/listener_select.rs b/third_party/multistream-select/src/listener_select.rs new file mode 100644 index 00000000..5386114f --- /dev/null +++ b/third_party/multistream-select/src/listener_select.rs @@ -0,0 +1,308 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//! Protocol negotiation strategies for the peer acting as the listener +//! in a multistream-select protocol negotiation. + +use crate::protocol::{HeaderLine, Message, MessageIO, Protocol, ProtocolError}; +use crate::{Negotiated, NegotiationError}; + +use futures::prelude::*; +use smallvec::SmallVec; +use std::{ + convert::TryFrom as _, + iter::FromIterator, + mem, + pin::Pin, + task::{Context, Poll}, +}; + +/// Returns a `Future` that negotiates a protocol on the given I/O stream +/// for a peer acting as the _listener_ (or _responder_). +/// +/// This function is given an I/O stream and a list of protocols and returns a +/// computation that performs the protocol negotiation with the remote. The +/// returned `Future` resolves with the name of the negotiated protocol and +/// a [`Negotiated`] I/O stream. +pub fn listener_select_proto(inner: R, protocols: I) -> ListenerSelectFuture +where + R: AsyncRead + AsyncWrite, + I: IntoIterator, + I::Item: AsRef, +{ + let protocols = protocols + .into_iter() + .filter_map(|n| match Protocol::try_from(n.as_ref()) { + Ok(p) => Some((n, p)), + Err(e) => { + log::warn!( + "Listener: Ignoring invalid protocol: {} due to {}", + n.as_ref(), + e + ); + None + } + }); + ListenerSelectFuture { + protocols: SmallVec::from_iter(protocols), + state: State::RecvHeader { + io: MessageIO::new(inner), + }, + last_sent_na: false, + } +} + +/// The `Future` returned by [`listener_select_proto`] that performs a +/// multistream-select protocol negotiation on an underlying I/O stream. +#[pin_project::pin_project] +pub struct ListenerSelectFuture { + // TODO: It would be nice if eventually N = Protocol, which has a + // few more implications on the API. + protocols: SmallVec<[(N, Protocol); 8]>, + state: State, + /// Whether the last message sent was a protocol rejection (i.e. `na\n`). + /// + /// If the listener reads garbage or EOF after such a rejection, + /// the dialer is likely using `V1Lazy` and negotiation must be + /// considered failed, but not with a protocol violation or I/O + /// error. + last_sent_na: bool, +} + +enum State { + RecvHeader { + io: MessageIO, + }, + SendHeader { + io: MessageIO, + }, + RecvMessage { + io: MessageIO, + }, + SendMessage { + io: MessageIO, + message: Message, + protocol: Option, + }, + Flush { + io: MessageIO, + protocol: Option, + }, + Done, +} + +impl Future for ListenerSelectFuture +where + // The Unpin bound here is required because we produce a `Negotiated` as the output. + // It also makes the implementation considerably easier to write. + R: AsyncRead + AsyncWrite + Unpin, + N: AsRef + Clone, +{ + type Output = Result<(N, Negotiated), NegotiationError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.project(); + + loop { + match mem::replace(this.state, State::Done) { + State::RecvHeader { mut io } => { + match io.poll_next_unpin(cx) { + Poll::Ready(Some(Ok(Message::Header(h)))) => match h { + HeaderLine::V1 => *this.state = State::SendHeader { io }, + }, + Poll::Ready(Some(Ok(_))) => { + return Poll::Ready(Err(ProtocolError::InvalidMessage.into())) + } + Poll::Ready(Some(Err(err))) => return Poll::Ready(Err(From::from(err))), + // Treat EOF error as [`NegotiationError::Failed`], not as + // [`NegotiationError::ProtocolError`], allowing dropping or closing an I/O + // stream as a permissible way to "gracefully" fail a negotiation. + Poll::Ready(None) => return Poll::Ready(Err(NegotiationError::Failed)), + Poll::Pending => { + *this.state = State::RecvHeader { io }; + return Poll::Pending; + } + } + } + + State::SendHeader { mut io } => { + match Pin::new(&mut io).poll_ready(cx) { + Poll::Pending => { + *this.state = State::SendHeader { io }; + return Poll::Pending; + } + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => return Poll::Ready(Err(From::from(err))), + } + + let msg = Message::Header(HeaderLine::V1); + if let Err(err) = Pin::new(&mut io).start_send(msg) { + return Poll::Ready(Err(From::from(err))); + } + + *this.state = State::Flush { io, protocol: None }; + } + + State::RecvMessage { mut io } => { + let msg = match Pin::new(&mut io).poll_next(cx) { + Poll::Ready(Some(Ok(msg))) => msg, + // Treat EOF error as [`NegotiationError::Failed`], not as + // [`NegotiationError::ProtocolError`], allowing dropping or closing an I/O + // stream as a permissible way to "gracefully" fail a negotiation. + // + // This is e.g. important when a listener rejects a protocol with + // [`Message::NotAvailable`] and the dialer does not have alternative + // protocols to propose. Then the dialer will stop the negotiation and drop + // the corresponding stream. As a listener this EOF should be interpreted as + // a failed negotiation. + Poll::Ready(None) => return Poll::Ready(Err(NegotiationError::Failed)), + Poll::Pending => { + *this.state = State::RecvMessage { io }; + return Poll::Pending; + } + Poll::Ready(Some(Err(err))) => { + if *this.last_sent_na { + // When we read garbage or EOF after having already rejected a + // protocol, the dialer is most likely using `V1Lazy` and has + // optimistically settled on this protocol, so this is really a + // failed negotiation, not a protocol violation. In this case + // the dialer also raises `NegotiationError::Failed` when finally + // reading the `N/A` response. + if let ProtocolError::InvalidMessage = &err { + log::trace!( + "Listener: Negotiation failed with invalid \ + message after protocol rejection." + ); + return Poll::Ready(Err(NegotiationError::Failed)); + } + if let ProtocolError::IoError(e) = &err { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + log::trace!( + "Listener: Negotiation failed with EOF \ + after protocol rejection." + ); + return Poll::Ready(Err(NegotiationError::Failed)); + } + } + } + + return Poll::Ready(Err(From::from(err))); + } + }; + + match msg { + Message::ListProtocols => { + let supported = + this.protocols.iter().map(|(_, p)| p).cloned().collect(); + let message = Message::Protocols(supported); + *this.state = State::SendMessage { + io, + message, + protocol: None, + } + } + Message::Protocol(p) => { + let protocol = this.protocols.iter().find_map(|(name, proto)| { + if &p == proto { + Some(name.clone()) + } else { + None + } + }); + + let message = if protocol.is_some() { + log::debug!("Listener: confirming protocol: {}", p); + Message::Protocol(p.clone()) + } else { + log::debug!("Listener: rejecting protocol: {}", p.as_ref()); + Message::NotAvailable + }; + + *this.state = State::SendMessage { + io, + message, + protocol, + }; + } + _ => return Poll::Ready(Err(ProtocolError::InvalidMessage.into())), + } + } + + State::SendMessage { + mut io, + message, + protocol, + } => { + match Pin::new(&mut io).poll_ready(cx) { + Poll::Pending => { + *this.state = State::SendMessage { + io, + message, + protocol, + }; + return Poll::Pending; + } + Poll::Ready(Ok(())) => {} + Poll::Ready(Err(err)) => return Poll::Ready(Err(From::from(err))), + } + + if let Message::NotAvailable = &message { + *this.last_sent_na = true; + } else { + *this.last_sent_na = false; + } + + if let Err(err) = Pin::new(&mut io).start_send(message) { + return Poll::Ready(Err(From::from(err))); + } + + *this.state = State::Flush { io, protocol }; + } + + State::Flush { mut io, protocol } => { + match Pin::new(&mut io).poll_flush(cx) { + Poll::Pending => { + *this.state = State::Flush { io, protocol }; + return Poll::Pending; + } + Poll::Ready(Ok(())) => { + // If a protocol has been selected, finish negotiation. + // Otherwise expect to receive another message. + match protocol { + Some(protocol) => { + log::debug!( + "Listener: sent confirmed protocol: {}", + protocol.as_ref() + ); + let io = Negotiated::completed(io.into_inner()); + return Poll::Ready(Ok((protocol, io))); + } + None => *this.state = State::RecvMessage { io }, + } + } + Poll::Ready(Err(err)) => return Poll::Ready(Err(From::from(err))), + } + } + + State::Done => panic!("State::poll called after completion"), + } + } + } +} diff --git a/third_party/multistream-select/src/negotiated.rs b/third_party/multistream-select/src/negotiated.rs new file mode 100644 index 00000000..941b6076 --- /dev/null +++ b/third_party/multistream-select/src/negotiated.rs @@ -0,0 +1,390 @@ +// Copyright 2019 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +use crate::protocol::{HeaderLine, Message, MessageReader, Protocol, ProtocolError}; + +use futures::{ + io::{IoSlice, IoSliceMut}, + prelude::*, + ready, +}; +use pin_project::pin_project; +use std::{ + error::Error, + fmt, io, mem, + pin::Pin, + task::{Context, Poll}, +}; + +/// An I/O stream that has settled on an (application-layer) protocol to use. +/// +/// A `Negotiated` represents an I/O stream that has _settled_ on a protocol +/// to use. In particular, it is not implied that all of the protocol negotiation +/// frames have yet been sent and / or received, just that the selected protocol +/// is fully determined. This is to allow the last protocol negotiation frames +/// sent by a peer to be combined in a single write, possibly piggy-backing +/// data from the negotiated protocol on top. +/// +/// Reading from a `Negotiated` I/O stream that still has pending negotiation +/// protocol data to send implicitly triggers flushing of all yet unsent data. +#[pin_project] +#[derive(Debug)] +pub struct Negotiated { + #[pin] + state: State, +} + +/// A `Future` that waits on the completion of protocol negotiation. +#[derive(Debug)] +pub struct NegotiatedComplete { + inner: Option>, +} + +impl Future for NegotiatedComplete +where + // `Unpin` is required not because of implementation details but because we produce the + // `Negotiated` as the output of the future. + TInner: AsyncRead + AsyncWrite + Unpin, +{ + type Output = Result, NegotiationError>; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut io = self + .inner + .take() + .expect("NegotiatedFuture called after completion."); + match Negotiated::poll(Pin::new(&mut io), cx) { + Poll::Pending => { + self.inner = Some(io); + Poll::Pending + } + Poll::Ready(Ok(())) => Poll::Ready(Ok(io)), + Poll::Ready(Err(err)) => { + self.inner = Some(io); + Poll::Ready(Err(err)) + } + } + } +} + +impl Negotiated { + /// Creates a `Negotiated` in state [`State::Completed`]. + pub(crate) fn completed(io: TInner) -> Self { + Negotiated { + state: State::Completed { io }, + } + } + + /// Creates a `Negotiated` in state [`State::Expecting`] that is still + /// expecting confirmation of the given `protocol`. + pub(crate) fn expecting( + io: MessageReader, + protocol: Protocol, + header: Option, + ) -> Self { + Negotiated { + state: State::Expecting { + io, + protocol, + header, + }, + } + } + + /// Polls the `Negotiated` for completion. + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> + where + TInner: AsyncRead + AsyncWrite + Unpin, + { + // Flush any pending negotiation data. + match self.as_mut().poll_flush(cx) { + Poll::Ready(Ok(())) => {} + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(e)) => { + // If the remote closed the stream, it is important to still + // continue reading the data that was sent, if any. + if e.kind() != io::ErrorKind::WriteZero { + return Poll::Ready(Err(e.into())); + } + } + } + + let mut this = self.project(); + + if let StateProj::Completed { .. } = this.state.as_mut().project() { + return Poll::Ready(Ok(())); + } + + // Read outstanding protocol negotiation messages. + loop { + match mem::replace(&mut *this.state, State::Invalid) { + State::Expecting { + mut io, + header, + protocol, + } => { + let msg = match Pin::new(&mut io).poll_next(cx)? { + Poll::Ready(Some(msg)) => msg, + Poll::Pending => { + *this.state = State::Expecting { + io, + header, + protocol, + }; + return Poll::Pending; + } + Poll::Ready(None) => { + return Poll::Ready(Err(ProtocolError::IoError( + io::ErrorKind::UnexpectedEof.into(), + ) + .into())); + } + }; + + if let Message::Header(h) = &msg { + if Some(h) == header.as_ref() { + *this.state = State::Expecting { + io, + protocol, + header: None, + }; + continue; + } + } + + if let Message::Protocol(p) = &msg { + if p.as_ref() == protocol.as_ref() { + log::debug!("Negotiated: Received confirmation for protocol: {}", p); + *this.state = State::Completed { + io: io.into_inner(), + }; + return Poll::Ready(Ok(())); + } + } + + return Poll::Ready(Err(NegotiationError::Failed)); + } + + _ => panic!("Negotiated: Invalid state"), + } + } + } + + /// Returns a [`NegotiatedComplete`] future that waits for protocol + /// negotiation to complete. + pub fn complete(self) -> NegotiatedComplete { + NegotiatedComplete { inner: Some(self) } + } +} + +/// The states of a `Negotiated` I/O stream. +#[pin_project(project = StateProj)] +#[derive(Debug)] +enum State { + /// In this state, a `Negotiated` is still expecting to + /// receive confirmation of the protocol it has optimistically + /// settled on. + Expecting { + /// The underlying I/O stream. + #[pin] + io: MessageReader, + /// The expected negotiation header/preamble (i.e. multistream-select version), + /// if one is still expected to be received. + header: Option, + /// The expected application protocol (i.e. name and version). + protocol: Protocol, + }, + + /// In this state, a protocol has been agreed upon and I/O + /// on the underlying stream can commence. + Completed { + #[pin] + io: R, + }, + + /// Temporary state while moving the `io` resource from + /// `Expecting` to `Completed`. + Invalid, +} + +impl AsyncRead for Negotiated +where + TInner: AsyncRead + AsyncWrite + Unpin, +{ + fn poll_read( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut [u8], + ) -> Poll> { + loop { + if let StateProj::Completed { io } = self.as_mut().project().state.project() { + // If protocol negotiation is complete, commence with reading. + return io.poll_read(cx, buf); + } + + // Poll the `Negotiated`, driving protocol negotiation to completion, + // including flushing of any remaining data. + match self.as_mut().poll(cx) { + Poll::Ready(Ok(())) => {} + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(err)) => return Poll::Ready(Err(From::from(err))), + } + } + } + + // TODO: implement once method is stabilized in the futures crate + /*unsafe fn initializer(&self) -> Initializer { + match &self.state { + State::Completed { io, .. } => io.initializer(), + State::Expecting { io, .. } => io.inner_ref().initializer(), + State::Invalid => panic!("Negotiated: Invalid state"), + } + }*/ + + fn poll_read_vectored( + mut self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &mut [IoSliceMut<'_>], + ) -> Poll> { + loop { + if let StateProj::Completed { io } = self.as_mut().project().state.project() { + // If protocol negotiation is complete, commence with reading. + return io.poll_read_vectored(cx, bufs); + } + + // Poll the `Negotiated`, driving protocol negotiation to completion, + // including flushing of any remaining data. + match self.as_mut().poll(cx) { + Poll::Ready(Ok(())) => {} + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(err)) => return Poll::Ready(Err(From::from(err))), + } + } + } +} + +impl AsyncWrite for Negotiated +where + TInner: AsyncWrite + AsyncRead + Unpin, +{ + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + match self.project().state.project() { + StateProj::Completed { io } => io.poll_write(cx, buf), + StateProj::Expecting { io, .. } => io.poll_write(cx, buf), + StateProj::Invalid => panic!("Negotiated: Invalid state"), + } + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.project().state.project() { + StateProj::Completed { io } => io.poll_flush(cx), + StateProj::Expecting { io, .. } => io.poll_flush(cx), + StateProj::Invalid => panic!("Negotiated: Invalid state"), + } + } + + fn poll_close(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + // Ensure all data has been flushed, including optimistic multistream-select messages. + ready!(self + .as_mut() + .poll_flush(cx) + .map_err(Into::::into)?); + + // Continue with the shutdown of the underlying I/O stream. + match self.project().state.project() { + StateProj::Completed { io, .. } => io.poll_close(cx), + StateProj::Expecting { io, .. } => { + let close_poll = io.poll_close(cx); + if let Poll::Ready(Ok(())) = close_poll { + log::debug!("Stream closed. Confirmation from remote for optimstic protocol negotiation still pending.") + } + close_poll + } + StateProj::Invalid => panic!("Negotiated: Invalid state"), + } + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[IoSlice<'_>], + ) -> Poll> { + match self.project().state.project() { + StateProj::Completed { io } => io.poll_write_vectored(cx, bufs), + StateProj::Expecting { io, .. } => io.poll_write_vectored(cx, bufs), + StateProj::Invalid => panic!("Negotiated: Invalid state"), + } + } +} + +/// Error that can happen when negotiating a protocol with the remote. +#[derive(Debug)] +pub enum NegotiationError { + /// A protocol error occurred during the negotiation. + ProtocolError(ProtocolError), + + /// Protocol negotiation failed because no protocol could be agreed upon. + Failed, +} + +impl From for NegotiationError { + fn from(err: ProtocolError) -> NegotiationError { + NegotiationError::ProtocolError(err) + } +} + +impl From for NegotiationError { + fn from(err: io::Error) -> NegotiationError { + ProtocolError::from(err).into() + } +} + +impl From for io::Error { + fn from(err: NegotiationError) -> io::Error { + if let NegotiationError::ProtocolError(e) = err { + return e.into(); + } + io::Error::new(io::ErrorKind::Other, err) + } +} + +impl Error for NegotiationError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + NegotiationError::ProtocolError(err) => Some(err), + _ => None, + } + } +} + +impl fmt::Display for NegotiationError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + match self { + NegotiationError::ProtocolError(p) => { + fmt.write_fmt(format_args!("Protocol error: {p}")) + } + NegotiationError::Failed => fmt.write_str("Protocol negotiation failed."), + } + } +} diff --git a/third_party/multistream-select/src/protocol.rs b/third_party/multistream-select/src/protocol.rs new file mode 100644 index 00000000..e2dfefb2 --- /dev/null +++ b/third_party/multistream-select/src/protocol.rs @@ -0,0 +1,529 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//! Multistream-select protocol messages an I/O operations for +//! constructing protocol negotiation flows. +//! +//! A protocol negotiation flow is constructed by using the +//! `Stream` and `Sink` implementations of `MessageIO` and +//! `MessageReader`. + +use crate::length_delimited::{LengthDelimited, LengthDelimitedReader}; +use crate::Version; + +use bytes::{BufMut, Bytes, BytesMut}; +use futures::{io::IoSlice, prelude::*, ready}; +use std::{ + convert::TryFrom, + error::Error, + fmt, io, + pin::Pin, + task::{Context, Poll}, +}; +use unsigned_varint as uvi; + +/// The maximum number of supported protocols that can be processed. +const MAX_PROTOCOLS: usize = 1000; + +/// The encoded form of a multistream-select 1.0.0 header message. +const MSG_MULTISTREAM_1_0: &[u8] = b"/multistream/1.0.0\n"; +/// The encoded form of a multistream-select 'na' message. +const MSG_PROTOCOL_NA: &[u8] = b"na\n"; +/// The encoded form of a multistream-select 'ls' message. +const MSG_LS: &[u8] = b"ls\n"; + +/// The multistream-select header lines preceeding negotiation. +/// +/// Every [`Version`] has a corresponding header line. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub(crate) enum HeaderLine { + /// The `/multistream/1.0.0` header line. + V1, +} + +impl From for HeaderLine { + fn from(v: Version) -> HeaderLine { + match v { + Version::V1 | Version::V1Lazy => HeaderLine::V1, + } + } +} + +/// A protocol (name) exchanged during protocol negotiation. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct Protocol(String); +impl AsRef for Protocol { + fn as_ref(&self) -> &str { + self.0.as_ref() + } +} + +impl TryFrom for Protocol { + type Error = ProtocolError; + + fn try_from(value: Bytes) -> Result { + // PLUTO PATCH: upstream rejects protocol names without a leading '/'. + // Relaxed to accept slash-less names (e.g. Charon's "charon/priority/2.0.0"), + // but an empty name is still invalid. See + // third_party/multistream-select/PATCHES.md. + if value.is_empty() { + return Err(ProtocolError::InvalidProtocol); + } + let protocol_as_string = + String::from_utf8(value.to_vec()).map_err(|_| ProtocolError::InvalidProtocol)?; + + Ok(Protocol(protocol_as_string)) + } +} + +impl TryFrom<&[u8]> for Protocol { + type Error = ProtocolError; + + fn try_from(value: &[u8]) -> Result { + Self::try_from(Bytes::copy_from_slice(value)) + } +} + +impl TryFrom<&str> for Protocol { + type Error = ProtocolError; + + fn try_from(value: &str) -> Result { + // PLUTO PATCH: accept slash-less protocol names (Charon interop), but + // reject an empty name. See third_party/multistream-select/PATCHES.md. + if value.is_empty() { + return Err(ProtocolError::InvalidProtocol); + } + Ok(Protocol(value.to_owned())) + } +} + +impl fmt::Display for Protocol { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.0) + } +} + +/// A multistream-select protocol message. +/// +/// Multistream-select protocol messages are exchanged with the goal +/// of agreeing on a application-layer protocol to use on an I/O stream. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Message { + /// A header message identifies the multistream-select protocol + /// that the sender wishes to speak. + Header(HeaderLine), + /// A protocol message identifies a protocol request or acknowledgement. + Protocol(Protocol), + /// A message through which a peer requests the complete list of + /// supported protocols from the remote. + ListProtocols, + /// A message listing all supported protocols of a peer. + Protocols(Vec), + /// A message signaling that a requested protocol is not available. + NotAvailable, +} + +impl Message { + /// Encodes a `Message` into its byte representation. + fn encode(&self, dest: &mut BytesMut) -> Result<(), ProtocolError> { + match self { + Message::Header(HeaderLine::V1) => { + dest.reserve(MSG_MULTISTREAM_1_0.len()); + dest.put(MSG_MULTISTREAM_1_0); + Ok(()) + } + Message::Protocol(p) => { + let len = p.as_ref().len() + 1; // + 1 for \n + dest.reserve(len); + dest.put(p.0.as_ref()); + dest.put_u8(b'\n'); + Ok(()) + } + Message::ListProtocols => { + dest.reserve(MSG_LS.len()); + dest.put(MSG_LS); + Ok(()) + } + Message::Protocols(ps) => { + let mut buf = uvi::encode::usize_buffer(); + let mut encoded = Vec::with_capacity(ps.len()); + for p in ps { + encoded.extend(uvi::encode::usize(p.as_ref().len() + 1, &mut buf)); // +1 for '\n' + encoded.extend_from_slice(p.0.as_ref()); + encoded.push(b'\n') + } + encoded.push(b'\n'); + dest.reserve(encoded.len()); + dest.put(encoded.as_ref()); + Ok(()) + } + Message::NotAvailable => { + dest.reserve(MSG_PROTOCOL_NA.len()); + dest.put(MSG_PROTOCOL_NA); + Ok(()) + } + } + } + + /// Decodes a `Message` from its byte representation. + fn decode(mut msg: Bytes) -> Result { + if msg == MSG_MULTISTREAM_1_0 { + return Ok(Message::Header(HeaderLine::V1)); + } + + if msg == MSG_PROTOCOL_NA { + return Ok(Message::NotAvailable); + } + + if msg == MSG_LS { + return Ok(Message::ListProtocols); + } + + // A single protocol line is at least one name byte plus a trailing line + // feed, with no other line feeds in between; an `ls` response always has + // more than one line feed, so the empty `Protocols([])` -> "\n" still + // parses as a (zero-entry) list rather than an empty protocol. PLUTO + // PATCH: upstream additionally required a leading `/` here, which would + // misclassify Charon's slash-less single-protocol line as an `ls` list; + // the `msg.len() > 1` guard replaces the role the leading `/` played in + // excluding the bare-"\n" case. See third_party/multistream-select/PATCHES.md. + if msg.len() > 1 && msg.last() == Some(&b'\n') && !msg[..msg.len() - 1].contains(&b'\n') { + let p = Protocol::try_from(msg.split_to(msg.len() - 1))?; + return Ok(Message::Protocol(p)); + } + + // At this point, it must be an `ls` response, i.e. one or more + // length-prefixed, newline-delimited protocol names. + let mut protocols = Vec::new(); + let mut remaining: &[u8] = &msg; + loop { + // A well-formed message must be terminated with a newline. + if remaining == [b'\n'] { + break; + } else if protocols.len() == MAX_PROTOCOLS { + return Err(ProtocolError::TooManyProtocols); + } + + // Decode the length of the next protocol name and check that + // it ends with a line feed. + let (len, tail) = uvi::decode::usize(remaining)?; + if len == 0 || len > tail.len() || tail[len - 1] != b'\n' { + return Err(ProtocolError::InvalidMessage); + } + + // Parse the protocol name. + let p = Protocol::try_from(Bytes::copy_from_slice(&tail[..len - 1]))?; + protocols.push(p); + + // Skip ahead to the next protocol. + remaining = &tail[len..]; + } + + Ok(Message::Protocols(protocols)) + } +} + +/// A `MessageIO` implements a [`Stream`] and [`Sink`] of [`Message`]s. +#[pin_project::pin_project] +pub(crate) struct MessageIO { + #[pin] + inner: LengthDelimited, +} + +impl MessageIO { + /// Constructs a new `MessageIO` resource wrapping the given I/O stream. + pub(crate) fn new(inner: R) -> MessageIO + where + R: AsyncRead + AsyncWrite, + { + Self { + inner: LengthDelimited::new(inner), + } + } + + /// Converts the [`MessageIO`] into a [`MessageReader`], dropping the + /// [`Message`]-oriented `Sink` in favour of direct `AsyncWrite` access + /// to the underlying I/O stream. + /// + /// This is typically done if further negotiation messages are expected to be + /// received but no more messages are written, allowing the writing of + /// follow-up protocol data to commence. + pub(crate) fn into_reader(self) -> MessageReader { + MessageReader { + inner: self.inner.into_reader(), + } + } + + /// Drops the [`MessageIO`] resource, yielding the underlying I/O stream. + /// + /// # Panics + /// + /// Panics if the read buffer or write buffer is not empty, meaning that an incoming + /// protocol negotiation frame has been partially read or an outgoing frame + /// has not yet been flushed. The read buffer is guaranteed to be empty whenever + /// `MessageIO::poll` returned a message. The write buffer is guaranteed to be empty + /// when the sink has been flushed. + pub(crate) fn into_inner(self) -> R { + self.inner.into_inner() + } +} + +impl Sink for MessageIO +where + R: AsyncWrite, +{ + type Error = ProtocolError; + + fn poll_ready(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_ready(cx).map_err(From::from) + } + + fn start_send(self: Pin<&mut Self>, item: Message) -> Result<(), Self::Error> { + let mut buf = BytesMut::new(); + item.encode(&mut buf)?; + self.project() + .inner + .start_send(buf.freeze()) + .map_err(From::from) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_flush(cx).map_err(From::from) + } + + fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_close(cx).map_err(From::from) + } +} + +impl Stream for MessageIO +where + R: AsyncRead, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match poll_stream(self.project().inner, cx) { + Poll::Pending => Poll::Pending, + Poll::Ready(None) => Poll::Ready(None), + Poll::Ready(Some(Ok(m))) => Poll::Ready(Some(Ok(m))), + Poll::Ready(Some(Err(err))) => Poll::Ready(Some(Err(err))), + } + } +} + +/// A `MessageReader` implements a `Stream` of `Message`s on an underlying +/// I/O resource combined with direct `AsyncWrite` access. +#[pin_project::pin_project] +#[derive(Debug)] +pub(crate) struct MessageReader { + #[pin] + inner: LengthDelimitedReader, +} + +impl MessageReader { + /// Drops the `MessageReader` resource, yielding the underlying I/O stream + /// together with the remaining write buffer containing the protocol + /// negotiation frame data that has not yet been written to the I/O stream. + /// + /// # Panics + /// + /// Panics if the read buffer or write buffer is not empty, meaning that either + /// an incoming protocol negotiation frame has been partially read, or an + /// outgoing frame has not yet been flushed. The read buffer is guaranteed to + /// be empty whenever `MessageReader::poll` returned a message. The write + /// buffer is guaranteed to be empty whenever the sink has been flushed. + pub(crate) fn into_inner(self) -> R { + self.inner.into_inner() + } +} + +impl Stream for MessageReader +where + R: AsyncRead, +{ + type Item = Result; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + poll_stream(self.project().inner, cx) + } +} + +impl AsyncWrite for MessageReader +where + TInner: AsyncWrite, +{ + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + self.project().inner.poll_write(cx, buf) + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_flush(cx) + } + + fn poll_close(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.project().inner.poll_close(cx) + } + + fn poll_write_vectored( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + bufs: &[IoSlice<'_>], + ) -> Poll> { + self.project().inner.poll_write_vectored(cx, bufs) + } +} + +fn poll_stream( + stream: Pin<&mut S>, + cx: &mut Context<'_>, +) -> Poll>> +where + S: Stream>, +{ + let msg = if let Some(msg) = ready!(stream.poll_next(cx)?) { + match Message::decode(msg) { + Ok(m) => m, + Err(err) => return Poll::Ready(Some(Err(err))), + } + } else { + return Poll::Ready(None); + }; + + log::trace!("Received message: {:?}", msg); + + Poll::Ready(Some(Ok(msg))) +} + +/// A protocol error. +#[derive(Debug)] +pub enum ProtocolError { + /// I/O error. + IoError(io::Error), + + /// Received an invalid message from the remote. + InvalidMessage, + + /// A protocol (name) is invalid. + InvalidProtocol, + + /// Too many protocols have been returned by the remote. + TooManyProtocols, +} + +impl From for ProtocolError { + fn from(err: io::Error) -> ProtocolError { + ProtocolError::IoError(err) + } +} + +impl From for io::Error { + fn from(err: ProtocolError) -> Self { + if let ProtocolError::IoError(e) = err { + return e; + } + io::ErrorKind::InvalidData.into() + } +} + +impl From for ProtocolError { + fn from(err: uvi::decode::Error) -> ProtocolError { + Self::from(io::Error::new(io::ErrorKind::InvalidData, err.to_string())) + } +} + +impl Error for ProtocolError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match *self { + ProtocolError::IoError(ref err) => Some(err), + _ => None, + } + } +} + +impl fmt::Display for ProtocolError { + fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> Result<(), fmt::Error> { + match self { + ProtocolError::IoError(e) => write!(fmt, "I/O error: {e}"), + ProtocolError::InvalidMessage => write!(fmt, "Received an invalid message."), + ProtocolError::InvalidProtocol => write!(fmt, "A protocol (name) is invalid."), + ProtocolError::TooManyProtocols => write!(fmt, "Too many protocols received."), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // PLUTO: upstream's quickcheck round-trip property test (and its `Arbitrary` + // impls) was removed — it depended on rust-libp2p's unpublished + // `quickcheck-ext`. The deterministic tests below cover the patched + // behaviour, including the empty-`Protocols` round-trip the property test + // would have exercised. + + // PLUTO PATCH tests: cover the relaxed (slash-less) and tightened (empty, + // empty-`ls`) behaviour. See PATCHES.md. + + /// A slash-less single protocol (e.g. Charon's `charon/priority/2.0.0`) + /// encodes and decodes back to the same `Message::Protocol`. + #[test] + fn slashless_single_protocol_round_trips() { + let proto = Protocol::try_from("charon/priority/2.0.0").expect("slash-less accepted"); + let msg = Message::Protocol(proto); + + let mut buf = BytesMut::new(); + msg.encode(&mut buf).expect("encode"); + let decoded = Message::decode(buf.freeze()).expect("decode"); + + assert_eq!(decoded, msg); + match decoded { + Message::Protocol(p) => assert_eq!(p.as_ref(), "charon/priority/2.0.0"), + other => panic!("expected single protocol, got {other:?}"), + } + } + + /// An empty protocol name is rejected on both conversion paths. + #[test] + fn empty_protocol_is_rejected() { + assert!(Protocol::try_from("").is_err()); + assert!(Protocol::try_from(Bytes::new()).is_err()); + assert!(Protocol::try_from(&b""[..]).is_err()); + } + + /// An empty `ls` response (`Protocols([])`, encoded as a bare "\n") still + /// round-trips as a zero-entry list, not as an empty single protocol. + #[test] + fn empty_protocols_list_round_trips() { + let msg = Message::Protocols(Vec::new()); + + let mut buf = BytesMut::new(); + msg.encode(&mut buf).expect("encode"); + let encoded = buf.freeze(); + assert_eq!(encoded.as_ref(), b"\n", "empty Protocols encodes to a bare newline"); + + let decoded = Message::decode(encoded).expect("decode"); + assert_eq!(decoded, Message::Protocols(Vec::new())); + } +} diff --git a/third_party/multistream-select/tests/dialer_select.rs b/third_party/multistream-select/tests/dialer_select.rs new file mode 100644 index 00000000..18f8238c --- /dev/null +++ b/third_party/multistream-select/tests/dialer_select.rs @@ -0,0 +1,202 @@ +// Copyright 2017 Parity Technologies (UK) Ltd. +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. + +//! Integration tests for protocol negotiation. + +use futures::prelude::*; +use multistream_select::{dialer_select_proto, listener_select_proto, NegotiationError, Version}; +use std::time::Duration; + +#[test] +fn select_proto_basic() { + async fn run(version: Version) { + let (client_connection, server_connection) = futures_ringbuf::Endpoint::pair(100, 100); + + let server = async_std::task::spawn(async move { + let protos = vec!["/proto1", "/proto2"]; + let (proto, mut io) = listener_select_proto(server_connection, protos) + .await + .unwrap(); + assert_eq!(proto, "/proto2"); + + let mut out = vec![0; 32]; + let n = io.read(&mut out).await.unwrap(); + out.truncate(n); + assert_eq!(out, b"ping"); + + io.write_all(b"pong").await.unwrap(); + io.flush().await.unwrap(); + }); + + let client = async_std::task::spawn(async move { + let protos = vec!["/proto3", "/proto2"]; + let (proto, mut io) = + dialer_select_proto(client_connection, protos.into_iter(), version) + .await + .unwrap(); + assert_eq!(proto, "/proto2"); + + io.write_all(b"ping").await.unwrap(); + io.flush().await.unwrap(); + + let mut out = vec![0; 32]; + let n = io.read(&mut out).await.unwrap(); + out.truncate(n); + assert_eq!(out, b"pong"); + }); + + server.await; + client.await; + } + + async_std::task::block_on(run(Version::V1)); + async_std::task::block_on(run(Version::V1Lazy)); +} + +/// Tests the expected behaviour of failed negotiations. +#[test] +fn negotiation_failed() { + let _ = env_logger::try_init(); + + async fn run( + Test { + version, + listen_protos, + dial_protos, + dial_payload, + }: Test, + ) { + let (client_connection, server_connection) = futures_ringbuf::Endpoint::pair(100, 100); + + let server = async_std::task::spawn(async move { + let io = match listener_select_proto(server_connection, listen_protos).await { + Ok((_, io)) => io, + Err(NegotiationError::Failed) => return, + Err(NegotiationError::ProtocolError(e)) => { + panic!("Unexpected protocol error {e}") + } + }; + match io.complete().await { + Err(NegotiationError::Failed) => {} + _ => panic!(), + } + }); + + let client = + async_std::task::spawn(async move { + let mut io = + match dialer_select_proto(client_connection, dial_protos.into_iter(), version) + .await + { + Err(NegotiationError::Failed) => return, + Ok((_, io)) => io, + Err(_) => panic!(), + }; + // The dialer may write a payload that is even sent before it + // got confirmation of the last proposed protocol, when `V1Lazy` + // is used. + io.write_all(&dial_payload).await.unwrap(); + match io.complete().await { + Err(NegotiationError::Failed) => {} + _ => panic!(), + } + }); + + server.await; + client.await; + } + + /// Parameters for a single test run. + #[derive(Clone)] + struct Test { + version: Version, + listen_protos: Vec<&'static str>, + dial_protos: Vec<&'static str>, + dial_payload: Vec, + } + + // Disjunct combinations of listen and dial protocols to test. + // + // The choices here cover the main distinction between a single + // and multiple protocols. + let protos = vec![ + (vec!["/proto1"], vec!["/proto2"]), + (vec!["/proto1", "/proto2"], vec!["/proto3", "/proto4"]), + ]; + + // The payloads that the dialer sends after "successful" negotiation, + // which may be sent even before the dialer got protocol confirmation + // when `V1Lazy` is used. + // + // The choices here cover the specific situations that can arise with + // `V1Lazy` and which must nevertheless behave identically to `V1` w.r.t. + // the outcome of the negotiation. + let payloads = vec![ + // No payload, in which case all versions should behave identically + // in any case, i.e. the baseline test. + vec![], + // With this payload and `V1Lazy`, the listener interprets the first + // `1` as a message length and encounters an invalid message (the + // second `1`). The listener is nevertheless expected to fail + // negotiation normally, just like with `V1`. + vec![1, 1], + // With this payload and `V1Lazy`, the listener interprets the first + // `42` as a message length and encounters unexpected EOF trying to + // read a message of that length. The listener is nevertheless expected + // to fail negotiation normally, just like with `V1` + vec![42, 1], + ]; + + for (listen_protos, dial_protos) in protos { + for dial_payload in payloads.clone() { + for &version in &[Version::V1, Version::V1Lazy] { + async_std::task::block_on(run(Test { + version, + listen_protos: listen_protos.clone(), + dial_protos: dial_protos.clone(), + dial_payload: dial_payload.clone(), + })) + } + } + } +} + +#[async_std::test] +async fn v1_lazy_do_not_wait_for_negotiation_on_poll_close() { + let (client_connection, _server_connection) = futures_ringbuf::Endpoint::pair(1024 * 1024, 1); + + let client = async_std::task::spawn(async move { + // Single protocol to allow for lazy (or optimistic) protocol negotiation. + let protos = vec!["/proto1"]; + let (proto, mut io) = + dialer_select_proto(client_connection, protos.into_iter(), Version::V1Lazy) + .await + .unwrap(); + assert_eq!(proto, "/proto1"); + + // client can close the connection even though protocol negotiation is not yet done, i.e. + // `_server_connection` had been untouched. + io.close().await.unwrap(); + }); + + async_std::future::timeout(Duration::from_secs(10), client) + .await + .unwrap(); +} From 587e53d0d4d66fb0094e7228969349a86d1d988f Mon Sep 17 00:00:00 2001 From: Quang Le Date: Fri, 19 Jun 2026 08:47:21 +0700 Subject: [PATCH 12/12] fix: priority test --- crates/priority/src/prioritiser.rs | 10 ++++++++- crates/priority/tests/prioritiser_test.rs | 27 +++++++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/crates/priority/src/prioritiser.rs b/crates/priority/src/prioritiser.rs index 45d7ecbe..2dd80c10 100644 --- a/crates/priority/src/prioritiser.rs +++ b/crates/priority/src/prioritiser.rs @@ -334,7 +334,15 @@ impl Prioritiser { match self.inner.deadliner.add(duty.clone()).await { AddOutcome::Scheduled => {} - AddOutcome::FailedToCompute => return Err(Error::DeadlineComputeFailed), + AddOutcome::FailedToCompute => { + // The deadliner shares the engine/instance cancellation token, so + // a failure while shutting down is a cancellation, not a genuine + // compute error — report it like the run-loop cancel path. + if ctx.is_cancelled() || self.inner.quit.is_cancelled() { + return Err(Error::Cancelled); + } + return Err(Error::DeadlineComputeFailed); + } AddOutcome::AlreadyExpired | AddOutcome::NoDeadline => { tracing::warn!(%duty, "Dropping priority protocol instance for expired duty"); return Ok(()); diff --git a/crates/priority/tests/prioritiser_test.rs b/crates/priority/tests/prioritiser_test.rs index 9587ffac..5ed5fdea 100644 --- a/crates/priority/tests/prioritiser_test.rs +++ b/crates/priority/tests/prioritiser_test.rs @@ -7,12 +7,12 @@ //! (proposed by all three) with score `n*1000`. use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, sync::{Arc, Mutex}, time::Duration, }; -use futures::StreamExt as _; +use futures::{FutureExt as _, StreamExt as _, future::select_all}; use libp2p::{ Multiaddr, PeerId, Swarm, core::{Transport as _, transport::MemoryTransport, upgrade::Version}, @@ -286,6 +286,29 @@ async fn three_host_prioritiser() { } } + // Wait until every host is connected to all its peers before exchanging. + // The priority exchange opens substreams on existing connections; launching + // it before the mesh is up would race connection setup and could drop an + // exchange, stalling a duty's consensus. + { + let mut connected: Vec> = vec![HashSet::new(); N]; + let mesh = async { + while connected.iter().any(|peers| peers.len() < N - 1) { + let next = hosts + .iter_mut() + .map(|h| h.swarm.select_next_some().boxed()) + .collect::>(); + let (event, idx, _) = select_all(next).await; + if let SwarmEvent::ConnectionEstablished { peer_id, .. } = event { + connected[idx].insert(peer_id); + } + } + }; + timeout(Duration::from_secs(30), mesh) + .await + .expect("full connection mesh within timeout"); + } + // Extract per-host prioritisers (with their key/peer id) and drive each // swarm in the background. Host `i` proposes priorities `0..=i`. let mut launchers = Vec::with_capacity(N);