From 28150a39e0caf4ac6e6e30017cd9fdf8f9f13d88 Mon Sep 17 00:00:00 2001 From: Chris O'Neil Date: Tue, 16 Jun 2026 18:20:10 +0100 Subject: [PATCH] fix(payment): widen receiver storage admission to uploader over-query window MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Direct client PUT receiver admission (`validate_store_membership`) rejected honest uploads on NAT-simulated testnets: the storer decided it was "not among this node's local 9 closest peers" for a chunk it is XOR-close to. Root cause: on a network with unreachable (NAT'd, relay-only) peers the client cannot quote the genuinely-closest nodes, so it legitimately pays a peer that ranks just past `close_group_size` in a storer's full local routing-table view (which still contains those closer, client-unreachable peers). The admission window was `close_group_size + STORAGE_ADMISSION_MARGIN` (= 9), narrower than the divergence, so the receiver rejected the paid PUT. On the single-node payment path — which fails the whole file on one un-stored chunk — this produced multiplicative per-file upload failures. v0.12.0-rc.1 (which had no admission gate) achieved 100% uploads on the same fault-injection environment; the gate was introduced in #136 and is the regression. Fix: size the storage-admission window to the uploader's quote over-query window (`close_group_size * STORAGE_ADMISSION_OVERQUERY_MULTIPLIER`, mirroring ant-client `CLOSE_GROUP_SIZE * FAULT_TOLERANT_QUOTE_QUERY_MULTIPLIER`), so a storer admits a chunk it is XOR-close to within the same window the uploader selected from. The check stays XOR-only (#140). Admission and stored-record retention stay coupled (a node that accepts a chunk also retains it); audit, quorum, and paid-list target sets are unchanged at `close_group_size`. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/replication/admission.rs | 5 ++-- src/replication/config.rs | 45 +++++++++++++++++++++++++++++------- src/replication/pruning.rs | 3 ++- src/storage/handler.rs | 13 +++++++++++ 4 files changed, 55 insertions(+), 11 deletions(-) diff --git a/src/replication/admission.rs b/src/replication/admission.rs index 3b99a802..b7536998 100644 --- a/src/replication/admission.rs +++ b/src/replication/admission.rs @@ -72,8 +72,9 @@ pub async fn is_in_paid_close_group( /// - **Cross-set precedence**: if a key appears in both sets, keep only the /// replica-hint entry. /// - **Replica hints**: admitted if `self` is in the storage-admission group -/// (`close_group_size + STORAGE_ADMISSION_MARGIN`) or key already exists in -/// local store / pending set. +/// (`storage_admission_width` = `close_group_size * +/// STORAGE_ADMISSION_OVERQUERY_MULTIPLIER`) or key already exists in local +/// store / pending set. /// - **Paid hints**: admitted if `self` is in `PaidCloseGroup(K)` or key is /// already in `PaidForList`. /// diff --git a/src/replication/config.rs b/src/replication/config.rs index b3cd9441..45d1eb95 100644 --- a/src/replication/config.rs +++ b/src/replication/config.rs @@ -23,13 +23,26 @@ use crate::ant_protocol::CLOSE_GROUP_SIZE; /// Maximum number of peers per k-bucket in the Kademlia routing table. pub const K_BUCKET_SIZE: usize = 20; -/// Extra local-routing-table positions accepted for local chunk storage -/// admission and stored-record pruning. +/// Multiplier applied to `close_group_size` to size the local chunk +/// storage-admission / stored-record-retention window. /// -/// This margin absorbs small local RT disagreement between peers. It does not -/// widen audit, quorum, or paid-list target sets; those remain strict +/// Mirrors the uploader's quote over-query window +/// (`CLOSE_GROUP_SIZE * FAULT_TOLERANT_QUOTE_QUERY_MULTIPLIER` in the ant-client +/// `get_store_quotes` path): the client picks a chunk's close group from its +/// `close_group_size * N` XOR-closest *reachable* peers. On a network with +/// unreachable (NAT'd, relay-only) nodes, the peer it legitimately pays can sit +/// several positions beyond `close_group_size` in a storer's *full* local +/// routing-table view, because that view also contains the closer peers the +/// client could not reach. A `close_group_size + small-margin` window rejects +/// those honest payments (the receiver decides it is "not responsible" for a +/// chunk it is genuinely XOR-close to); sizing admission to the same over-query +/// width the uploader selected from removes that false rejection while keeping +/// the check XOR-only (see `validate_store_membership`). +/// +/// This widens only local storage admission and stored-record retention. It +/// does NOT widen audit, quorum, or paid-list target sets; those remain strict /// `close_group_size` / paid-list group checks. -pub const STORAGE_ADMISSION_MARGIN: usize = 2; +pub const STORAGE_ADMISSION_OVERQUERY_MULTIPLIER: usize = 2; /// Full-network target for required positive presence votes. /// @@ -49,9 +62,15 @@ pub const NEIGHBOR_SYNC_PEER_COUNT: usize = 4; /// Width used when deciding whether this node may locally store or retain a /// chunk. +/// +/// Sized to the uploader's quote over-query window +/// (`close_group_size * STORAGE_ADMISSION_OVERQUERY_MULTIPLIER`) so a storer +/// does not reject an honestly-paid chunk it is XOR-close to merely because the +/// uploader, on a NAT'd network, had to pay a peer that ranks just outside the +/// strict close group in the storer's full local view. #[must_use] pub const fn storage_admission_width(close_group_size: usize) -> usize { - close_group_size.saturating_add(STORAGE_ADMISSION_MARGIN) + close_group_size.saturating_mul(STORAGE_ADMISSION_OVERQUERY_MULTIPLIER) } /// Minimum neighbor-sync cadence. Actual interval is randomized within @@ -433,13 +452,23 @@ mod tests { } #[test] - fn storage_admission_width_adds_margin() { + fn storage_admission_width_is_overquery_window() { const TEST_CLOSE_GROUP_SIZE: usize = 7; + // Admission mirrors the uploader's quote over-query window + // (`CLOSE_GROUP_SIZE * FAULT_TOLERANT_QUOTE_QUERY_MULTIPLIER`, where the + // ant-client multiplier is 2), so an honestly-paid receiver ranking just + // outside the strict close group is still admitted. assert_eq!( storage_admission_width(TEST_CLOSE_GROUP_SIZE), - TEST_CLOSE_GROUP_SIZE + STORAGE_ADMISSION_MARGIN + TEST_CLOSE_GROUP_SIZE * STORAGE_ADMISSION_OVERQUERY_MULTIPLIER ); + // Concretely: the production close group of 7 admits the 14 closest. + assert_eq!(storage_admission_width(7), 14); + // The widened window must still be strictly wider than the strict close + // group (otherwise the recalibration is a no-op). + assert!(storage_admission_width(TEST_CLOSE_GROUP_SIZE) > TEST_CLOSE_GROUP_SIZE); + // Saturates rather than overflowing on a degenerate config. assert_eq!(storage_admission_width(usize::MAX), usize::MAX); } diff --git a/src/replication/pruning.rs b/src/replication/pruning.rs index 68ebb8cc..2fe39761 100644 --- a/src/replication/pruning.rs +++ b/src/replication/pruning.rs @@ -196,7 +196,8 @@ struct PruneAuditReportState { /// /// For each stored record K: /// - If `self` is within the storage-admission group -/// (`close_group_size + STORAGE_ADMISSION_MARGIN`): clear +/// (`storage_admission_width` = `close_group_size * +/// STORAGE_ADMISSION_OVERQUERY_MULTIPLIER`): clear /// `RecordOutOfRangeFirstSeen`. /// - If not in that group: set timestamp if not already set; delete if the /// timestamp is at least `PRUNE_HYSTERESIS_DURATION` old and all but one diff --git a/src/storage/handler.rs b/src/storage/handler.rs index dc7c690b..95c27842 100644 --- a/src/storage/handler.rs +++ b/src/storage/handler.rs @@ -412,6 +412,19 @@ impl AntProtocol { }; let self_id = *p2p_node.peer_id(); + // `admission_width` is the uploader's quote over-query window + // (`close_group_size * STORAGE_ADMISSION_OVERQUERY_MULTIPLIER`), not the + // strict close group. On a network with unreachable (NAT'd, relay-only) + // peers the client cannot quote the genuinely-closest nodes, so the peer + // it legitimately pays can rank several positions beyond + // `close_group_size` in this node's *full* local routing-table view + // (which still contains those unreachable closer peers). Verifying + // against the same over-query width the uploader selected from prevents + // this node from rejecting an honest PUT for a chunk it is XOR-close to; + // narrower windows produced multiplicative per-file upload failures on + // NAT-simulated testnets. Steady-state responsibility stays strict — the + // record is pruned back to the close group once replication heals it + // (see `replication::pruning`). let admission_width = storage_admission_width(self.payment_verifier.close_group_size()); // Storage-responsibility *verification* must mirror the uploader's pure // XOR-distance peer selection. `find_closest_nodes_local_with_self`