From 5078eb365af765ead766ded14475f332db838fee Mon Sep 17 00:00:00 2001 From: danielntmd <162406516+danielntmd@users.noreply.github.com> Date: Wed, 11 Feb 2026 03:57:38 -0500 Subject: [PATCH 01/27] chore: Should fix proving benchmarks by reducing committee lag (#20381) This should fix the timeouts on the test. We reduce the lag by 1 epoch or about ~40 minutes and bump up the timeout by 30 minutes. Since we run the test for a full epoch, we add the buffer in case of worst case hitting epoch boundary (which is usually hit due to wallet setup). - includes updates to .env files that reference the old lag variable - bump up benchmark timeout by 30 minutes Co-authored-by: danielntmd --- .github/workflows/nightly-spartan-bench.yml | 4 ++-- spartan/environments/five-tps-long-epoch.env | 3 ++- spartan/environments/five-tps-short-epoch.env | 3 ++- spartan/environments/prove-n-tps-fake.env | 3 ++- spartan/environments/prove-n-tps-real.env | 3 ++- spartan/environments/ten-tps-long-epoch.env | 3 ++- spartan/environments/ten-tps-short-epoch.env | 3 ++- 7 files changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nightly-spartan-bench.yml b/.github/workflows/nightly-spartan-bench.yml index c5501376da29..e331be8f8ec5 100644 --- a/.github/workflows/nightly-spartan-bench.yml +++ b/.github/workflows/nightly-spartan-bench.yml @@ -152,7 +152,7 @@ jobs: fi - name: Run proving benchmarks - timeout-minutes: 150 + timeout-minutes: 180 env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} @@ -162,7 +162,7 @@ jobs: GCP_PROJECT_ID: ${{ secrets.GCP_PROJECT_ID }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} RUN_ID: ${{ github.run_id }} - AWS_SHUTDOWN_TIME: 150 + AWS_SHUTDOWN_TIME: 180 NO_SPOT: 1 run: | ./.github/ci3.sh network-proving-bench prove-n-tps-fake prove-n-tps-fake "aztecprotocol/aztec:${{ steps.nightly-tag.outputs.nightly_tag }}" diff --git a/spartan/environments/five-tps-long-epoch.env b/spartan/environments/five-tps-long-epoch.env index d72481da0000..84bd8ee6e591 100644 --- a/spartan/environments/five-tps-long-epoch.env +++ b/spartan/environments/five-tps-long-epoch.env @@ -19,7 +19,8 @@ REDEPLOY_ROLLUP_CONTRACTS=true VERIFY_CONTRACTS=false DESTROY_AZTEC_INFRA=true -AZTEC_LAG_IN_EPOCHS=1 +AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1 +AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1 OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET diff --git a/spartan/environments/five-tps-short-epoch.env b/spartan/environments/five-tps-short-epoch.env index 7051151d52ef..56141ee724c4 100644 --- a/spartan/environments/five-tps-short-epoch.env +++ b/spartan/environments/five-tps-short-epoch.env @@ -19,7 +19,8 @@ REDEPLOY_ROLLUP_CONTRACTS=true VERIFY_CONTRACTS=false DESTROY_AZTEC_INFRA=true -AZTEC_LAG_IN_EPOCHS=1 +AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1 +AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1 OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET diff --git a/spartan/environments/prove-n-tps-fake.env b/spartan/environments/prove-n-tps-fake.env index 7b228b2db601..bed763852285 100644 --- a/spartan/environments/prove-n-tps-fake.env +++ b/spartan/environments/prove-n-tps-fake.env @@ -5,7 +5,8 @@ GCP_REGION=us-west1-a AZTEC_EPOCH_DURATION=32 AZTEC_SLOT_DURATION=72 AZTEC_PROOF_SUBMISSION_EPOCHS=1 -AZTEC_LAG_IN_EPOCHS=1 +AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1 +AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1 CREATE_ETH_DEVNET=true DESTROY_NAMESPACE=true diff --git a/spartan/environments/prove-n-tps-real.env b/spartan/environments/prove-n-tps-real.env index 407409f9cd34..9b8989671922 100644 --- a/spartan/environments/prove-n-tps-real.env +++ b/spartan/environments/prove-n-tps-real.env @@ -5,7 +5,8 @@ GCP_REGION=us-west1-a AZTEC_EPOCH_DURATION=32 AZTEC_SLOT_DURATION=72 AZTEC_PROOF_SUBMISSION_EPOCHS=1 -AZTEC_LAG_IN_EPOCHS=1 +AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1 +AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1 CREATE_ETH_DEVNET=true DESTROY_NAMESPACE=true diff --git a/spartan/environments/ten-tps-long-epoch.env b/spartan/environments/ten-tps-long-epoch.env index aba4e34ce816..39ea3d75e197 100644 --- a/spartan/environments/ten-tps-long-epoch.env +++ b/spartan/environments/ten-tps-long-epoch.env @@ -19,7 +19,8 @@ REDEPLOY_ROLLUP_CONTRACTS=true VERIFY_CONTRACTS=false DESTROY_AZTEC_INFRA=true -AZTEC_LAG_IN_EPOCHS=1 +AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1 +AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1 OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET diff --git a/spartan/environments/ten-tps-short-epoch.env b/spartan/environments/ten-tps-short-epoch.env index bef68af7972e..35868695e0f6 100644 --- a/spartan/environments/ten-tps-short-epoch.env +++ b/spartan/environments/ten-tps-short-epoch.env @@ -19,7 +19,8 @@ REDEPLOY_ROLLUP_CONTRACTS=true VERIFY_CONTRACTS=false DESTROY_AZTEC_INFRA=true -AZTEC_LAG_IN_EPOCHS=1 +AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1 +AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1 OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET From f993d795f6f7de5d725e5ee39e94cee32b2c7491 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Tue, 10 Feb 2026 23:42:12 +0000 Subject: [PATCH 02/27] Initial commit --- yarn-project/foundation/src/config/env_var.ts | 1 + yarn-project/p2p/src/client/factory.ts | 1 + yarn-project/p2p/src/client/interface.ts | 3 +++ yarn-project/p2p/src/client/p2p_client.ts | 10 ++++++++ yarn-project/p2p/src/config.ts | 8 +++++++ .../eviction/eviction_manager.test.ts | 2 ++ .../fee_payer_balance_eviction_rule.test.ts | 1 + .../fee_payer_balance_pre_add_rule.test.ts | 1 + .../invalid_txs_after_mining_rule.test.ts | 1 + .../invalid_txs_after_reorg_rule.test.ts | 1 + .../low_priority_pre_add_rule.test.ts | 1 + .../eviction/nullifier_conflict_rule.test.ts | 1 + .../src/mem_pools/tx_pool_v2/interfaces.ts | 3 +++ .../mem_pools/tx_pool_v2/tx_metadata.test.ts | 2 ++ .../src/mem_pools/tx_pool_v2/tx_metadata.ts | 4 ++++ .../mem_pools/tx_pool_v2/tx_pool_indices.ts | 23 +++++++++++++++++++ .../src/mem_pools/tx_pool_v2/tx_pool_v2.ts | 8 ++++++- .../mem_pools/tx_pool_v2/tx_pool_v2_impl.ts | 11 +++++++++ .../p2p/src/test-helpers/testbench-utils.ts | 4 ++++ .../sequencer/checkpoint_proposal_job.test.ts | 16 ++++++------- .../checkpoint_proposal_job.timing.test.ts | 2 +- .../src/sequencer/checkpoint_proposal_job.ts | 2 +- .../sequencer-client/src/test/utils.ts | 1 + .../txe/src/state_machine/dummy_p2p_client.ts | 4 ++++ 24 files changed, 100 insertions(+), 11 deletions(-) diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index e5cc85f214de..2c1e45906d48 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -144,6 +144,7 @@ export type EnvVar = | 'P2P_DROP_TX' | 'P2P_DROP_TX_CHANCE' | 'P2P_TX_POOL_DELETE_TXS_AFTER_REORG' + | 'P2P_MIN_TX_POOL_AGE_MS' | 'DEBUG_P2P_INSTRUMENT_MESSAGES' | 'PEER_ID_PRIVATE_KEY' | 'PEER_ID_PRIVATE_KEY_PATH' diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index 9cfb36eaa5ce..4da8ddda7796 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -117,6 +117,7 @@ export async function createP2PClient( maxPendingTxCount: config.maxPendingTxCount, archivedTxLimit: config.archivedTxLimit, }, + dateProvider, ); const mempools: MemPools = { diff --git a/yarn-project/p2p/src/client/interface.ts b/yarn-project/p2p/src/client/interface.ts index 350fc90c1ffb..73d0d870e887 100644 --- a/yarn-project/p2p/src/client/interface.ts +++ b/yarn-project/p2p/src/client/interface.ts @@ -166,6 +166,9 @@ export type P2P = P2PApiFull & /** Returns an iterator over pending txs on the mempool. */ iteratePendingTxs(): AsyncIterableIterator; + /** Returns an iterator over pending txs that have been in the pool long enough to be eligible for block building. */ + iterateEligiblePendingTxs(): AsyncIterableIterator; + /** Returns the number of pending txs in the mempool. */ getPendingTxCount(): Promise; diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index 5ef8c4db63ed..4e0a69acdff2 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -488,6 +488,16 @@ export class P2PClient } } + public async *iterateEligiblePendingTxs(): AsyncIterableIterator { + const maxReceivedAt = this._dateProvider.now() - this.config.minTxPoolAgeMs; + for (const txHash of await this.txPool.getEligiblePendingTxHashes(maxReceivedAt)) { + const tx = await this.txPool.getTxByHash(txHash); + if (tx) { + yield tx; + } + } + } + /** * Returns a transaction in the transaction pool by its hash. * @param txHash - Hash of the transaction to look for in the pool. diff --git a/yarn-project/p2p/src/config.ts b/yarn-project/p2p/src/config.ts index 4443a9eab9e3..ad47120b786b 100644 --- a/yarn-project/p2p/src/config.ts +++ b/yarn-project/p2p/src/config.ts @@ -190,6 +190,9 @@ export interface P2PConfig /** Broadcast block proposals even when a conflicting proposal for the same slot already exists in the pool (for testing purposes only). */ broadcastEquivocatedProposals?: boolean; + + /** Minimum age (ms) a transaction must have been in the pool before it's eligible for block building. */ + minTxPoolAgeMs: number; } export const DEFAULT_P2P_PORT = 40400; @@ -464,6 +467,11 @@ export const p2pConfigMappings: ConfigMappingsType = { 'Broadcast block proposals even when a conflicting proposal for the same slot already exists in the pool (for testing purposes only).', ...booleanConfigHelper(false), }, + minTxPoolAgeMs: { + env: 'P2P_MIN_TX_POOL_AGE_MS', + description: 'Minimum age (ms) a transaction must have been in the pool before it is eligible for block building.', + ...numberConfigHelper(2_000), + }, ...sharedSequencerConfigMappings, ...p2pReqRespConfigMappings, ...batchTxRequesterConfigMappings, diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.test.ts index 7e44fb8364ca..bb134e70df2a 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/eviction_manager.test.ts @@ -182,6 +182,7 @@ describe('EvictionManager', () => { feeLimit: 100n, nullifiers: [`0x${txHash.slice(2)}null1`], includeByTimestamp: 0n, + receivedAt: 0, data: stubTxMetaValidationData(), }); @@ -316,6 +317,7 @@ describe('EvictionManager', () => { feeLimit: 100n, nullifiers: [`0x${txHash.slice(2)}null1`], includeByTimestamp: 0n, + receivedAt: 0, data: stubTxMetaValidationData(), }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.test.ts index 64b514b04681..de7c1d2bf76f 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.test.ts @@ -42,6 +42,7 @@ describe('FeePayerBalanceEvictionRule', () => { feeLimit: opts.feeLimit ?? 100n, nullifiers: [`0x${txHash.slice(2)}null1`], includeByTimestamp: 0n, + receivedAt: 0, data: stubTxMetaValidationData(), }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.test.ts index e839c7adff0b..2a9ca2ea552e 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_pre_add_rule.test.ts @@ -23,6 +23,7 @@ describe('FeePayerBalancePreAddRule', () => { feeLimit: opts.feeLimit ?? 100n, nullifiers: [`0x${txHash.slice(2)}null1`], includeByTimestamp: 0n, + receivedAt: 0, data: stubTxMetaValidationData(), }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.test.ts index 5a822681a09d..51fa1fd3b3cb 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.test.ts @@ -36,6 +36,7 @@ describe('InvalidTxsAfterMiningRule', () => { feeLimit: 100n, nullifiers, includeByTimestamp, + receivedAt: 0, data: stubTxMetaValidationData({ includeByTimestamp }), }; }; diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.test.ts index 8ce5108cf7b6..b07974d893b0 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.test.ts @@ -30,6 +30,7 @@ describe('InvalidTxsAfterReorgRule', () => { feeLimit: 100n, nullifiers: [`0x${txHash.slice(2)}null1`], includeByTimestamp: 0n, + receivedAt: 0, data: stubTxMetaValidationData(), }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.test.ts index cc3f655b9b0f..c4ff83c31aab 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_pre_add_rule.test.ts @@ -15,6 +15,7 @@ describe('LowPriorityPreAddRule', () => { feeLimit: 100n, nullifiers: [`0x${txHash.slice(2)}null1`], includeByTimestamp: 0n, + receivedAt: 0, data: stubTxMetaValidationData(), }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.test.ts index 10ee39758573..507f2718c678 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/nullifier_conflict_rule.test.ts @@ -20,6 +20,7 @@ describe('NullifierConflictRule', () => { feeLimit: 1000n, nullifiers, includeByTimestamp: 0n, + receivedAt: 0, data: stubTxMetaValidationData(), }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts index 15db226efaa8..5f7d0fcaa058 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts @@ -187,6 +187,9 @@ export interface TxPoolV2 extends TypedEventEmitter { /** Gets pending transaction hashes sorted by priority (highest first) */ getPendingTxHashes(): Promise; + /** Gets pending transaction hashes that have been in the pool since before maxReceivedAt, sorted by priority (highest first) */ + getEligiblePendingTxHashes(maxReceivedAt: number): Promise; + /** Gets the count of pending transactions */ getPendingTxCount(): Promise; diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.test.ts index efe38ea6b786..287883580a1b 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.test.ts @@ -47,6 +47,7 @@ describe('TxMetaData', () => { feeLimit: 1000n, nullifiers: [], includeByTimestamp: 0n, + receivedAt: 0, data: stubTxMetaValidationData(), }); @@ -79,6 +80,7 @@ describe('TxMetaData', () => { feeLimit: 1000n, nullifiers, includeByTimestamp: 0n, + receivedAt: 0, data: stubTxMetaValidationData(), }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.ts index 96686068c8c9..529484c84b6f 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_metadata.ts @@ -60,6 +60,9 @@ export type TxMetaData = { /** Validator-compatible data, providing the same access patterns as Tx.data */ readonly data: TxMetaValidationData; + + /** Timestamp (ms) when the tx was received into the pool. 0 for hydrated txs (always eligible). */ + receivedAt: number; }; /** Transaction state derived from TxMetaData fields and pool protection status */ @@ -92,6 +95,7 @@ export async function buildTxMetaData(tx: Tx): Promise { feeLimit, nullifiers, includeByTimestamp, + receivedAt: 0, data: { getNonEmptyNullifiers: () => nullifierFrs, includeByTimestamp, diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts index f664123872ac..45bcc5b17774 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts @@ -89,6 +89,29 @@ export class TxPoolIndices { } } + /** + * Iterates pending transaction hashes in priority order, skipping txs received after maxReceivedAt. + * @param order - 'desc' for highest priority first, 'asc' for lowest priority first + * @param maxReceivedAt - Only yield txs with receivedAt <= this value + */ + *iterateEligiblePendingByPriority(order: 'asc' | 'desc', maxReceivedAt: number): Generator { + const feeCompareFn = order === 'desc' ? (a: bigint, b: bigint) => compareFee(b, a) : compareFee; + const hashCompareFn = order === 'desc' ? (a: string, b: string) => compareTxHash(b, a) : compareTxHash; + + const sortedFees = [...this.#pendingByPriority.keys()].sort(feeCompareFn); + + for (const fee of sortedFees) { + const hashesAtFee = this.#pendingByPriority.get(fee)!; + const sortedHashes = [...hashesAtFee].sort(hashCompareFn); + for (const hash of sortedHashes) { + const meta = this.#metadata.get(hash); + if (meta && meta.receivedAt <= maxReceivedAt) { + yield hash; + } + } + } + } + /** Iterates all metadata entries */ *iterateMetadata(): Generator<[string, TxMetaData]> { yield* this.#metadata; diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts index bee685d64f69..692450a34afd 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts @@ -1,6 +1,7 @@ import { SlotNumber } from '@aztec/foundation/branded-types'; import { type Logger, createLogger } from '@aztec/foundation/log'; import { SerialQueue } from '@aztec/foundation/queue'; +import { DateProvider } from '@aztec/foundation/timer'; import type { TypedEventEmitter } from '@aztec/foundation/types'; import type { AztecAsyncKVStore } from '@aztec/kv-store'; import type { L2Block, L2BlockId } from '@aztec/stdlib/block'; @@ -35,6 +36,7 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte deps: TxPoolV2Dependencies, telemetry: TelemetryClient = getTelemetryClient(), config: Partial = {}, + dateProvider: DateProvider = new DateProvider(), log = createLogger('p2p:tx_pool_v2'), ) { super(); @@ -59,7 +61,7 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte }; // Create the implementation - this.#impl = new TxPoolV2Impl(store, archiveStore, deps, callbacks, config, log); + this.#impl = new TxPoolV2Impl(store, archiveStore, deps, callbacks, config, dateProvider, log); } // ============================================================================ @@ -132,6 +134,10 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte return this.#queue.put(() => Promise.resolve(this.#impl.getPendingTxHashes())); } + getEligiblePendingTxHashes(maxReceivedAt: number): Promise { + return this.#queue.put(() => Promise.resolve(this.#impl.getEligiblePendingTxHashes(maxReceivedAt))); + } + getPendingTxCount(): Promise { return this.#queue.put(() => Promise.resolve(this.#impl.getPendingTxCount())); } diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts index 54bfb3b7ec0c..543b944c6781 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts @@ -1,5 +1,6 @@ import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types'; import type { Logger } from '@aztec/foundation/log'; +import type { DateProvider } from '@aztec/foundation/timer'; import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store'; import { ProtocolContractAddress } from '@aztec/protocol-contracts'; import { computeFeePayerBalanceStorageSlot } from '@aztec/protocol-contracts/fee-juice'; @@ -64,6 +65,7 @@ export class TxPoolV2Impl { #archive: TxArchive; #deletedPool: DeletedPool; #evictionManager: EvictionManager; + #dateProvider: DateProvider; #log: Logger; #callbacks: TxPoolV2Callbacks; @@ -73,6 +75,7 @@ export class TxPoolV2Impl { deps: TxPoolV2Dependencies, callbacks: TxPoolV2Callbacks, config: Partial = {}, + dateProvider: DateProvider, log: Logger, ) { this.#store = store; @@ -85,6 +88,7 @@ export class TxPoolV2Impl { this.#config = { ...DEFAULT_TX_POOL_V2_CONFIG, ...config }; this.#archive = new TxArchive(archiveStore, this.#config.archivedTxLimit, log); this.#deletedPool = new DeletedPool(store, this.#txsDB, log); + this.#dateProvider = dateProvider; this.#log = log; this.#callbacks = callbacks; @@ -549,6 +553,12 @@ export class TxPoolV2Impl { return [...this.#indices.iteratePendingByPriority('desc')].map(hash => TxHash.fromString(hash)); } + getEligiblePendingTxHashes(maxReceivedAt: number): TxHash[] { + return [...this.#indices.iterateEligiblePendingByPriority('desc', maxReceivedAt)].map(hash => + TxHash.fromString(hash), + ); + } + getPendingTxCount(): number { return this.#indices.getPendingTxCount(); } @@ -629,6 +639,7 @@ export class TxPoolV2Impl { ): Promise { const txHashStr = tx.getTxHash().toString(); const meta = await buildTxMetaData(tx); + meta.receivedAt = this.#dateProvider.now(); await this.#txsDB.set(txHashStr, tx.toBuffer()); this.#callbacks.onTxsAdded([tx], opts); diff --git a/yarn-project/p2p/src/test-helpers/testbench-utils.ts b/yarn-project/p2p/src/test-helpers/testbench-utils.ts index 2afb25e9f37f..58819792a5c0 100644 --- a/yarn-project/p2p/src/test-helpers/testbench-utils.ts +++ b/yarn-project/p2p/src/test-helpers/testbench-utils.ts @@ -163,6 +163,10 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 { return Promise.resolve([...this.txsByHash.keys()].map(key => TxHash.fromString(key))); } + getEligiblePendingTxHashes(_maxReceivedAt: number): Promise { + return this.getPendingTxHashes(); + } + getPendingTxCount(): Promise { return Promise.resolve(this.txsByHash.size); } diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts index a9d5ad82750c..58b0d8ca3db3 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts @@ -462,7 +462,7 @@ describe('CheckpointProposalJob', () => { // Set up p2p mocks p2p.getPendingTxCount.mockResolvedValue(txs.length); - p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); + p2p.iterateEligiblePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); // Create blocks with incrementing block numbers const blocks: Awaited>[] = []; @@ -685,7 +685,7 @@ describe('CheckpointProposalJob', () => { const block = await makeBlock(txs, globalVariables); p2p.getPendingTxCount.mockResolvedValue(10); - p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); + p2p.iterateEligiblePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); checkpointBuilder.seedBlocks([block], [txs]); @@ -744,7 +744,7 @@ describe('CheckpointProposalJob', () => { const block = await makeBlock(txs, globalVariables); p2p.getPendingTxCount.mockResolvedValue(10); - p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); + p2p.iterateEligiblePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); checkpointBuilder.seedBlocks([block], [txs]); @@ -771,7 +771,7 @@ describe('CheckpointProposalJob', () => { const txs = await Promise.all([makeTx(1, chainId), makeTx(2, chainId), makeTx(3, chainId)]); p2p.getPendingTxCount.mockResolvedValue(10); - p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); + p2p.iterateEligiblePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); // Create 2 blocks - block 1 has 2 txs, block 2 has 1 tx const block1 = await makeBlock(txs.slice(0, 2), globalVariables); @@ -867,7 +867,7 @@ describe('CheckpointProposalJob', () => { const txs = await Promise.all([makeTx(1, chainId)]); p2p.getPendingTxCount.mockResolvedValue(txs.length); - p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); + p2p.iterateEligiblePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); const checkpoint = await job.execute(); @@ -880,7 +880,7 @@ describe('CheckpointProposalJob', () => { // Mock minimal txs (less than minTxsPerBlock) p2p.getPendingTxCount.mockResolvedValue(1); const txs = await Promise.all([makeTx(1, chainId)]); - p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); + p2p.iterateEligiblePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); const block = await makeBlock(txs, globalVariables); checkpointBuilder.seedBlocks([block], [txs]); @@ -904,7 +904,7 @@ describe('CheckpointProposalJob', () => { const block = await makeBlock(txs, globalVariables); p2p.getPendingTxCount.mockResolvedValue(txs.length); - p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); + p2p.iterateEligiblePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); checkpointBuilder.seedBlocks([block], [txs]); @@ -922,7 +922,7 @@ describe('CheckpointProposalJob', () => { it('handles block build failure gracefully', async () => { const txs = await Promise.all([makeTx(1, chainId)]); p2p.getPendingTxCount.mockResolvedValue(txs.length); - p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); + p2p.iterateEligiblePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); // Set up MockCheckpointBuilder to throw on build checkpointBuilder.errorOnBuild = new Error('Block build failed'); diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts index e34e29fd82e5..b25291dfc60d 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts @@ -273,7 +273,7 @@ describe('CheckpointProposalJob Timing Tests', () => { /** Set up p2p mock to return the given transactions */ function mockP2pWithTxs(txs: Tx[]): void { p2p.getPendingTxCount.mockResolvedValue(txs.length); - p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); + p2p.iterateEligiblePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); } /** Create attestations for the given block */ diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index 1da1d7c3100f..e593152ca6b4 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -516,7 +516,7 @@ export class CheckpointProposalJob implements Traceable { // Create iterator to pending txs. We filter out txs already included in previous blocks in the checkpoint // just in case p2p failed to sync the provisional block and didn't get to remove those txs from the mempool yet. const pendingTxs = filter( - this.p2pClient.iteratePendingTxs(), + this.p2pClient.iterateEligiblePendingTxs(), tx => !txHashesAlreadyIncluded.has(tx.txHash.toString()), ); diff --git a/yarn-project/sequencer-client/src/test/utils.ts b/yarn-project/sequencer-client/src/test/utils.ts index c79e65f75138..ce670d41f6d9 100644 --- a/yarn-project/sequencer-client/src/test/utils.ts +++ b/yarn-project/sequencer-client/src/test/utils.ts @@ -56,6 +56,7 @@ export async function makeBlock(txs: Tx[], globalVariables: GlobalVariables): Pr export function mockPendingTxs(p2p: MockProxy, txs: Tx[]): void { p2p.getPendingTxCount.mockResolvedValue(txs.length); p2p.iteratePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); + p2p.iterateEligiblePendingTxs.mockImplementation(() => mockTxIterator(Promise.resolve(txs))); } /** diff --git a/yarn-project/txe/src/state_machine/dummy_p2p_client.ts b/yarn-project/txe/src/state_machine/dummy_p2p_client.ts index f1b02767d255..8d4fcd249100 100644 --- a/yarn-project/txe/src/state_machine/dummy_p2p_client.ts +++ b/yarn-project/txe/src/state_machine/dummy_p2p_client.ts @@ -99,6 +99,10 @@ export class DummyP2P implements P2P { throw new Error('DummyP2P does not implement "iteratePendingTxs"'); } + public iterateEligiblePendingTxs(): AsyncIterableIterator { + throw new Error('DummyP2P does not implement "iterateEligiblePendingTxs"'); + } + public getPendingTxCount(): Promise { throw new Error('DummyP2P does not implement "getPendingTxCount"'); } From f75d5dbce1f1641b87952eb81518c276891a62b8 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 11 Feb 2026 09:37:25 +0000 Subject: [PATCH 03/27] refactor: move maxReceivedAt computation inside tx pool implementation Co-Authored-By: Claude Opus 4.6 --- yarn-project/p2p/src/client/factory.ts | 1 + yarn-project/p2p/src/client/p2p_client.ts | 3 +-- yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts | 7 +++++-- yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts | 4 ++-- .../p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts | 3 ++- yarn-project/p2p/src/test-helpers/testbench-utils.ts | 2 +- 6 files changed, 12 insertions(+), 8 deletions(-) diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index 4da8ddda7796..30299c55aada 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -116,6 +116,7 @@ export async function createP2PClient( { maxPendingTxCount: config.maxPendingTxCount, archivedTxLimit: config.archivedTxLimit, + minTxPoolAgeMs: config.minTxPoolAgeMs, }, dateProvider, ); diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index 4e0a69acdff2..966d11119782 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -489,8 +489,7 @@ export class P2PClient } public async *iterateEligiblePendingTxs(): AsyncIterableIterator { - const maxReceivedAt = this._dateProvider.now() - this.config.minTxPoolAgeMs; - for (const txHash of await this.txPool.getEligiblePendingTxHashes(maxReceivedAt)) { + for (const txHash of await this.txPool.getEligiblePendingTxHashes()) { const tx = await this.txPool.getTxByHash(txHash); if (tx) { yield tx; diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts index 5f7d0fcaa058..1057f57e3954 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/interfaces.ts @@ -37,6 +37,8 @@ export type TxPoolV2Config = { maxPendingTxCount: number; /** Maximum number of archived transactions to retain (0 = disabled) */ archivedTxLimit: number; + /** Minimum age (ms) a transaction must have been in the pool before it's eligible for block building */ + minTxPoolAgeMs: number; }; /** @@ -45,6 +47,7 @@ export type TxPoolV2Config = { export const DEFAULT_TX_POOL_V2_CONFIG: TxPoolV2Config = { maxPendingTxCount: 0, // 0 = disabled archivedTxLimit: 0, // 0 = disabled + minTxPoolAgeMs: 2_000, }; /** @@ -187,8 +190,8 @@ export interface TxPoolV2 extends TypedEventEmitter { /** Gets pending transaction hashes sorted by priority (highest first) */ getPendingTxHashes(): Promise; - /** Gets pending transaction hashes that have been in the pool since before maxReceivedAt, sorted by priority (highest first) */ - getEligiblePendingTxHashes(maxReceivedAt: number): Promise; + /** Gets pending transaction hashes that have been in the pool long enough per minTxPoolAgeMs, sorted by priority (highest first) */ + getEligiblePendingTxHashes(): Promise; /** Gets the count of pending transactions */ getPendingTxCount(): Promise; diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts index 692450a34afd..c37702c77a8f 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.ts @@ -134,8 +134,8 @@ export class AztecKVTxPoolV2 extends (EventEmitter as new () => TypedEventEmitte return this.#queue.put(() => Promise.resolve(this.#impl.getPendingTxHashes())); } - getEligiblePendingTxHashes(maxReceivedAt: number): Promise { - return this.#queue.put(() => Promise.resolve(this.#impl.getEligiblePendingTxHashes(maxReceivedAt))); + getEligiblePendingTxHashes(): Promise { + return this.#queue.put(() => Promise.resolve(this.#impl.getEligiblePendingTxHashes())); } getPendingTxCount(): Promise { diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts index 543b944c6781..fb2d0e24a7ed 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts @@ -553,7 +553,8 @@ export class TxPoolV2Impl { return [...this.#indices.iteratePendingByPriority('desc')].map(hash => TxHash.fromString(hash)); } - getEligiblePendingTxHashes(maxReceivedAt: number): TxHash[] { + getEligiblePendingTxHashes(): TxHash[] { + const maxReceivedAt = this.#dateProvider.now() - this.#config.minTxPoolAgeMs; return [...this.#indices.iterateEligiblePendingByPriority('desc', maxReceivedAt)].map(hash => TxHash.fromString(hash), ); diff --git a/yarn-project/p2p/src/test-helpers/testbench-utils.ts b/yarn-project/p2p/src/test-helpers/testbench-utils.ts index 58819792a5c0..c8c81dc7b997 100644 --- a/yarn-project/p2p/src/test-helpers/testbench-utils.ts +++ b/yarn-project/p2p/src/test-helpers/testbench-utils.ts @@ -163,7 +163,7 @@ export class InMemoryTxPool extends EventEmitter implements TxPoolV2 { return Promise.resolve([...this.txsByHash.keys()].map(key => TxHash.fromString(key))); } - getEligiblePendingTxHashes(_maxReceivedAt: number): Promise { + getEligiblePendingTxHashes(): Promise { return this.getPendingTxHashes(); } From 360aa034923e6efb56019fce4a545443bd2586e6 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 11 Feb 2026 09:46:18 +0000 Subject: [PATCH 04/27] test(p2p): add tests for minimum transaction age eligibility Co-Authored-By: Claude Opus 4.6 --- .../mem_pools/tx_pool_v2/tx_pool_v2.test.ts | 178 ++++++++++++++++++ .../mem_pools/tx_pool_v2/tx_pool_v2_impl.ts | 3 + 2 files changed, 181 insertions(+) diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts index 7b943c56d6b5..4b2273837ba2 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts @@ -1,6 +1,7 @@ import { BlockNumber, CheckpointNumber, IndexWithinCheckpoint, SlotNumber } from '@aztec/foundation/branded-types'; import { timesAsync } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; +import { DateProvider } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { RevertCode } from '@aztec/stdlib/avm'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; @@ -4424,4 +4425,181 @@ describe('TxPoolV2', () => { expect(await pool.getMinedTxCount()).toBe(0); }); }); + + describe('minimum transaction age (getEligiblePendingTxHashes)', () => { + let ageStore: Awaited>; + let ageArchiveStore: Awaited>; + let agePool: AztecKVTxPoolV2; + let mockDateProvider: DateProvider; + let currentTime: number; + + beforeEach(async () => { + currentTime = 10_000; + mockDateProvider = { now: () => currentTime } as DateProvider; + + ageStore = await openTmpStore('p2p-age'); + ageArchiveStore = await openTmpStore('archive-age'); + agePool = new AztecKVTxPoolV2( + ageStore, + ageArchiveStore, + { + l2BlockSource: mockL2BlockSource, + worldStateSynchronizer: mockWorldState, + createTxValidator: () => Promise.resolve(alwaysValidValidator), + }, + undefined, // telemetry + { minTxPoolAgeMs: 2_000 }, + mockDateProvider, + ); + await agePool.start(); + }); + + afterEach(async () => { + await agePool.stop(); + await ageStore.delete(); + await ageArchiveStore.delete(); + }); + + it('newly added tx is not eligible before minTxPoolAgeMs elapses', async () => { + const tx = await mockTxWithFee(1, 10); + await agePool.addPendingTxs([tx]); + + // getPendingTxHashes returns the tx regardless of age + expect(await agePool.getPendingTxHashes()).toHaveLength(1); + + // At the same time, the tx is NOT eligible (added at 10000, cutoff is 10000 - 2000 = 8000) + expect(await agePool.getEligiblePendingTxHashes()).toHaveLength(0); + }); + + it('tx becomes eligible after minTxPoolAgeMs elapses', async () => { + const tx = await mockTxWithFee(1, 10); + await agePool.addPendingTxs([tx]); + + // Advance time past the minimum age + currentTime = 12_001; + + const eligible = await agePool.getEligiblePendingTxHashes(); + expect(toStrings(eligible)).toEqual([hashOf(tx)]); + }); + + it('tx becomes eligible at exactly minTxPoolAgeMs', async () => { + const tx = await mockTxWithFee(1, 10); + await agePool.addPendingTxs([tx]); + + // Advance time to exactly the min age boundary (added at 10000, need 12000) + currentTime = 12_000; + + const eligible = await agePool.getEligiblePendingTxHashes(); + expect(eligible).toHaveLength(1); + }); + + it('filters ineligible txs while returning eligible ones', async () => { + // Add tx1 at time 10000 + const tx1 = await mockTxWithFee(1, 10); + await agePool.addPendingTxs([tx1]); + + // Advance time and add tx2 at time 11000 + currentTime = 11_000; + const tx2 = await mockTxWithFee(2, 20); + await agePool.addPendingTxs([tx2]); + + // At time 12500: tx1 (added at 10000) is eligible, tx2 (added at 11000) is not + currentTime = 12_500; + + const eligible = await agePool.getEligiblePendingTxHashes(); + expect(toStrings(eligible)).toEqual([hashOf(tx1)]); + + // All are still in getPendingTxHashes + expect(await agePool.getPendingTxHashes()).toHaveLength(2); + }); + + it('eligible txs are returned in priority order', async () => { + // Add three txs at the same time with different priorities + const txLow = await mockTxWithFee(1, 5); + const txMid = await mockTxWithFee(2, 10); + const txHigh = await mockTxWithFee(3, 20); + await agePool.addPendingTxs([txLow, txMid, txHigh]); + + // Advance time so all are eligible + currentTime = 13_000; + + const eligible = await agePool.getEligiblePendingTxHashes(); + expect(toStrings(eligible)).toEqual([hashOf(txHigh), hashOf(txMid), hashOf(txLow)]); + }); + + it('hydrated txs are immediately eligible (receivedAt = 0)', async () => { + // Add a tx, stop, and re-hydrate into a new pool + const tx = await mockTxWithFee(1, 10); + await agePool.addPendingTxs([tx]); + await agePool.stop(); + + // Set time to exactly minTxPoolAgeMs — hydrated txs (receivedAt=0) should still be eligible + // because 0 <= (2000 - 2000) = 0 + currentTime = 2_000; + + const hydratedPool = new AztecKVTxPoolV2( + ageStore, + ageArchiveStore, + { + l2BlockSource: mockL2BlockSource, + worldStateSynchronizer: mockWorldState, + createTxValidator: () => Promise.resolve(alwaysValidValidator), + }, + undefined, + { minTxPoolAgeMs: 2_000 }, + mockDateProvider, + ); + await hydratedPool.start(); + + // Hydrated tx should be immediately eligible even at time 0 + const eligible = await hydratedPool.getEligiblePendingTxHashes(); + expect(toStrings(eligible)).toEqual([hashOf(tx)]); + + await hydratedPool.stop(); + }); + + it('updateConfig changes minTxPoolAgeMs', async () => { + const tx = await mockTxWithFee(1, 10); + await agePool.addPendingTxs([tx]); + + // Not eligible yet at default 2000ms + expect(await agePool.getEligiblePendingTxHashes()).toHaveLength(0); + + // Reduce the minimum age to 0ms + await agePool.updateConfig({ minTxPoolAgeMs: 0 }); + + // Now the tx should be immediately eligible + const eligible = await agePool.getEligiblePendingTxHashes(); + expect(toStrings(eligible)).toEqual([hashOf(tx)]); + }); + + it('minTxPoolAgeMs of 0 makes all txs immediately eligible', async () => { + await agePool.updateConfig({ minTxPoolAgeMs: 0 }); + + const tx = await mockTxWithFee(1, 10); + await agePool.addPendingTxs([tx]); + + const eligible = await agePool.getEligiblePendingTxHashes(); + expect(toStrings(eligible)).toEqual([hashOf(tx)]); + }); + + it('protected and mined txs are excluded from eligible pending', async () => { + const txPending = await mockTxWithFee(1, 10); + const txProtected = await mockTxWithFee(2, 20); + const txMined = await mockTxWithFee(3, 30); + + await agePool.addPendingTxs([txPending, txProtected, txMined]); + + // Advance time so all are old enough + currentTime = 13_000; + + // Transition txProtected to protected and txMined to mined + await agePool.addProtectedTxs([txProtected], slot1Header); + await agePool.addMinedTxs([txMined], slot1Header); + + // Only the pending tx should appear in eligible results + const eligible = await agePool.getEligiblePendingTxHashes(); + expect(toStrings(eligible)).toEqual([hashOf(txPending)]); + }); + }); }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts index fb2d0e24a7ed..28053eee58de 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts @@ -604,6 +604,9 @@ export class TxPoolV2Impl { this.#config.archivedTxLimit = config.archivedTxLimit; this.#archive.updateLimit(config.archivedTxLimit); } + if (config.minTxPoolAgeMs !== undefined) { + this.#config.minTxPoolAgeMs = config.minTxPoolAgeMs; + } // Update eviction rules with new config this.#evictionManager.updateConfig(config); } From 578208e1d19069cca350721e653c1ad0e8a53454 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 11 Feb 2026 09:49:10 +0000 Subject: [PATCH 05/27] Default to zero min age in tests --- yarn-project/end-to-end/src/fixtures/setup.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/yarn-project/end-to-end/src/fixtures/setup.ts b/yarn-project/end-to-end/src/fixtures/setup.ts index eb323801fa1b..ede28472769e 100644 --- a/yarn-project/end-to-end/src/fixtures/setup.ts +++ b/yarn-project/end-to-end/src/fixtures/setup.ts @@ -289,6 +289,8 @@ export async function setup( config.enforceTimeTable = !!opts.enforceTimeTable; config.listenAddress = '127.0.0.1'; + config.minTxPoolAgeMs = opts.minTxPoolAgeMs ?? 0; + const logger = getLogger(); // Create a temp directory for any services that need it and cleanup later From 005cf8b569e2eda4cb3bf5715d1f492617008284 Mon Sep 17 00:00:00 2001 From: ludamad Date: Wed, 11 Feb 2026 10:19:04 +0000 Subject: [PATCH 06/27] chore: standalone forge broadcast wrapper with retry, timeout, and anvil detection (#19824) --- l1-contracts/package.json | 1 + l1-contracts/scripts/forge_broadcast.js | 340 ++++++++++++++++++ l1-contracts/scripts/run_rollup_upgrade.sh | 12 +- l1-contracts/scripts/test_rollup_upgrade.sh | 16 +- l1-contracts/yarn.lock | 112 +++--- .../ethereum/src/deploy_aztec_l1_contracts.ts | 64 ++-- .../scripts/copy-foundry-artifacts.sh | 5 +- 7 files changed, 455 insertions(+), 95 deletions(-) create mode 100755 l1-contracts/scripts/forge_broadcast.js diff --git a/l1-contracts/package.json b/l1-contracts/package.json index 8d7e6e085142..f7c83b6017d4 100644 --- a/l1-contracts/package.json +++ b/l1-contracts/package.json @@ -1,6 +1,7 @@ { "name": "@aztec/l1-contracts", "version": "0.1.0", + "type": "module", "license": "Apache-2.0", "description": "Aztec contracts for the Ethereum mainnet and testnets", "devDependencies": { diff --git a/l1-contracts/scripts/forge_broadcast.js b/l1-contracts/scripts/forge_broadcast.js new file mode 100755 index 000000000000..d8df4462617a --- /dev/null +++ b/l1-contracts/scripts/forge_broadcast.js @@ -0,0 +1,340 @@ +#!/usr/bin/env node +// Note: this would be .ts but Node.js refuses to load .ts from node_modules. + +// forge_broadcast.js - Reliable forge script broadcast with retry and timeout. +// +// Wraps `forge script` with: +// 1. --batch-size 8 to prevent forge broadcast hangs (forge bug with large RPC batches) +// 2. External timeout (forge's --timeout is unreliable for broadcast hangs) +// 3. Retry with --resume on real chains, or full retry from scratch on anvil +// +// Anvil's auto-miner has a race condition where batched transactions can get stranded +// in the mempool — they arrive after the auto-miner already triggered for the batch, +// and sit waiting for the next trigger that never comes. Neither evm_mine nor --resume +// can recover these stuck transactions. Interval mining (--block-time) avoids this issue. +// +// On anvil, we work around this by clearing broadcast artifacts and retrying from scratch. +// On real chains (where this anvil-specific bug doesn't apply), we use --resume. +// +// Usage: +// ./scripts/forge_broadcast.js +// +// Pass the same args you'd pass to `forge script`, WITHOUT --broadcast or --batch-size. +// The wrapper adds those automatically. +// +// Example: +// ./scripts/forge_broadcast.js script/deploy/Deploy.s.sol:Deploy \ +// --rpc-url "$RPC_URL" --private-key "$KEY" -vvv +// +// Environment variables: +// FORGE_BROADCAST_TIMEOUT - Override timeout per attempt in seconds (auto-detected from chain ID) +// FORGE_BROADCAST_MAX_RETRIES - Max retries after initial attempt (default: 3) +// +// Uses only Node.js built-ins (no external dependencies). + +import { spawn } from "node:child_process"; +import { rmSync, writeSync } from "node:fs"; +import { request as httpRequest } from "node:http"; +import { request as httpsRequest } from "node:https"; + +// Chain IDs for timeout selection. +const MAINNET_CHAIN_ID = 1; +const SEPOLIA_CHAIN_ID = 11155111; + +// Timeout per attempt: 300s for mainnet/sepolia (real chains are slow), 50s for everything else. +// FORGE_BROADCAST_TIMEOUT env var overrides the auto-detected value. +function getDefaultTimeout(chainId) { + if (chainId === MAINNET_CHAIN_ID || chainId === SEPOLIA_CHAIN_ID) return 300; + return 50; +} + +const MAX_RETRIES = parseInt( + process.env.FORGE_BROADCAST_MAX_RETRIES ?? "3", + 10, +); + +// Batch size of 8 prevents forge from hanging during broadcast. +// See: https://github.com/foundry-rs/foundry/issues/6796 +const BATCH_SIZE = 8; +const KILL_GRACE = 15_000; +// Exit code indicating a timeout, matching the `timeout` coreutil convention. +const EXIT_TIMEOUT = 124; +// Delay before retry to let pending transactions settle in the mempool. +const RETRY_DELAY = 10_000; + +function log(msg) { + process.stderr.write(`[forge_broadcast] ${msg}\n`); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Extract --rpc-url value from forge args. */ +function extractRpcUrl(args) { + for (let i = 0; i < args.length - 1; i++) { + if (args[i] === "--rpc-url") return args[i + 1]; + } + return undefined; +} + +/** Strip --verify from args, returning the filtered args and whether --verify was present. */ +function extractVerifyFlag(args) { + const filtered = args.filter((a) => a !== "--verify"); + return { args: filtered, verify: filtered.length !== args.length }; +} + +const RPC_TIMEOUT = 10_000; + +/** JSON-RPC call using Node.js built-ins. Rejects on JSON-RPC errors and timeouts. */ +function rpcCall(rpcUrl, method, params) { + return new Promise((resolve, reject) => { + const url = new URL(rpcUrl); + const body = JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }); + const reqFn = url.protocol === "https:" ? httpsRequest : httpRequest; + + const timer = setTimeout(() => { + req.destroy(); + reject(new Error(`RPC call ${method} timed out after ${RPC_TIMEOUT}ms`)); + }, RPC_TIMEOUT); + + const req = reqFn( + url, + { method: "POST", headers: { "Content-Type": "application/json" } }, + (res) => { + let data = ""; + res.on("data", (chunk) => (data += chunk)); + res.on("end", () => { + clearTimeout(timer); + try { + const parsed = JSON.parse(data); + if (parsed.error) { + reject( + new Error( + `RPC error for ${method}: ${JSON.stringify(parsed.error)}`, + ), + ); + } else { + resolve(parsed.result); + } + } catch { + reject(new Error(`Bad RPC response: ${data.slice(0, 200)}`)); + } + }); + }, + ); + req.on("error", (err) => { + clearTimeout(timer); + reject(err); + }); + req.write(body); + req.end(); + }); +} + +/** Detect if the RPC endpoint is an anvil dev node via web3_clientVersion. */ +async function detectAnvil(rpcUrl) { + try { + const version = await rpcCall(rpcUrl, "web3_clientVersion", []); + return version.toLowerCase().includes("anvil"); + } catch { + return false; + } +} + +/** Get the chain ID from the RPC endpoint. */ +async function getChainId(rpcUrl) { + try { + const result = await rpcCall(rpcUrl, "eth_chainId", []); + return parseInt(result, 16); + } catch { + return undefined; + } +} + +function runForge(args, timeoutSecs) { + return new Promise((resolve) => { + const proc = spawn( + "forge", + ["script", ...args, "--broadcast", "--batch-size", String(BATCH_SIZE)], + { + stdio: ["ignore", "pipe", "inherit"], // buffer stdout, pass stderr through + }, + ); + + const stdout = []; + proc.stdout.on("data", (chunk) => stdout.push(chunk)); + + let timedOut = false; + let settled = false; + let killTimer; + + const timer = setTimeout(() => { + timedOut = true; + proc.kill("SIGTERM"); + killTimer = setTimeout(() => proc.kill("SIGKILL"), KILL_GRACE); + }, timeoutSecs * 1000); + + const finish = (code) => { + if (settled) return; + settled = true; + clearTimeout(timer); + clearTimeout(killTimer); + resolve({ exitCode: timedOut ? EXIT_TIMEOUT : code, stdout }); + }; + + proc.on("error", () => finish(1)); + proc.on("close", (code) => finish(code ?? 1)); + }); +} + +// Main + +// Strip --verify from args so it doesn't run during broadcast attempts. Verification +// happens after all receipts are collected (foundry-rs/foundry crates/script/src/lib.rs:333-338) +// and forge exits non-zero if ANY verification fails (crates/script/src/verify.rs), even when +// all transactions landed. We run verification as a separate step after broadcast succeeds. +const { args: forgeArgs, verify: wantsVerify } = extractVerifyFlag( + process.argv.slice(2), +); +const rpcUrl = extractRpcUrl(forgeArgs); + +// Query chain info from RPC at startup. +const chainId = rpcUrl ? await getChainId(rpcUrl) : undefined; +const TIMEOUT = process.env.FORGE_BROADCAST_TIMEOUT + ? parseInt(process.env.FORGE_BROADCAST_TIMEOUT, 10) + : getDefaultTimeout(chainId); + +log( + `chain_id=${chainId ?? "unknown"}, timeout=${TIMEOUT}s, max_retries=${MAX_RETRIES}, batch_size=${BATCH_SIZE}${wantsVerify ? ", verify=true (after broadcast)" : ""}`, +); + +// Detect anvil once at startup. On anvil, retries reset the chain and start from scratch +// instead of using --resume, because anvil's auto-miner can strand transactions in the +// mempool in an unrecoverable state (neither evm_mine nor --resume can flush them). +const isAnvil = rpcUrl ? await detectAnvil(rpcUrl) : false; +if (isAnvil) { + log("Detected anvil — retries will reset chain instead of using --resume."); +} + +/** + * Run contract verification via `forge script --resume --verify --broadcast` (no timeout). + * Verification uses broadcast artifacts + re-compilation — it doesn't need simulation data. + * See: foundry-rs/foundry crates/script/src/build.rs (CompiledState::resume) and + * crates/script/src/verify.rs (verify_contracts). + * Failure is logged but doesn't affect the exit code — transactions already landed. + */ +async function runVerification(args) { + log("Running contract verification (no timeout)..."); + const verifyResult = await new Promise((resolve) => { + const proc = spawn( + "forge", + ["script", ...args, "--broadcast", "--resume", "--verify"], + { + stdio: ["ignore", "inherit", "inherit"], + }, + ); + let settled = false; + proc.on("error", () => { + if (!settled) { + settled = true; + resolve(1); + } + }); + proc.on("close", (code) => { + if (!settled) { + settled = true; + resolve(code ?? 1); + } + }); + }); + if (verifyResult === 0) { + log("Contract verification succeeded."); + } else { + log( + `Contract verification failed (exit ${verifyResult}). Transactions are on-chain; verify manually if needed.`, + ); + } +} + +/** Write buffered stdout to fd 1 (synchronous) and exit. */ +function emitAndExit(result, code) { + const data = Buffer.concat(result.stdout); + if (data.length > 0) { + writeSync(1, data); + } + process.exit(code); +} + +/** Run verification if requested, then emit stdout and exit. */ +async function verifyAndExit(result) { + if (wantsVerify) { + await runVerification(forgeArgs); + } + emitAndExit(result, 0); +} + +// Attempt 1: initial broadcast +log(`Attempt 1/${MAX_RETRIES + 1}: broadcasting...`); +let result = await runForge(forgeArgs, TIMEOUT); + +if (result.exitCode === 0) { + log("Broadcast succeeded on first attempt."); + await verifyAndExit(result); +} + +log( + `Attempt 1 ${result.exitCode === EXIT_TIMEOUT ? `timed out after ${TIMEOUT}s` : `failed (exit ${result.exitCode})`}.`, +); + +for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { + log(`Waiting ${RETRY_DELAY / 1000}s before retry...`); + await sleep(RETRY_DELAY); + + if (isAnvil) { + // On anvil: retry from scratch instead of --resume. + // + // Anvil's auto-miner has a race condition where batched transactions can arrive + // after the auto-miner already triggered, stranding them in the mempool. --resume + // just waits for these same stuck transactions and hangs again. A fresh retry + // re-simulates from current chain state and re-sends, which works because: + // - Forge computes new nonces from on-chain state + // - New transactions replace any stuck ones with the same nonce + // - The race condition is intermittent (~0.04%), so retries almost always succeed + rmSync("broadcast", { recursive: true, force: true }); + + log( + `Attempt ${attempt + 1}/${MAX_RETRIES + 1}: retrying from scratch (anvil)...`, + ); + result = await runForge(forgeArgs, TIMEOUT); + } else { + // On real chains: use --resume to pick up unmined transactions. + // --resume re-reads broadcast artifacts and resubmits unmined transactions. + // NOTE: --resume skips simulation, so console.log output (e.g. JSON deploy results) + // is only produced on the first attempt. We keep the first attempt's stdout (`result`) + // and only check the exit code from the --resume attempt. + log(`Attempt ${attempt + 1}/${MAX_RETRIES + 1}: --resume`); + const resumeResult = await runForge([...forgeArgs, "--resume"], TIMEOUT); + + if (resumeResult.exitCode === 0) { + log(`Broadcast succeeded on attempt ${attempt + 1}.`); + // Emit the first attempt's stdout which has the JSON simulation output. + await verifyAndExit(result); + } + log( + `Attempt ${attempt + 1} ${resumeResult.exitCode === EXIT_TIMEOUT ? `timed out after ${TIMEOUT}s` : `failed (exit ${resumeResult.exitCode})`}.`, + ); + continue; + } + + if (result.exitCode === 0) { + log(`Broadcast succeeded on attempt ${attempt + 1}.`); + await verifyAndExit(result); + } + log( + `Attempt ${attempt + 1} ${result.exitCode === EXIT_TIMEOUT ? `timed out after ${TIMEOUT}s` : `failed (exit ${result.exitCode})`}.`, + ); +} + +log(`All ${MAX_RETRIES + 1} attempts failed.`); +emitAndExit(result, result.exitCode); diff --git a/l1-contracts/scripts/run_rollup_upgrade.sh b/l1-contracts/scripts/run_rollup_upgrade.sh index f8f25c77f7e1..3cabda3ecf09 100755 --- a/l1-contracts/scripts/run_rollup_upgrade.sh +++ b/l1-contracts/scripts/run_rollup_upgrade.sh @@ -20,10 +20,10 @@ echo "=== Deploying rollup upgrade ===" echo "Registry: $registry_address" REGISTRY_ADDRESS="$registry_address" \ - REAL_VERIFIER="${REAL_VERIFIER:-true}" \ - forge script script/deploy/DeployRollupForUpgrade.s.sol:DeployRollupForUpgrade \ - --rpc-url "$L1_RPC_URL" \ - --private-key "$ROLLUP_DEPLOYMENT_PRIVATE_KEY" \ - --broadcast \ - ${ETHERSCAN_API_KEY:+--verify} \ +REAL_VERIFIER="${REAL_VERIFIER:-true}" \ +./scripts/forge_broadcast.js \ + script/deploy/DeployRollupForUpgrade.s.sol:DeployRollupForUpgrade \ + --rpc-url "$L1_RPC_URL" \ + --private-key "$ROLLUP_DEPLOYMENT_PRIVATE_KEY" \ + ${ETHERSCAN_API_KEY:+--verify} \ -vvv diff --git a/l1-contracts/scripts/test_rollup_upgrade.sh b/l1-contracts/scripts/test_rollup_upgrade.sh index 38e79b46d763..5633984d94b9 100755 --- a/l1-contracts/scripts/test_rollup_upgrade.sh +++ b/l1-contracts/scripts/test_rollup_upgrade.sh @@ -17,19 +17,25 @@ cleanup() { } trap cleanup EXIT -echo "=== Starting anvil ===" -anvil & +# Clean stale broadcast artifacts from previous runs to avoid nonce conflicts. +rm -rf broadcast/ + +# Use a random port to avoid conflicts with other anvil instances. +ANVIL_PORT="${ANVIL_PORT:-$(shuf -i 10000-60000 -n 1)}" + +echo "=== Starting anvil on port $ANVIL_PORT ===" +anvil --port "$ANVIL_PORT" & anvil_pid=$! sleep 2 -export L1_RPC_URL="http://127.0.0.1:8545" +export L1_RPC_URL="http://127.0.0.1:$ANVIL_PORT" export ROLLUP_DEPLOYMENT_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" echo "=== Deploying initial L1 contracts ===" -forge script script/deploy/DeployAztecL1Contracts.s.sol:DeployAztecL1Contracts \ +./scripts/forge_broadcast.js \ + script/deploy/DeployAztecL1Contracts.s.sol:DeployAztecL1Contracts \ --rpc-url "$L1_RPC_URL" \ --private-key "$ROLLUP_DEPLOYMENT_PRIVATE_KEY" \ - --broadcast \ --json > /tmp/initial_deploy.jsonl deploy_json=$(head -1 /tmp/initial_deploy.jsonl | jq -r '.logs[0]' | sed 's/JSON DEPLOY RESULT: //') diff --git a/l1-contracts/yarn.lock b/l1-contracts/yarn.lock index 9c68cb8b8b46..26f8e9c14913 100644 --- a/l1-contracts/yarn.lock +++ b/l1-contracts/yarn.lock @@ -6,9 +6,9 @@ __metadata: cacheKey: 10c0 "@adraffy/ens-normalize@npm:^1.11.0": - version: 1.11.0 - resolution: "@adraffy/ens-normalize@npm:1.11.0" - checksum: 10c0/5111d0f1a273468cb5661ed3cf46ee58de8f32f84e2ebc2365652e66c1ead82649df94c736804e2b9cfa831d30ef24e1cc3575d970dbda583416d3a98d8870a6 + version: 1.11.1 + resolution: "@adraffy/ens-normalize@npm:1.11.1" + checksum: 10c0/b364e2a57131db278ebf2f22d1a1ac6d8aea95c49dd2bbbc1825870b38aa91fd8816aba580a1f84edc50a45eb6389213dacfd1889f32893afc8549a82d304767 languageName: node linkType: hard @@ -23,20 +23,20 @@ __metadata: linkType: soft "@babel/code-frame@npm:^7.0.0": - version: 7.27.1 - resolution: "@babel/code-frame@npm:7.27.1" + version: 7.29.0 + resolution: "@babel/code-frame@npm:7.29.0" dependencies: - "@babel/helper-validator-identifier": "npm:^7.27.1" + "@babel/helper-validator-identifier": "npm:^7.28.5" js-tokens: "npm:^4.0.0" picocolors: "npm:^1.1.1" - checksum: 10c0/5dd9a18baa5fce4741ba729acc3a3272c49c25cb8736c4b18e113099520e7ef7b545a4096a26d600e4416157e63e87d66db46aa3fbf0a5f2286da2705c12da00 + checksum: 10c0/d34cc504e7765dfb576a663d97067afb614525806b5cad1a5cc1a7183b916fec8ff57fa233585e3926fd5a9e6b31aae6df91aa81ae9775fb7a28f658d3346f0d languageName: node linkType: hard -"@babel/helper-validator-identifier@npm:^7.27.1": - version: 7.27.1 - resolution: "@babel/helper-validator-identifier@npm:7.27.1" - checksum: 10c0/c558f11c4871d526498e49d07a84752d1800bf72ac0d3dad100309a2eaba24efbf56ea59af5137ff15e3a00280ebe588560534b0e894a4750f8b1411d8f78b84 +"@babel/helper-validator-identifier@npm:^7.28.5": + version: 7.28.5 + resolution: "@babel/helper-validator-identifier@npm:7.28.5" + checksum: 10c0/42aaebed91f739a41f3d80b72752d1f95fd7c72394e8e4bd7cdd88817e0774d80a432451bcba17c2c642c257c483bf1d409dd4548883429ea9493a3bc4ab0847 languageName: node linkType: hard @@ -142,11 +142,11 @@ __metadata: linkType: hard "@noble/curves@npm:^1.9.1, @noble/curves@npm:~1.9.0": - version: 1.9.2 - resolution: "@noble/curves@npm:1.9.2" + version: 1.9.7 + resolution: "@noble/curves@npm:1.9.7" dependencies: "@noble/hashes": "npm:1.8.0" - checksum: 10c0/21d049ae4558beedbf5da0004407b72db84360fa29d64822d82dc9e80251e1ecb46023590cc4b20e70eed697d1b87279b4911dc39f8694c51c874289cfc8e9a7 + checksum: 10c0/150014751ebe8ca06a8654ca2525108452ea9ee0be23430332769f06808cddabfe84f248b6dbf836916bc869c27c2092957eec62c7506d68a1ed0a624017c2a3 languageName: node linkType: hard @@ -190,14 +190,14 @@ __metadata: languageName: node linkType: hard -"@pnpm/npm-conf@npm:^2.1.0": - version: 2.3.1 - resolution: "@pnpm/npm-conf@npm:2.3.1" +"@pnpm/npm-conf@npm:^3.0.2": + version: 3.0.2 + resolution: "@pnpm/npm-conf@npm:3.0.2" dependencies: "@pnpm/config.env-replace": "npm:^1.1.0" "@pnpm/network.ca-file": "npm:^1.0.1" config-chain: "npm:^1.1.11" - checksum: 10c0/778a3a34ff7d6000a2594d2a9821f873f737bc56367865718b2cf0ba5d366e49689efe7975148316d7afd8e6f1dcef7d736fbb6ea7ef55caadd1dc93a36bb302 + checksum: 10c0/50026ae4cac7d5d055d4dd4b2886fbc41964db6179406cf2decf625e7a280fbfffd47380df584c085464deba060101169caca5f79e6a062b6c25b527bf60cb67 languageName: node linkType: hard @@ -265,9 +265,9 @@ __metadata: linkType: hard "@solidity-parser/parser@npm:^0.20.0": - version: 0.20.1 - resolution: "@solidity-parser/parser@npm:0.20.1" - checksum: 10c0/fa08c719bace194cb82be80f0efd9c57863aea831ff587a9268752a84a35f00daa8c28c9f5587c64d5cbb969a98f8df714088acdd581702376e45d48d57ee8af + version: 0.20.2 + resolution: "@solidity-parser/parser@npm:0.20.2" + checksum: 10c0/23b0b7ed343a4fa55cb8621cf4fc81b0bdd791a4ee7e96ced7778db07de66d4e418db2c2dc1525b1f82fc0310ac829c78382327292b37e35cb30a19961fcb429 languageName: node linkType: hard @@ -290,9 +290,9 @@ __metadata: linkType: hard "@types/http-cache-semantics@npm:^4.0.2": - version: 4.0.4 - resolution: "@types/http-cache-semantics@npm:4.0.4" - checksum: 10c0/51b72568b4b2863e0fe8d6ce8aad72a784b7510d72dc866215642da51d84945a9459fa89f49ec48f1e9a1752e6a78e85a4cda0ded06b1c73e727610c925f9ce6 + version: 4.2.0 + resolution: "@types/http-cache-semantics@npm:4.2.0" + checksum: 10c0/82dd33cbe7d4843f1e884a251c6a12d385b62274353b9db167462e7fbffdbb3a83606f9952203017c5b8cabbd7b9eef0cf240a3a9dedd20f69875c9701939415 languageName: node linkType: hard @@ -304,17 +304,17 @@ __metadata: linkType: hard "abitype@npm:^1.0.8": - version: 1.0.8 - resolution: "abitype@npm:1.0.8" + version: 1.2.3 + resolution: "abitype@npm:1.2.3" peerDependencies: typescript: ">=5.0.4" - zod: ^3 >=3.22.0 + zod: ^3.22.0 || ^4.0.0 peerDependenciesMeta: typescript: optional: true zod: optional: true - checksum: 10c0/d3393f32898c1f0f6da4eed2561da6830dcd0d5129a160fae9517214236ee6a6c8e5a0380b8b960c5bc1b949320bcbd015ec7f38b5d7444f8f2b854a1b5dd754 + checksum: 10c0/c8740de1ae4961723a153224a52cb9a34a57903fb5c2ad61d5082b0b79b53033c9335381aa8c663c7ec213c9955a9853f694d51e95baceedef27356f7745c634 languageName: node linkType: hard @@ -501,14 +501,14 @@ __metadata: linkType: hard "debug@npm:^4.3.4": - version: 4.4.1 - resolution: "debug@npm:4.4.1" + version: 4.4.3 + resolution: "debug@npm:4.4.3" dependencies: ms: "npm:^2.1.3" peerDependenciesMeta: supports-color: optional: true - checksum: 10c0/d2b44bc1afd912b49bb7ebb0d50a860dc93a4dd7d946e8de94abc957bb63726b7dd5aa48c18c2386c379ec024c46692e15ed3ed97d481729f929201e671fcd55 + checksum: 10c0/d79136ec6c83ecbefd0f6a5593da6a9c91ec4d7ddc4b54c883d6e71ec9accb5f67a1a5e96d00a328196b5b5c86d365e98d8a3a70856aaf16b4e7b1985e67f5a6 languageName: node linkType: hard @@ -543,11 +543,11 @@ __metadata: linkType: hard "error-ex@npm:^1.3.1": - version: 1.3.2 - resolution: "error-ex@npm:1.3.2" + version: 1.3.4 + resolution: "error-ex@npm:1.3.4" dependencies: is-arrayish: "npm:^0.2.1" - checksum: 10c0/ba827f89369b4c93382cfca5a264d059dfefdaa56ecc5e338ffa58a6471f5ed93b71a20add1d52290a4873d92381174382658c885ac1a2305f7baca363ce9cce + checksum: 10c0/b9e34ff4778b8f3b31a8377e1c654456f4c41aeaa3d10a1138c3b7635d8b7b2e03eb2475d46d8ae055c1f180a1063e100bffabf64ea7e7388b37735df5328664 languageName: node linkType: hard @@ -605,9 +605,9 @@ __metadata: linkType: hard "fast-uri@npm:^3.0.1": - version: 3.0.6 - resolution: "fast-uri@npm:3.0.6" - checksum: 10c0/74a513c2af0584448aee71ce56005185f81239eab7a2343110e5bad50c39ad4fb19c5a6f99783ead1cac7ccaf3461a6034fda89fffa2b30b6d99b9f21c2f9d29 + version: 3.1.0 + resolution: "fast-uri@npm:3.1.0" + checksum: 10c0/44364adca566f70f40d1e9b772c923138d47efeac2ae9732a872baafd77061f26b097ba2f68f0892885ad177becd065520412b8ffeec34b16c99433c5b9e2de7 languageName: node linkType: hard @@ -758,13 +758,13 @@ __metadata: linkType: hard "js-yaml@npm:^4.1.0": - version: 4.1.0 - resolution: "js-yaml@npm:4.1.0" + version: 4.1.1 + resolution: "js-yaml@npm:4.1.1" dependencies: argparse: "npm:^2.0.1" bin: js-yaml: bin/js-yaml.js - checksum: 10c0/184a24b4eaacfce40ad9074c64fd42ac83cf74d8c8cd137718d456ced75051229e5061b8633c3366b8aada17945a7a356b337828c19da92b51ae62126575018f + checksum: 10c0/561c7d7088c40a9bb53cc75becbfb1df6ae49b34b5e6e5a81744b14ae8667ec564ad2527709d1a6e7d5e5fa6d483aa0f373a50ad98d42fde368ec4a190d4fae7 languageName: node linkType: hard @@ -829,9 +829,9 @@ __metadata: linkType: hard "lodash@npm:^4.17.21": - version: 4.17.21 - resolution: "lodash@npm:4.17.21" - checksum: 10c0/d8cbea072bb08655bb4c989da418994b073a608dffa608b09ac04b43a791b12aeae7cd7ad919aa4c925f33b48490b5cfe6c1f71d827956071dae2e7bb3a6b74c + version: 4.17.23 + resolution: "lodash@npm:4.17.23" + checksum: 10c0/1264a90469f5bb95d4739c43eb6277d15b6d9e186df4ac68c3620443160fc669e2f14c11e7d8b2ccf078b81d06147c01a8ccced9aab9f9f63d50dcf8cace6bf6 languageName: node linkType: hard @@ -887,9 +887,9 @@ __metadata: linkType: hard "normalize-url@npm:^8.0.0": - version: 8.0.2 - resolution: "normalize-url@npm:8.0.2" - checksum: 10c0/1c62eee6ce184ad4a463ff2984ce5e440a5058c9dd7c5ef80c0a7696bbb1d3638534e266afb14ef9678dfa07fb6c980ef4cde990c80eeee55900c378b7970584 + version: 8.1.1 + resolution: "normalize-url@npm:8.1.1" + checksum: 10c0/1beb700ce42acb2288f39453cdf8001eead55bbf046d407936a40404af420b8c1c6be97a869884ae9e659d7b1c744e40e905c875ac9290644eec2e3e6fb0b370 languageName: node linkType: hard @@ -903,8 +903,8 @@ __metadata: linkType: hard "ox@npm:^0.8.3": - version: 0.8.3 - resolution: "ox@npm:0.8.3" + version: 0.8.9 + resolution: "ox@npm:0.8.9" dependencies: "@adraffy/ens-normalize": "npm:^1.11.0" "@noble/ciphers": "npm:^1.3.0" @@ -919,7 +919,7 @@ __metadata: peerDependenciesMeta: typescript: optional: true - checksum: 10c0/3fd2ef5322c2999331cfe46121b3685615236c30f94ca3d956097126e1ce04758c96a76e75198b6c6e34c4a0c5e2dbae7bfa0224ba7e38b9518558f0a3105123 + checksum: 10c0/d3a0c4e3f908e0d18914f17d9c832e777de6caf7d8395bf35978d9cca196e540fe63c230a40bfe462eb3a320c1aba0749e0bede0a5fc4e8501b20d4e784c64ad languageName: node linkType: hard @@ -1036,11 +1036,11 @@ __metadata: linkType: hard "registry-auth-token@npm:^5.0.1": - version: 5.1.0 - resolution: "registry-auth-token@npm:5.1.0" + version: 5.1.1 + resolution: "registry-auth-token@npm:5.1.1" dependencies: - "@pnpm/npm-conf": "npm:^2.1.0" - checksum: 10c0/316229bd8a4acc29a362a7a3862ff809e608256f0fd9e0b133412b43d6a9ea18743756a0ec5ee1467a5384e1023602b85461b3d88d1336b11879e42f7cf02c12 + "@pnpm/npm-conf": "npm:^3.0.2" + checksum: 10c0/86b0f7fd87d327cb4177fee69bcf96563147ea72e206bc9c7a6a50a51c785a31b83a6c45956a489ed292d23b908b2755a075d0b2f7fec1ba91b1fb800b24cee3 languageName: node linkType: hard @@ -1084,11 +1084,11 @@ __metadata: linkType: hard "semver@npm:^7.3.7, semver@npm:^7.5.2, semver@npm:^7.5.4": - version: 7.7.2 - resolution: "semver@npm:7.7.2" + version: 7.7.4 + resolution: "semver@npm:7.7.4" bin: semver: bin/semver.js - checksum: 10c0/aca305edfbf2383c22571cb7714f48cadc7ac95371b4b52362fb8eeffdfbc0de0669368b82b2b15978f8848f01d7114da65697e56cd8c37b0dab8c58e543f9ea + checksum: 10c0/5215ad0234e2845d4ea5bb9d836d42b03499546ddafb12075566899fc617f68794bb6f146076b6881d755de17d6c6cc73372555879ec7dce2c2feee947866ad2 languageName: node linkType: hard diff --git a/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts b/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts index e6a6143c4843..6497f5c0400e 100644 --- a/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts +++ b/yarn-project/ethereum/src/deploy_aztec_l1_contracts.ts @@ -31,9 +31,9 @@ const logger = createLogger('ethereum:deploy_aztec_l1_contracts'); const JSON_DEPLOY_RESULT_PREFIX = 'JSON DEPLOY RESULT:'; /** - * Runs a process with the given command, arguments, and environment. - * If the process outputs a line starting with JSON_DEPLOY_RESULT_PREFIX, - * the JSON is parsed and returned. + * Runs a process and parses JSON deploy results from stdout. + * Lines starting with JSON_DEPLOY_RESULT_PREFIX are parsed and returned. + * All other stdout goes to logger.info, stderr goes to logger.warn. */ function runProcess( command: string, @@ -49,26 +49,41 @@ function runProcess( }); let result: T | undefined; + let parseError: Error | undefined; + let settled = false; readline.createInterface({ input: proc.stdout }).on('line', line => { const trimmedLine = line.trim(); if (trimmedLine.startsWith(JSON_DEPLOY_RESULT_PREFIX)) { const jsonStr = trimmedLine.slice(JSON_DEPLOY_RESULT_PREFIX.length).trim(); - // TODO(AD): should this be a zod parse? - result = JSON.parse(jsonStr); + try { + result = JSON.parse(jsonStr); + } catch { + parseError = new Error(`Failed to parse deploy result JSON: ${jsonStr.slice(0, 200)}`); + } } else { logger.info(line); } }); - readline.createInterface({ input: proc.stderr }).on('line', logger.error.bind(logger)); + readline.createInterface({ input: proc.stderr }).on('line', logger.warn.bind(logger)); proc.on('error', error => { + if (settled) { + return; + } + settled = true; reject(new Error(`Failed to spawn ${command}: ${error.message}`)); }); proc.on('close', code => { + if (settled) { + return; + } + settled = true; if (code !== 0) { - reject(new Error(`${command} exited with code ${code}. See logs for details.\n`)); + reject(new Error(`${command} exited with code ${code}`)); + } else if (parseError) { + reject(parseError); } else { resolve(result); } @@ -321,11 +336,8 @@ export async function deployAztecL1Contracts( ); } - // From heuristic testing. More caused issues with anvil. - const MAGIC_ANVIL_BATCH_SIZE = 8; - // Anvil seems to stall with unbounded batch size. Otherwise no max batch size is desirable. + const scriptPath = join(getL1ContractsPath(), 'scripts', 'forge_broadcast.js'); const forgeArgs = [ - 'script', FORGE_SCRIPT, '--sig', 'run()', @@ -333,9 +345,6 @@ export async function deployAztecL1Contracts( privateKey, '--rpc-url', rpcUrl, - '--broadcast', - '--batch-size', - MAGIC_ANVIL_BATCH_SIZE.toString(), ...(shouldVerify ? ['--verify'] : []), ]; const forgeEnv = { @@ -344,7 +353,12 @@ export async function deployAztecL1Contracts( FOUNDRY_PROFILE: chainId === mainnet.id ? 'production' : undefined, ...getDeployAztecL1ContractsEnvVars(args), }; - const result = await runProcess('forge', forgeArgs, forgeEnv, l1ContractsPath); + const result = await runProcess( + process.execPath, + [scriptPath, ...forgeArgs], + forgeEnv, + l1ContractsPath, + ); if (!result) { throw new Error('Forge script did not output deployment result'); } @@ -587,17 +601,8 @@ export const deployRollupForUpgrade = async ( const FORGE_SCRIPT = 'script/deploy/DeployRollupForUpgrade.s.sol'; await maybeForgeForceProductionBuild(l1ContractsPath, FORGE_SCRIPT, chainId); - const forgeArgs = [ - 'script', - FORGE_SCRIPT, - '--sig', - 'run()', - '--private-key', - privateKey, - '--rpc-url', - rpcUrl, - '--broadcast', - ]; + const scriptPath = join(getL1ContractsPath(), 'scripts', 'forge_broadcast.js'); + const forgeArgs = [FORGE_SCRIPT, '--sig', 'run()', '--private-key', privateKey, '--rpc-url', rpcUrl]; const forgeEnv = { FOUNDRY_PROFILE: chainId === mainnet.id ? 'production' : undefined, // Env vars required by l1-contracts/script/deploy/RollupConfiguration.sol. @@ -606,7 +611,12 @@ export const deployRollupForUpgrade = async ( ...getDeployRollupForUpgradeEnvVars(args), }; - const result = await runProcess('forge', forgeArgs, forgeEnv, l1ContractsPath); + const result = await runProcess( + process.execPath, + [scriptPath, ...forgeArgs], + forgeEnv, + l1ContractsPath, + ); if (!result) { throw new Error('Forge script did not output deployment result'); } diff --git a/yarn-project/l1-artifacts/scripts/copy-foundry-artifacts.sh b/yarn-project/l1-artifacts/scripts/copy-foundry-artifacts.sh index 8607c60dee9b..cb2f03b42cc5 100755 --- a/yarn-project/l1-artifacts/scripts/copy-foundry-artifacts.sh +++ b/yarn-project/l1-artifacts/scripts/copy-foundry-artifacts.sh @@ -23,7 +23,10 @@ cp -rp "$src/script/deploy" "l1-contracts/script/" # only deploy/, other script mkdir -p "l1-contracts/test/script" cp -p "$src/test/shouting.t.sol" "l1-contracts/test/" cp -p "$src"/test/script/*.sol "l1-contracts/test/script/" -cp -p "$src"/{foundry.toml,foundry.lock,solc-*} "l1-contracts/" +cp -p "$src"/{foundry.toml,foundry.lock,package.json,solc-*} "l1-contracts/" +# Copy the forge broadcast wrapper (now a plain .js source file). +mkdir -p "l1-contracts/scripts" +cp -p "$src/scripts/forge_broadcast.js" "l1-contracts/scripts/" abs_dest=$(pwd)/l1-contracts # Keep only the foundry relevant files from lib (cd "$src" && find lib \( -name "*.sol" -o -name "remappings.txt" -o -name "foundry.toml" \) -exec cp --parents -t "$abs_dest" {} +) From d09b5f380f04f058969139b30b3163da960b7bc2 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 11 Feb 2026 10:53:20 +0000 Subject: [PATCH 07/27] feat(p2p): slot-based soft deletion for TxPoolV2 Deleted transactions (eviction, validation failure, failed execution) are now slot-soft-deleted instead of immediately hard-deleted. They remain in the database for the rest of the current slot so other nodes can still fetch them via reqresp, then are cleaned up when prepareForSlot advances to a new slot. This coexists with the existing prune-based soft deletion which keeps txs from pruned blocks until their original mined block is finalized. Co-Authored-By: Claude Opus 4.6 --- .../p2p/src/mem_pools/tx_pool_v2/README.md | 68 ++-- .../mem_pools/tx_pool_v2/deleted_pool.test.ts | 171 +++++++++- .../src/mem_pools/tx_pool_v2/deleted_pool.ts | 113 +++++-- .../tx_pool_v2/tx_pool_v2.compat.test.ts | 10 +- .../mem_pools/tx_pool_v2/tx_pool_v2.test.ts | 315 +++++++++++++++--- .../mem_pools/tx_pool_v2/tx_pool_v2_impl.ts | 4 + 6 files changed, 578 insertions(+), 103 deletions(-) diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/README.md b/yarn-project/p2p/src/mem_pools/tx_pool_v2/README.md index c69e5ef8e9c6..3efd91ab7a3e 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/README.md +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/README.md @@ -30,19 +30,20 @@ TxPoolV2 manages transactions through a state machine with clear transitions: └─────────────────────────────────────┘ (reorg) │ │ │ handleFinalizedBlock() │ eviction after reorg + │ / eviction / failed exec │ (validation failure) ▼ ▼ ┌─────────────────────────────────────┐ ┌─────────────────────────────────────┐ -│ DELETED │ │ SOFT-DELETED │ -│ (hard-deleted or archived) │ │ (kept in DB for debugging) │ +│ SLOT-SOFT-DELETED │ │ PRUNE-SOFT-DELETED │ +│ (kept in DB until next slot) │ │ (kept in DB until finalized) │ +└─────────────────────────────────────┘ └─────────────────────────────────────┘ + │ │ + │ prepareForSlot() │ handleFinalizedBlock() + │ (slot advanced) │ (mined block finalized) + ▼ ▼ +┌─────────────────────────────────────┐ ┌─────────────────────────────────────┐ +│ HARD-DELETED │ │ HARD-DELETED │ +│ (permanently removed from DB) │ │ (permanently removed from DB) │ └─────────────────────────────────────┘ └─────────────────────────────────────┘ - │ - │ handleFinalizedBlock() - │ (mined block finalized) - ▼ - ┌─────────────────────────────────────┐ - │ HARD-DELETED │ - │ (permanently removed from DB) │ - └─────────────────────────────────────┘ ``` ## Key Components @@ -62,13 +63,11 @@ Core implementation containing: ### DeletedPool (`deleted_pool.ts`) -Manages soft deletion of transactions from pruned blocks: -- When a reorg (chain prune) occurs, transactions from pruned blocks are tracked with their original mined block number -- When these transactions are later evicted (e.g., failed validation, nullifier conflict), they are "soft-deleted" instead of removed -- Soft-deleted transactions remain in the database for debugging and potential resubmission -- When the original mined block is finalized on the new chain, soft-deleted transactions are permanently hard-deleted +Manages all transaction deletions in the pool with two soft-deletion mechanisms: +- **Slot-based**: Non-pruned txs are kept in DB until the next slot, allowing other nodes to fetch them via reqresp +- **Prune-based**: Txs from pruned blocks are kept in DB until their original mined block is finalized -This ensures transactions from reorged blocks are kept around until we're certain they won't be needed. +All deletions go through `DeletedPool.deleteTx()`, which routes to the appropriate path based on whether the tx is tracked as being from a pruned block. ### TxMetaData (`tx_metadata.ts`) @@ -86,27 +85,40 @@ Lightweight metadata stored alongside each transaction: State is derived by TxPoolIndices: - `mined` if `minedL2BlockId` is set - `protected` if in protection map -- `deleted` if soft-deleted (from a pruned block, evicted but kept in DB) +- `deleted` if soft-deleted (slot-based or prune-based, evicted but kept in DB) - `pending` otherwise ## Soft Deletion -When a chain reorganization occurs, transactions that were mined in pruned blocks are handled specially: +Deleted transactions are kept in the database for a grace period before being permanently removed. There are two soft-deletion mechanisms: + +### Slot-Based Soft Deletion + +When a transaction is deleted from the pool (eviction, validation failure, failed execution) and is **not** from a pruned block, it is "slot-soft-deleted": + +1. **Soft Delete**: The tx is removed from indices but kept in the database, tagged with the current slot number +2. **Retrieval**: Slot-soft-deleted txs can still be retrieved via `getTxByHash` and return status `'deleted'` from `getTxStatus` +3. **Hard Delete**: When `prepareForSlot` advances to a new slot, txs deleted in earlier slots are permanently removed +4. **Re-addition**: If a slot-soft-deleted tx is re-added to the pool, the slot-deleted tracking is cleared + +This allows other nodes to still fetch recently-deleted transactions via reqresp during the current slot. + +### Prune-Based Soft Deletion + +When a chain reorganization occurs, transactions that were mined in pruned blocks are handled with longer retention: 1. **Tracking**: When `handlePrunedBlocks` is called, all un-mined transactions are tracked by their original mined block number -2. **Soft Delete**: If these transactions are later evicted (failed validation, nullifier conflict, etc.), they are "soft-deleted" - removed from indices but kept in the database -3. **Retrieval**: Soft-deleted transactions can still be retrieved via `getTxByHash` and `hasTxs`, and return status `'deleted'` from `getTxStatus` +2. **Soft Delete**: If these transactions are later evicted (failed validation, nullifier conflict, etc.), they are "prune-soft-deleted" - removed from indices but kept in the database +3. **Retrieval**: Prune-soft-deleted txs can still be retrieved via `getTxByHash` and return status `'deleted'` from `getTxStatus` 4. **Hard Delete**: When `handleFinalizedBlock` is called and the finalized block number reaches or exceeds the transaction's original mined block, the transaction is permanently removed +5. **Re-addition**: If a prune-soft-deleted tx is re-added, the `softDeleted` flag is reset to `false` but the prune tracking is preserved, so a subsequent deletion still uses the prune path -This design allows: -- Debugging reorg scenarios by keeping transaction data available -- Potential resubmission of transactions that failed validation after a reorg -- Clean eventual cleanup once we're certain the transaction won't be needed +Prune-soft-deleted transactions are **not** affected by slot cleanup - they survive across slot boundaries until finalized. -**Example scenario:** +**Prune example:** 1. Tx mined at block 10 2. Chain prunes to block 5 (tx becomes un-mined, tracked as minedAtBlock=10) -3. Tx fails validation and is soft-deleted +3. Tx fails validation and is prune-soft-deleted 4. Block 9 finalized → tx still in DB (minedAtBlock=10 > finalized=9) 5. Block 10 finalized → tx hard-deleted (minedAtBlock=10 ≤ finalized=10) @@ -117,6 +129,10 @@ If the tx is re-mined at a higher block before being soft-deleted: 4. Block 10 finalized → tx still in DB 5. Block 15 finalized → tx hard-deleted +### Hydration + +On node restart, slot-soft-deleted transactions are immediately hard-deleted (they are stale by definition). Prune-soft-deleted transactions are loaded from the database and tracked normally. + ## Architecture: Pre-add vs Post-event Rules **Pre-add rules** (run during `addPendingTxs`): diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.test.ts index 23330d5c9c84..072c20614dc5 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.test.ts @@ -1,4 +1,4 @@ -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { createLogger } from '@aztec/foundation/log'; import type { AztecAsyncMap } from '@aztec/kv-store'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; @@ -120,27 +120,25 @@ describe('DeletedPool', () => { }); describe('deleteTx', () => { - it('soft-deletes tx from pruned block (keeps in DB)', async () => { + it('prune-soft-deletes tx from pruned block (keeps in DB)', async () => { await txsDB.set('tx1', Buffer.from('data1')); await pool.markFromPrunedBlock([{ txHash: 'tx1', minedAtBlock: BlockNumber(5) }]); expect(pool.isSoftDeleted('tx1')).toBe(false); - const result = await pool.deleteTx('tx1'); + await pool.deleteTx('tx1'); - expect(result).toBe('soft'); expect(pool.isSoftDeleted('tx1')).toBe(true); expect(await txsDB.getAsync('tx1')).toBeDefined(); // Still in DB }); - it('hard-deletes tx NOT from pruned block (removes from DB)', async () => { + it('slot-soft-deletes tx NOT from pruned block (keeps in DB)', async () => { await txsDB.set('tx1', Buffer.from('data1')); // tx1 is NOT marked as from pruned block - const result = await pool.deleteTx('tx1'); + await pool.deleteTx('tx1'); - expect(result).toBe('hard'); - expect(pool.isSoftDeleted('tx1')).toBe(false); - expect(await txsDB.getAsync('tx1')).toBeUndefined(); // Removed from DB + expect(pool.isSoftDeleted('tx1')).toBe(true); + expect(await txsDB.getAsync('tx1')).toBeDefined(); }); }); @@ -208,8 +206,7 @@ describe('DeletedPool', () => { expect(pool.isFromPrunedBlock('tx1')).toBe(true); // 3. Soft-delete the tx (e.g., due to eviction) - const result = await pool.deleteTx('tx1'); - expect(result).toBe('soft'); + await pool.deleteTx('tx1'); expect(pool.isSoftDeleted('tx1')).toBe(true); expect(await txsDB.getAsync('tx1')).toBeDefined(); // Still in DB @@ -372,6 +369,158 @@ describe('DeletedPool', () => { }); }); + describe('slot-based soft deletion', () => { + it('slot-soft-deletes tx NOT from pruned block (stays in DB, isSoftDeleted true)', async () => { + await txsDB.set('tx1', Buffer.from('data1')); + + await pool.deleteTx('tx1'); + + expect(pool.isSoftDeleted('tx1')).toBe(true); + expect(await txsDB.getAsync('tx1')).toBeDefined(); + }); + + it('prune-soft-delete takes precedence over slot-soft-delete', async () => { + await txsDB.set('tx1', Buffer.from('data1')); + await pool.markFromPrunedBlock([{ txHash: 'tx1', minedAtBlock: BlockNumber(5) }]); + + await pool.deleteTx('tx1'); + + // Should use prune path, not slot path + expect(pool.isSoftDeleted('tx1')).toBe(true); + expect(pool.isFromPrunedBlock('tx1')).toBe(true); + }); + + it('cleanupSlotDeleted hard-deletes txs from earlier slots', async () => { + await txsDB.set('tx1', Buffer.from('data1')); + await txsDB.set('tx2', Buffer.from('data2')); + + // Delete both txs at slot 0 (default) + await pool.deleteTx('tx1'); + await pool.deleteTx('tx2'); + expect(pool.isSoftDeleted('tx1')).toBe(true); + expect(pool.isSoftDeleted('tx2')).toBe(true); + + // Advance to slot 1 - should hard-delete both (deleted at slot 0 < 1) + await pool.cleanupSlotDeleted(SlotNumber(1)); + + expect(pool.isSoftDeleted('tx1')).toBe(false); + expect(pool.isSoftDeleted('tx2')).toBe(false); + expect(await txsDB.getAsync('tx1')).toBeUndefined(); + expect(await txsDB.getAsync('tx2')).toBeUndefined(); + }); + + it('cleanupSlotDeleted preserves txs from current slot', async () => { + // Advance to slot 5 first + await pool.cleanupSlotDeleted(SlotNumber(5)); + + await txsDB.set('tx1', Buffer.from('data1')); + await pool.deleteTx('tx1'); // Deleted at slot 5 + + // Cleanup at slot 5 - should NOT hard-delete (deleted at 5, not < 5) + await pool.cleanupSlotDeleted(SlotNumber(5)); + + expect(pool.isSoftDeleted('tx1')).toBe(true); + expect(await txsDB.getAsync('tx1')).toBeDefined(); + + // Cleanup at slot 6 - NOW should hard-delete (deleted at 5 < 6) + await pool.cleanupSlotDeleted(SlotNumber(6)); + + expect(pool.isSoftDeleted('tx1')).toBe(false); + expect(await txsDB.getAsync('tx1')).toBeUndefined(); + }); + + it('cleanupSlotDeleted does NOT affect prune-soft-deleted txs', async () => { + await txsDB.set('tx1', Buffer.from('data1')); + await pool.markFromPrunedBlock([{ txHash: 'tx1', minedAtBlock: BlockNumber(5) }]); + await pool.deleteTx('tx1'); // Prune-soft-deleted + + // Advance to slot 10 - should NOT affect prune-soft-deleted tx + await pool.cleanupSlotDeleted(SlotNumber(10)); + + expect(pool.isSoftDeleted('tx1')).toBe(true); + expect(await txsDB.getAsync('tx1')).toBeDefined(); + expect(pool.isFromPrunedBlock('tx1')).toBe(true); + }); + + it('clearSoftDeleted removes tx from slot-deleted tracking', async () => { + await txsDB.set('tx1', Buffer.from('data1')); + await pool.deleteTx('tx1'); + expect(pool.isSoftDeleted('tx1')).toBe(true); + + await pool.clearSoftDeleted('tx1'); + + expect(pool.isSoftDeleted('tx1')).toBe(false); + // Tx is still in DB (clearSoftDeleted doesn't delete the tx data) + expect(await txsDB.getAsync('tx1')).toBeDefined(); + }); + + it('clearSoftDeleted resets prune softDeleted flag but preserves prune tracking', async () => { + await txsDB.set('tx1', Buffer.from('data1')); + await pool.markFromPrunedBlock([{ txHash: 'tx1', minedAtBlock: BlockNumber(5) }]); + await pool.deleteTx('tx1'); + expect(pool.isSoftDeleted('tx1')).toBe(true); + expect(pool.isFromPrunedBlock('tx1')).toBe(true); + + await pool.clearSoftDeleted('tx1'); + + // No longer soft-deleted, but still tracked as from pruned block + expect(pool.isSoftDeleted('tx1')).toBe(false); + expect(pool.isFromPrunedBlock('tx1')).toBe(true); + }); + + it('clearSoftDeleted is a no-op for non-soft-deleted txs', async () => { + await txsDB.set('tx1', Buffer.from('data1')); + + // Should not throw or error + await pool.clearSoftDeleted('tx1'); + expect(pool.isSoftDeleted('tx1')).toBe(false); + }); + + it('hydration hard-deletes all slot-deleted txs', async () => { + await txsDB.set('tx1', Buffer.from('data1')); + await txsDB.set('tx2', Buffer.from('data2')); + + // Slot-delete both + await pool.deleteTx('tx1'); + await pool.deleteTx('tx2'); + expect(await txsDB.getAsync('tx1')).toBeDefined(); + expect(await txsDB.getAsync('tx2')).toBeDefined(); + + // Simulate restart + const pool2 = new DeletedPool(store, txsDB, createLogger('test2')); + await pool2.hydrateFromDatabase(); + + // Both should be hard-deleted after hydration + expect(await txsDB.getAsync('tx1')).toBeUndefined(); + expect(await txsDB.getAsync('tx2')).toBeUndefined(); + expect(pool2.isSoftDeleted('tx1')).toBe(false); + expect(pool2.isSoftDeleted('tx2')).toBe(false); + }); + + it('hydration does not affect prune-soft-deleted txs', async () => { + await txsDB.set('tx1', Buffer.from('data1')); + await txsDB.set('tx2', Buffer.from('data2')); + + // Prune-soft-delete tx1, slot-delete tx2 + await pool.markFromPrunedBlock([{ txHash: 'tx1', minedAtBlock: BlockNumber(5) }]); + await pool.deleteTx('tx1'); + await pool.deleteTx('tx2'); + + // Simulate restart + const pool2 = new DeletedPool(store, txsDB, createLogger('test2')); + await pool2.hydrateFromDatabase(); + + // tx1 (prune-soft-deleted) should still be in DB + expect(await txsDB.getAsync('tx1')).toBeDefined(); + expect(pool2.isSoftDeleted('tx1')).toBe(true); + expect(pool2.isFromPrunedBlock('tx1')).toBe(true); + + // tx2 (slot-deleted) should be hard-deleted + expect(await txsDB.getAsync('tx2')).toBeUndefined(); + expect(pool2.isSoftDeleted('tx2')).toBe(false); + }); + }); + describe('getCount and getPrunedTxHashes', () => { it('returns correct count', async () => { expect(pool.getCount()).toBe(0); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts index 8eeda41937fc..e4eea8793967 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts @@ -1,6 +1,6 @@ -import { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types'; import type { Logger } from '@aztec/foundation/log'; -import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store'; +import type { AztecAsyncKVStore, AztecAsyncMap, AztecAsyncSet } from '@aztec/kv-store'; /** * State stored for each transaction from a pruned block. @@ -39,15 +39,17 @@ function deserializeState(buffer: Buffer): DeletedTxState { * When a chain prune (reorg) happens, transactions from pruned blocks are tracked here. * This class is responsible for ALL deletion decisions: * - * - Transactions from pruned blocks are "soft deleted" - removed from indices but kept - * in the database for later re-execution - * - Transactions NOT from pruned blocks are "hard deleted" - completely removed from DB + * - Transactions from pruned blocks are "prune-soft-deleted" - removed from indices but kept + * in the database for later re-execution until their mined block is finalized + * - Transactions NOT from pruned blocks are "slot-soft-deleted" - kept in the database + * until the next slot, so other nodes can still fetch them via reqresp * - * When a block is finalized, soft-deleted transactions that were originally mined at or - * before that block number are permanently (hard) deleted. + * When a block is finalized, prune-soft-deleted transactions that were originally mined at or + * before that block number are permanently (hard) deleted. Slot-soft-deleted transactions + * are hard-deleted when `prepareForSlot` advances to a new slot. */ export class DeletedPool { - /** Persisted map: txHash -> DeletedTxState (serialized) */ + /** Persisted map: txHash -> DeletedTxState (serialized) - for prune-based soft deletions */ #deletedTxsDB: AztecAsyncMap; /** Reference to the main txs database for hard deletion */ @@ -56,16 +58,27 @@ export class DeletedPool { /** In-memory state for transactions from pruned blocks */ #state: Map = new Map(); + /** In-memory tracking: txHash -> slot at which the tx was deleted */ + #slotDeletedTxs: Map = new Map(); + + /** Persisted set tracking which txs are slot-deleted, for hydration cleanup. */ + #slotDeletedDB: AztecAsyncSet; + + /** Current slot number, updated by cleanupSlotDeleted */ + #currentSlot: SlotNumber = SlotNumber(0); + #log: Logger; constructor(store: AztecAsyncKVStore, txsDB: AztecAsyncMap, log: Logger) { this.#deletedTxsDB = store.openMap('deleted_txs'); + this.#slotDeletedDB = store.openSet('slot_deleted_txs'); this.#txsDB = txsDB; this.#log = log; } /** * Loads state from the database on startup. + * Slot-deleted txs are stale after restart and are immediately hard-deleted. */ async hydrateFromDatabase(): Promise { let prunedCount = 0; @@ -83,6 +96,18 @@ export class DeletedPool { if (prunedCount > 0 || softDeletedCount > 0) { this.#log.info(`Loaded ${prunedCount} txs from pruned blocks, ${softDeletedCount} soft-deleted`); } + + // Slot-deleted txs are stale after restart - hard-delete them all + let slotDeletedCount = 0; + for await (const txHash of this.#slotDeletedDB.entriesAsync()) { + await this.#txsDB.delete(txHash); + await this.#slotDeletedDB.delete(txHash); + slotDeletedCount++; + } + + if (slotDeletedCount > 0) { + this.#log.info(`Hard-deleted ${slotDeletedCount} stale slot-deleted txs on startup`); + } } /** @@ -123,27 +148,25 @@ export class DeletedPool { /** * Deletes a transaction. This is the single entry point for ALL deletions. + * The tx is always soft-deleted (kept in DB): * - * - If the tx is from a pruned block: soft-delete (keep in DB, mark as deleted) - * - If the tx is NOT from a pruned block: hard-delete (remove from DB) - * - * @returns 'soft' if soft-deleted, 'hard' if hard-deleted + * - If the tx is from a pruned block: prune-soft-delete (kept until finalized) + * - If the tx is NOT from a pruned block: slot-soft-delete (kept until next slot) */ - async deleteTx(txHash: string): Promise<'soft' | 'hard'> { + async deleteTx(txHash: string): Promise { const existing = this.#state.get(txHash); if (existing !== undefined) { - // Soft delete - keep in DB + // Prune-soft-delete - keep in DB until finalized const state: DeletedTxState = { minedAtBlock: existing.minedAtBlock, softDeleted: true, }; this.#state.set(txHash, state); await this.#deletedTxsDB.set(txHash, serializeState(state)); - return 'soft'; } else { - // Hard delete - remove from DB - await this.#txsDB.delete(txHash); - return 'hard'; + // Slot-soft-delete - keep in DB until next slot + this.#slotDeletedTxs.set(txHash, this.#currentSlot); + await this.#slotDeletedDB.add(txHash); } } @@ -176,10 +199,10 @@ export class DeletedPool { } /** - * Checks if a transaction is soft-deleted. + * Checks if a transaction is soft-deleted (either prune-based or slot-based). */ isSoftDeleted(txHash: string): boolean { - return this.#state.get(txHash)?.softDeleted ?? false; + return (this.#state.get(txHash)?.softDeleted ?? false) || this.#slotDeletedTxs.has(txHash); } /** @@ -218,6 +241,56 @@ export class DeletedPool { return toHardDelete; } + /** + * Cleans up slot-deleted transactions from previous slots. + * Called at the start of prepareForSlot. Updates #currentSlot and hard-deletes + * any txs that were deleted in an earlier slot. + */ + async cleanupSlotDeleted(currentSlot: SlotNumber): Promise { + const previousSlot = this.#currentSlot; + this.#currentSlot = currentSlot; + + const toHardDelete: string[] = []; + for (const [txHash, deletedAtSlot] of this.#slotDeletedTxs) { + if (deletedAtSlot < currentSlot) { + toHardDelete.push(txHash); + } + } + + if (toHardDelete.length === 0) { + return; + } + + for (const txHash of toHardDelete) { + this.#slotDeletedTxs.delete(txHash); + await this.#slotDeletedDB.delete(txHash); + await this.#txsDB.delete(txHash); + } + + this.#log.debug( + `Cleaned up ${toHardDelete.length} slot-deleted txs from slot ${previousSlot} (now slot ${currentSlot})`, + ); + } + + /** + * Clears soft-deletion status for a transaction being re-added to the pool. + * Removes slot-deleted tracking entirely, and resets the prune-soft-deleted flag + * while preserving the prune tracking itself (so a subsequent delete still uses + * the prune path). + */ + async clearSoftDeleted(txHash: string): Promise { + if (this.#slotDeletedTxs.has(txHash)) { + this.#slotDeletedTxs.delete(txHash); + await this.#slotDeletedDB.delete(txHash); + } + const existing = this.#state.get(txHash); + if (existing?.softDeleted) { + const state: DeletedTxState = { minedAtBlock: existing.minedAtBlock, softDeleted: false }; + this.#state.set(txHash, state); + await this.#deletedTxsDB.set(txHash, serializeState(state)); + } + } + /** * Gets the count of transactions from pruned blocks. */ diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.compat.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.compat.test.ts index 8b76a026d01a..7f0e52312729 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.compat.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.compat.test.ts @@ -161,10 +161,10 @@ describe('TxPoolV2 Compatibility Tests', () => { await pool.addPendingTxs([pendingTx, minedTx]); await pool.handleMinedBlock(makeBlock([minedTx], block1Header)); - // Delete a pending tx via handleFailedExecution - should be permanently deleted + // Delete a pending tx via handleFailedExecution - should be slot-soft-deleted await pool.handleFailedExecution([pendingTx.getTxHash()]); - expect(await pool.getTxByHash(pendingTx.getTxHash())).toBeUndefined(); - expect(await pool.getTxStatus(pendingTx.getTxHash())).toBeUndefined(); + expect(await pool.getTxByHash(pendingTx.getTxHash())).toBeDefined(); + expect(await pool.getTxStatus(pendingTx.getTxHash())).toBe('deleted'); expect(await pool.getPendingTxCount()).toEqual(0); }); @@ -305,8 +305,8 @@ describe('TxPoolV2 Compatibility Tests', () => { // Delete mined tx via finalization await pool.handleFinalizedBlock(block1Header); - // Verify mined tx is deleted - expect(await pool.getTxStatus(txs[0].getTxHash())).toBeUndefined(); + // Verify mined tx is deleted (slot-soft-deleted) + expect(await pool.getTxStatus(txs[0].getTxHash())).toBe('deleted'); // Verify remaining pending count expect(await pool.getPendingTxCount()).toBe(2); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts index 7b943c56d6b5..d226f41aa10f 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2.test.ts @@ -1043,7 +1043,7 @@ describe('TxPoolV2', () => { expect(toStrings(result.accepted)).toContain(hashOf(txPreProtected)); // txExisting was evicted by txNormal (normal eviction still works) - expect(await pool.getTxStatus(txExisting.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txExisting.getTxHash())).toBe('deleted'); expect(await pool.getTxStatus(txNormal.getTxHash())).toBe('pending'); expect(await pool.getTxStatus(txPreProtected.getTxHash())).toBe('protected'); }); @@ -1073,7 +1073,7 @@ describe('TxPoolV2', () => { // Pool should have: tx2 (200), txNormal (150), txPreProtected (1 - protected) // tx1 (100) was evicted to make room for txNormal - expect(await pool.getTxStatus(tx1.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx1.getTxHash())).toBe('deleted'); expect(await pool.getTxStatus(tx2.getTxHash())).toBe('pending'); expect(await pool.getTxStatus(txNormal.getTxHash())).toBe('pending'); expect(await pool.getTxStatus(txPreProtected.getTxHash())).toBe('protected'); @@ -1211,7 +1211,7 @@ describe('TxPoolV2', () => { // High priority tx should now be pending, low priority tx should be deleted expect(await pool.getTxStatus(txHigh.getTxHash())).toBe('pending'); - expect(await pool.getTxStatus(txLow.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txLow.getTxHash())).toBe('deleted'); expect(await pool.getPendingTxCount()).toBe(1); }); }); @@ -1356,7 +1356,7 @@ describe('TxPoolV2', () => { const pending = toStrings(await pool.getPendingTxHashes()); expect(pending).toHaveLength(1); expect(pending).toContain(hashOf(txProtected)); - expect(await pool.getTxStatus(txPending.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txPending.getTxHash())).toBe('deleted'); expectRemovedTxs(txPending); // txPending evicted due to nullifier conflict }); @@ -1382,7 +1382,7 @@ describe('TxPoolV2', () => { const pending = toStrings(await pool.getPendingTxHashes()); expect(pending).toHaveLength(1); expect(pending).toContain(hashOf(txPending)); - expect(await pool.getTxStatus(txProtected.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txProtected.getTxHash())).toBe('deleted'); expectRemovedTxs(txProtected); // txProtected deleted due to lower priority }); @@ -1405,8 +1405,8 @@ describe('TxPoolV2', () => { const pending = toStrings(await pool.getPendingTxHashes()); expect(pending).toHaveLength(1); expect(pending).toContain(hashOf(tx2)); // tx2 has fee=15, highest - expect(await pool.getTxStatus(tx1.getTxHash())).toBeUndefined(); - expect(await pool.getTxStatus(tx3.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx1.getTxHash())).toBe('deleted'); + expect(await pool.getTxStatus(tx3.getTxHash())).toBe('deleted'); expectRemovedTxs(tx1, tx3); // Lower priority txs deleted }); @@ -1434,8 +1434,8 @@ describe('TxPoolV2', () => { const pending = toStrings(await pool.getPendingTxHashes()); expect(pending).toHaveLength(1); expect(pending).toContain(hashOf(txProtected)); - expect(await pool.getTxStatus(txPending1.getTxHash())).toBeUndefined(); - expect(await pool.getTxStatus(txPending2.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txPending1.getTxHash())).toBe('deleted'); + expect(await pool.getTxStatus(txPending2.getTxHash())).toBe('deleted'); expectRemovedTxs(txPending1, txPending2); // Both evicted }); @@ -1463,7 +1463,7 @@ describe('TxPoolV2', () => { expect(pending).toHaveLength(2); expect(pending).toContain(hashOf(txPendingHigh)); expect(pending).toContain(hashOf(txPendingLow)); - expect(await pool.getTxStatus(txProtected.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txProtected.getTxHash())).toBe('deleted'); expectRemovedTxs(txProtected); // txProtected deleted }); }); @@ -1512,8 +1512,8 @@ describe('TxPoolV2', () => { const pending = toStrings(await pool.getPendingTxHashes()); expect(pending).toHaveLength(1); expect(pending).toContain(hashOf(txMined)); - // txPending was never mined, never in a pruned block, so it's hard-deleted - expect(await pool.getTxStatus(txPending.getTxHash())).toBeUndefined(); + // txPending was never mined, never in a pruned block, so it's slot-soft-deleted + expect(await pool.getTxStatus(txPending.getTxHash())).toBe('deleted'); expectRemovedTxs(txPending); // txPending evicted due to nullifier conflict }); @@ -1615,9 +1615,9 @@ describe('TxPoolV2', () => { const pending = toStrings(await pool.getPendingTxHashes()); expect(pending).toHaveLength(1); expect(pending).toContain(hashOf(txMined)); - // txPending1 and txPending2 were never mined, never in a pruned block, so hard-deleted - expect(await pool.getTxStatus(txPending1.getTxHash())).toBeUndefined(); - expect(await pool.getTxStatus(txPending2.getTxHash())).toBeUndefined(); + // txPending1 and txPending2 were never mined, never in a pruned block, so slot-soft-deleted + expect(await pool.getTxStatus(txPending1.getTxHash())).toBe('deleted'); + expect(await pool.getTxStatus(txPending2.getTxHash())).toBe('deleted'); expectRemovedTxs(txPending1, txPending2); // Both evicted }); @@ -1697,7 +1697,7 @@ describe('TxPoolV2', () => { // Unprotect - tx should be deleted due to validation failure await poolWithValidator.prepareForSlot(SlotNumber(2)); - expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBeUndefined(); + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('deleted'); expect(await poolWithValidator.getPendingTxCount()).toBe(0); }); @@ -1733,7 +1733,7 @@ describe('TxPoolV2', () => { await poolWithValidator.prepareForSlot(SlotNumber(2)); expect(await poolWithValidator.getTxStatus(txValid.getTxHash())).toBe('pending'); - expect(await poolWithValidator.getTxStatus(txInvalid.getTxHash())).toBeUndefined(); + expect(await poolWithValidator.getTxStatus(txInvalid.getTxHash())).toBe('deleted'); expect(await poolWithValidator.getTxStatus(txAlsoValid.getTxHash())).toBe('pending'); expect(await poolWithValidator.getPendingTxCount()).toBe(2); }); @@ -1832,7 +1832,7 @@ describe('TxPoolV2', () => { const pending = toStrings(await poolWithValidator.getPendingTxHashes()); expect(pending).toHaveLength(1); expect(pending).toContain(hashOf(txPending)); - expect(await poolWithValidator.getTxStatus(txProtected.getTxHash())).toBeUndefined(); + expect(await poolWithValidator.getTxStatus(txProtected.getTxHash())).toBe('deleted'); }); }); @@ -2270,7 +2270,7 @@ describe('TxPoolV2', () => { expect(await poolWithValidator.getTxStatus(txHigherPriority.getTxHash())).toBe('pending'); }); - it('tx not in pruned block that is deleted should be hard-deleted', async () => { + it('tx not in pruned block that is deleted should be slot-soft-deleted', async () => { const tx = await mockTx(1); // Add tx as pending (never mined, so never pruned) @@ -2287,9 +2287,9 @@ describe('TxPoolV2', () => { await poolWithValidator.addProtectedTxs([tx], slot1Header); await poolWithValidator.prepareForSlot(SlotNumber(2)); - // The tx was never in a pruned block, so it should be HARD-deleted - expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBeUndefined(); - expect(await poolWithValidator.getTxByHash(tx.getTxHash())).toBeUndefined(); + // The tx was never in a pruned block, so it should be slot-soft-deleted + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('deleted'); + expect(await poolWithValidator.getTxByHash(tx.getTxHash())).toBeDefined(); }); }); }); @@ -2302,7 +2302,7 @@ describe('TxPoolV2', () => { await pool.handleFailedExecution([tx.getTxHash()]); - expect(await pool.getTxStatus(tx.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); expect(await pool.getPendingTxCount()).toBe(0); expectRemovedTxs(tx); }); @@ -2339,8 +2339,8 @@ describe('TxPoolV2', () => { await pool.handleFinalizedBlock(slot1Header); - expect(await pool.getTxStatus(tx.getTxHash())).toBeUndefined(); - expect(await pool.getTxByHash(tx.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); + expect(await pool.getTxByHash(tx.getTxHash())).toBeDefined(); expectRemovedTxs(tx); // Now the tx is actually deleted }); @@ -2354,7 +2354,7 @@ describe('TxPoolV2', () => { await pool.handleFinalizedBlock(slot1Header); - expect(await pool.getTxStatus(tx.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); const archived = await pool.getArchivedTxByHash(tx.getTxHash()); expect(archived).toBeDefined(); expect(archived!.getTxHash().toString()).toEqual(hashOf(tx)); @@ -2380,7 +2380,7 @@ describe('TxPoolV2', () => { await pool.handleFinalizedBlock(slot1Header); expectRemovedTxs(tx); // Actually deleted - expect(await pool.getTxStatus(tx.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); }); it('pending -> protected -> pending (slot passed)', async () => { @@ -2429,7 +2429,7 @@ describe('TxPoolV2', () => { await pool.handleFinalizedBlock(slot1Header); expectRemovedTxs(tx); // Actually deleted - expect(await pool.getTxStatus(tx.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); }); it('N/A -> mined -> deleted (prover flow)', async () => { @@ -2441,7 +2441,7 @@ describe('TxPoolV2', () => { await pool.handleFinalizedBlock(slot1Header); expectRemovedTxs(tx); // Actually deleted - expect(await pool.getTxStatus(tx.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); }); }); @@ -2709,7 +2709,7 @@ describe('TxPoolV2', () => { expect(toStrings(result.accepted)).toContain(hashOf(txHigh)); expect(result.rejected).toHaveLength(0); expect(await pool.getPendingTxCount()).toBe(1); - expect(await pool.getTxStatus(txLow.getTxHash())).toBeUndefined(); // evicted + expect(await pool.getTxStatus(txLow.getTxHash())).toBe('deleted'); // evicted expect(await pool.getTxStatus(txHigh.getTxHash())).toBe('pending'); }); @@ -2827,7 +2827,7 @@ describe('TxPoolV2', () => { // txMed (higher priority) should remain pending expect(await pool.getTxStatus(txMed.getTxHash())).toBe('pending'); // txLow (lower priority) should be evicted due to insufficient balance - expect(await pool.getTxStatus(txLow.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txLow.getTxHash())).toBe('deleted'); }); it('evicts low-priority txs after CHAIN_PRUNED when balance is insufficient', async () => { @@ -3116,7 +3116,7 @@ describe('TxPoolV2', () => { await pool.handleMinedBlock(makeBlock([txToMine], slot1Header)); // txPending should be evicted (nullifier conflict with mined tx) - expect(await pool.getTxStatus(txPending.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txPending.getTxHash())).toBe('deleted'); // txToMine should be mined expect(await pool.getTxStatus(txToMine.getTxHash())).toBe('mined'); }); @@ -3143,7 +3143,7 @@ describe('TxPoolV2', () => { await pool.handleMinedBlock(makeBlock([txUnknown], slot1Header)); // txPending should be evicted because the block contains a conflicting nullifier - expect(await pool.getTxStatus(txPending.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txPending.getTxHash())).toBe('deleted'); // txUnknown should NOT be in the pool (we never added it) expect(await pool.getTxStatus(txUnknown.getTxHash())).toBeUndefined(); }); @@ -3166,8 +3166,8 @@ describe('TxPoolV2', () => { // Mine block with unknown txs - tx1 and tx2 should be evicted, tx3 should remain await pool.handleMinedBlock(makeBlock([unknownTx1, unknownTx2], slot1Header)); - expect(await pool.getTxStatus(tx1.getTxHash())).toBeUndefined(); - expect(await pool.getTxStatus(tx2.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx1.getTxHash())).toBe('deleted'); + expect(await pool.getTxStatus(tx2.getTxHash())).toBe('deleted'); expect(await pool.getTxStatus(tx3.getTxHash())).toBe('pending'); expect(await pool.getPendingTxCount()).toBe(1); }); @@ -3187,7 +3187,7 @@ describe('TxPoolV2', () => { await pool.handleMinedBlock(makeBlock([txUnknown], slot1Header)); // txPending should be evicted even though only the second nullifier conflicts - expect(await pool.getTxStatus(txPending.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txPending.getTxHash())).toBe('deleted'); }); it('does not evict protected txs when block contains conflicting nullifiers', async () => { @@ -3280,7 +3280,7 @@ describe('TxPoolV2', () => { await pool.addPendingTxs([tx4]); expect(await pool.getPendingTxCount()).toBe(3); - expect(await pool.getTxStatus(txs[0].getTxHash())).toBeUndefined(); // fee=10 evicted + expect(await pool.getTxStatus(txs[0].getTxHash())).toBe('deleted'); // fee=10 evicted expect(await pool.getTxStatus(tx4.getTxHash())).toBe('pending'); // fee=15 kept }); @@ -3395,8 +3395,8 @@ describe('TxPoolV2', () => { await pool.handleMinedBlock(makeBlock([tx], slot1Header)); await pool.handleFinalizedBlock(slot1Header); - expect(await pool.getTxStatus(tx.getTxHash())).toBeUndefined(); - expect(await pool.getTxByHash(tx.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); + expect(await pool.getTxByHash(tx.getTxHash())).toBeDefined(); }); it('handles duplicate handleMinedBlock calls', async () => { @@ -3836,7 +3836,7 @@ describe('TxPoolV2', () => { expect(result.ignored).toHaveLength(0); expect(result.rejected).toHaveLength(0); expect(await pool.getPendingTxCount()).toBe(2); - expect(await pool.getTxStatus(tx1.getTxHash())).toBeUndefined(); // evicted + expect(await pool.getTxStatus(tx1.getTxHash())).toBe('deleted'); // evicted expect(await pool.getTxStatus(tx2.getTxHash())).toBe('pending'); expect(await pool.getTxStatus(tx3.getTxHash())).toBe('pending'); }); @@ -3910,7 +3910,7 @@ describe('TxPoolV2', () => { expect(result.ignored).toHaveLength(0); expect(result.rejected).toHaveLength(0); expect(await pool.getPendingTxCount()).toBe(1); - expect(await pool.getTxStatus(txLow.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(txLow.getTxHash())).toBe('deleted'); expect(await pool.getTxStatus(txHigh.getTxHash())).toBe('pending'); }); @@ -4424,4 +4424,237 @@ describe('TxPoolV2', () => { expect(await pool.getMinedTxCount()).toBe(0); }); }); + + describe('slot-based soft deletion', () => { + it('deleted tx is retrievable and has deleted status within the same slot', async () => { + const tx = await mockTx(1); + await pool.addPendingTxs([tx]); + expectAddedTxs(tx); + + await pool.handleFailedExecution([tx.getTxHash()]); + expectRemovedTxs(tx); + + // Tx is soft-deleted: status is 'deleted' but still in DB + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); + expect(await pool.getTxByHash(tx.getTxHash())).toBeDefined(); + }); + + it('prepareForSlot hard-deletes txs from previous slots', async () => { + const tx = await mockTx(1); + await pool.addPendingTxs([tx]); + expectAddedTxs(tx); + + // Delete in slot 1 + await pool.prepareForSlot(SlotNumber(1)); + await pool.handleFailedExecution([tx.getTxHash()]); + expectRemovedTxs(tx); + + // Still retrievable in same slot + expect(await pool.getTxByHash(tx.getTxHash())).toBeDefined(); + + // Advance to slot 2 - should hard-delete + await pool.prepareForSlot(SlotNumber(2)); + + expect(await pool.getTxByHash(tx.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBeUndefined(); + }); + + it('prepareForSlot with same slot preserves current-slot deletions', async () => { + const tx = await mockTx(1); + await pool.addPendingTxs([tx]); + expectAddedTxs(tx); + + await pool.prepareForSlot(SlotNumber(1)); + await pool.handleFailedExecution([tx.getTxHash()]); + expectRemovedTxs(tx); + + // Call prepareForSlot again with same slot number + await pool.prepareForSlot(SlotNumber(1)); + + // Tx should still be retrievable (same slot) + expect(await pool.getTxByHash(tx.getTxHash())).toBeDefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); + }); + + it('evicted tx is retrievable until next slot', async () => { + // Setup pool with size limit of 1 + await pool.updateConfig({ maxPendingTxCount: 1 }); + + const tx1 = await mockTxWithFee(1, 10); + const tx2 = await mockTxWithFee(2, 20); + + await pool.prepareForSlot(SlotNumber(1)); + await pool.addPendingTxs([tx1]); + expectAddedTxs(tx1); + + // tx2 has higher fee, so tx1 gets evicted + await pool.addPendingTxs([tx2]); + expectAddedTxs(tx2); + expectRemovedTxs(tx1); + + // Evicted tx1 is still retrievable (slot-soft-deleted) + expect(await pool.getTxStatus(tx1.getTxHash())).toBe('deleted'); + expect(await pool.getTxByHash(tx1.getTxHash())).toBeDefined(); + + // Advance slot - tx1 should be hard-deleted + await pool.prepareForSlot(SlotNumber(2)); + expect(await pool.getTxByHash(tx1.getTxHash())).toBeUndefined(); + }); + + it('re-added tx after slot-soft-delete is not cleaned up by prepareForSlot', async () => { + const tx = await mockTx(1); + + await pool.prepareForSlot(SlotNumber(1)); + await pool.addPendingTxs([tx]); + expectAddedTxs(tx); + + // Delete in slot 1 + await pool.handleFailedExecution([tx.getTxHash()]); + expectRemovedTxs(tx); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); + + // Re-add while still soft deleted + await pool.addPendingTxs([tx]); + expectAddedTxs(tx); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('pending'); + + // Advance to slot 2 - tx should NOT be cleaned up since it was re-added + await pool.prepareForSlot(SlotNumber(2)); + + expect(await pool.getTxStatus(tx.getTxHash())).toBe('pending'); + expect(await pool.getTxByHash(tx.getTxHash())).toBeDefined(); + expect(await pool.getPendingTxCount()).toBe(1); + }); + + it('re-added tx after prune-soft-delete is not cleaned up by handleFinalizedBlock', async () => { + const tx = await mockTx(1); + + // Add, mine at block 1, prune + await pool.addPendingTxs([tx]); + expectAddedTxs(tx); + await pool.handleMinedBlock(makeBlock([tx], slot1Header)); + expectNoCallbacks(); + await pool.handlePrunedBlocks(block0Id); + + // Tx is restored to pending (valid by default) + expect(await pool.getTxStatus(tx.getTxHash())).toBe('pending'); + + // Delete the tx + await pool.handleFailedExecution([tx.getTxHash()]); + expectRemovedTxs(tx); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); + + // Re-add the tx + await pool.addPendingTxs([tx]); + expectAddedTxs(tx); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('pending'); + + // Finalize block 1 - should NOT delete the re-added tx + await pool.handleFinalizedBlock(slot1Header); + + expect(await pool.getTxStatus(tx.getTxHash())).toBe('pending'); + expect(await pool.getTxByHash(tx.getTxHash())).toBeDefined(); + expect(await pool.getPendingTxCount()).toBe(1); + }); + + it('re-added then re-deleted prune tx remains prune-soft-deleted until finalized', async () => { + const tx = await mockTx(1); + + // Add, mine at block 1, prune + await pool.addPendingTxs([tx]); + expectAddedTxs(tx); + await pool.handleMinedBlock(makeBlock([tx], slot1Header)); + expectNoCallbacks(); + await pool.handlePrunedBlocks(block0Id); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('pending'); + + // Delete, re-add, delete again + await pool.handleFailedExecution([tx.getTxHash()]); + expectRemovedTxs(tx); + await pool.addPendingTxs([tx]); + expectAddedTxs(tx); + await pool.handleFailedExecution([tx.getTxHash()]); + expectRemovedTxs(tx); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); + + // Advance slot - tx should survive because it's prune-soft-deleted, not slot-soft-deleted + await pool.prepareForSlot(SlotNumber(1)); + await pool.prepareForSlot(SlotNumber(2)); + await pool.prepareForSlot(SlotNumber(3)); + + // Still retrievable (prune-soft-deleted, not affected by slot cleanup) + expect(await pool.getTxByHash(tx.getTxHash())).toBeDefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); + + // Finalize block 1 - now the tx should be hard-deleted + await pool.handleFinalizedBlock(slot1Header); + + expect(await pool.getTxByHash(tx.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBeUndefined(); + }); + + it('prune-soft-deleted tx is not affected by slot cleanup', async () => { + const mockValidator = mock>(); + mockValidator.validateTx.mockResolvedValue({ result: 'valid' }); + const validatorStore = await openTmpStore('p2p-slot-prune'); + const validatorArchiveStore = await openTmpStore('archive-slot-prune'); + const poolWithValidator = new AztecKVTxPoolV2(validatorStore, validatorArchiveStore, { + l2BlockSource: mockL2BlockSource, + worldStateSynchronizer: mockWorldState, + createTxValidator: () => Promise.resolve(mockValidator), + }); + await poolWithValidator.start(); + + try { + const tx = await mockTx(1); + + // Add, mine, prune with rejection + await poolWithValidator.addPendingTxs([tx]); + await poolWithValidator.handleMinedBlock(makeBlock([tx], slot1Header)); + + mockValidator.validateTx.mockResolvedValue({ result: 'invalid', reason: ['expired'] }); + await poolWithValidator.handlePrunedBlocks(block0Id); + + // Tx is prune-soft-deleted + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('deleted'); + expect(await poolWithValidator.getTxByHash(tx.getTxHash())).toBeDefined(); + + // Advance many slots + await poolWithValidator.prepareForSlot(SlotNumber(5)); + await poolWithValidator.prepareForSlot(SlotNumber(10)); + + // Still present - prune deletions are not cleaned up by slot advancement + expect(await poolWithValidator.getTxByHash(tx.getTxHash())).toBeDefined(); + expect(await poolWithValidator.getTxStatus(tx.getTxHash())).toBe('deleted'); + + // Only finalization cleans it up + await poolWithValidator.handleFinalizedBlock(slot1Header); + expect(await poolWithValidator.getTxByHash(tx.getTxHash())).toBeUndefined(); + } finally { + await poolWithValidator.stop(); + await validatorStore.delete(); + await validatorArchiveStore.delete(); + } + }); + + it('finalized mined tx is slot-soft-deleted and cleaned next slot', async () => { + const tx = await mockTx(1); + await pool.addPendingTxs([tx]); + expectAddedTxs(tx); + await pool.handleMinedBlock(makeBlock([tx], slot1Header)); + + await pool.prepareForSlot(SlotNumber(1)); + await pool.handleFinalizedBlock(slot1Header); + expectRemovedTxs(tx); + + // Tx is slot-soft-deleted (was never pruned, so uses slot path) + expect(await pool.getTxStatus(tx.getTxHash())).toBe('deleted'); + expect(await pool.getTxByHash(tx.getTxHash())).toBeDefined(); + + // Advance slot - hard-deleted + await pool.prepareForSlot(SlotNumber(2)); + expect(await pool.getTxByHash(tx.getTxHash())).toBeUndefined(); + expect(await pool.getTxStatus(tx.getTxHash())).toBeUndefined(); + }); + }); }); diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts index 54bfb3b7ec0c..69908067396c 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts @@ -393,6 +393,9 @@ export class TxPoolV2Impl { } async prepareForSlot(slotNumber: SlotNumber): Promise { + // Step 0: Clean up slot-deleted txs from previous slots + await this.#deletedPool.cleanupSlotDeleted(slotNumber); + // Step 1: Find expired protected txs const expiredProtected = this.#indices.findExpiredProtectedTxs(slotNumber); @@ -631,6 +634,7 @@ export class TxPoolV2Impl { const meta = await buildTxMetaData(tx); await this.#txsDB.set(txHashStr, tx.toBuffer()); + await this.#deletedPool.clearSoftDeleted(txHashStr); this.#callbacks.onTxsAdded([tx], opts); if (state === 'pending') { From 70d6f1e7c0ad846d06ff9354d8a4f76d56089c98 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 11 Feb 2026 11:07:58 +0000 Subject: [PATCH 08/27] chore: benchmark tx val (#20227) add benchmarks around tx validators --- .../tx_validator/tx_validator_bench.test.ts | 450 ++++++++++++++++++ 1 file changed, 450 insertions(+) create mode 100644 yarn-project/p2p/src/msg_validators/tx_validator/tx_validator_bench.test.ts diff --git a/yarn-project/p2p/src/msg_validators/tx_validator/tx_validator_bench.test.ts b/yarn-project/p2p/src/msg_validators/tx_validator/tx_validator_bench.test.ts new file mode 100644 index 000000000000..6bf8b7588215 --- /dev/null +++ b/yarn-project/p2p/src/msg_validators/tx_validator/tx_validator_bench.test.ts @@ -0,0 +1,450 @@ +import { CONTRACT_CLASS_LOG_SIZE_IN_FIELDS, MAX_NULLIFIERS_PER_TX, NULLIFIER_SUBTREE_HEIGHT } from '@aztec/constants'; +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { padArrayEnd, times } from '@aztec/foundation/collection'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { Timer } from '@aztec/foundation/timer'; +import { ProtocolContractAddress } from '@aztec/protocol-contracts'; +import { computeFeePayerBalanceLeafSlot } from '@aztec/protocol-contracts/fee-juice'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; +import type { ContractDataSource } from '@aztec/stdlib/contract'; +import { GasFees } from '@aztec/stdlib/gas'; +import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server'; +import { LogHash } from '@aztec/stdlib/kernel'; +import { ContractClassLogFields } from '@aztec/stdlib/logs'; +import { makeSelector, mockTx, mockTxForRollup } from '@aztec/stdlib/testing'; +import { DatabasePublicStateSource, MerkleTreeId, PublicDataTreeLeaf } from '@aztec/stdlib/trees'; +import type { Tx } from '@aztec/stdlib/tx'; +import { NativeWorldStateService } from '@aztec/world-state'; + +import { jest } from '@jest/globals'; +import { type MockProxy, mock, mockFn } from 'jest-mock-extended'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { type RecordableHistogram, createHistogram } from 'node:perf_hooks'; + +import { ArchiveCache } from './archive_cache.js'; +import { BlockHeaderTxValidator } from './block_header_validator.js'; +import { DataTxValidator } from './data_validator.js'; +import { DoubleSpendTxValidator, type NullifierSource } from './double_spend_validator.js'; +import { GasTxValidator } from './gas_validator.js'; +import { MetadataTxValidator } from './metadata_validator.js'; +import { PhasesTxValidator } from './phases_validator.js'; +import { SizeTxValidator } from './size_validator.js'; +import { patchNonRevertibleFn } from './test_utils.js'; +import { TimestampTxValidator } from './timestamp_validator.js'; +import { TxPermittedValidator } from './tx_permitted_validator.js'; + +jest.setTimeout(300_000); + +const RUNS = 50; + +type BenchEntry = { + name: string; + histogram: RecordableHistogram; + unit: 'ms' | 'us'; +}; + +describe('TxValidator: Benchmarks', () => { + const entries: BenchEntry[] = []; + + // Pre-built txs and validators + let privateTx: Tx; + let publicTx: Tx; + let ccLogTx: Tx; + let metadataTx: Tx; + let timestampTx: Tx; + let permittedTx: Tx; + let sizeTx: Tx; + let doubleSpendTx: Tx; + let gasTx: Tx; + let phasesPrivateTx: Tx; + let phasesPublicTx: Tx; + let blockHeaderTx: Tx; + + let dataValidator: DataTxValidator; + let metadataValidator: MetadataTxValidator; + let timestampValidator: TimestampTxValidator; + let permittedValidator: TxPermittedValidator; + let sizeValidator: SizeTxValidator; + let doubleSpendValidator: DoubleSpendTxValidator; + let gasValidator: GasTxValidator; + let phasesValidator: PhasesTxValidator; + let blockHeaderValidator: BlockHeaderTxValidator; + let worldStateService: NativeWorldStateService; + + // Deterministic CC log fields for the poseidon2 microbenchmark + let ccLogFields: ContractClassLogFields; + + function makeEntry(name: string, unit: 'ms' | 'us'): BenchEntry { + const entry: BenchEntry = { name, histogram: createHistogram(), unit }; + entries.push(entry); + return entry; + } + + beforeAll(async () => { + // --- Build all mock txs --- + + // DataTxValidator: private tx (no public calls, no CC logs) + privateTx = await mockTxForRollup(1); + + // DataTxValidator: public tx + publicTx = await mockTx(2, { + numberOfNonRevertiblePublicCallRequests: 2, + numberOfRevertiblePublicCallRequests: 2, + hasPublicTeardownCallRequest: true, + }); + + // DataTxValidator: tx with CC log + ccLogTx = await mockTx(3, { + numberOfNonRevertiblePublicCallRequests: 2, + numberOfRevertiblePublicCallRequests: 2, + hasPublicTeardownCallRequest: true, + }); + ccLogFields = ContractClassLogFields.random(); + const logHash = await ccLogFields.hash(); + const scopedLogHash = LogHash.from({ value: logHash, length: CONTRACT_CLASS_LOG_SIZE_IN_FIELDS }).scope( + AztecAddress.fromNumber(1), + ); + ccLogTx.contractClassLogFields.push(ccLogFields); + ccLogTx.data.forPublic!.nonRevertibleAccumulatedData.contractClassLogsHashes[0] = scopedLogHash; + await ccLogTx.recomputeHash(); + + // MetadataTxValidator + const chainId = new Fr(1); + const rollupVersion = new Fr(1); + const vkTreeRoot = new Fr(100); + const protocolContractsHash = new Fr(200); + metadataTx = await mockTx(4, { chainId, version: rollupVersion, vkTreeRoot, protocolContractsHash }); + metadataValidator = new MetadataTxValidator({ + l1ChainId: chainId, + rollupVersion, + vkTreeRoot, + protocolContractsHash, + }); + + // TimestampTxValidator + timestampTx = await mockTx(5); + timestampTx.data.includeByTimestamp = BigInt(Math.floor(Date.now() / 1000)) + 3600n; + timestampValidator = new TimestampTxValidator({ + timestamp: BigInt(Math.floor(Date.now() / 1000)), + blockNumber: BlockNumber(3), + }); + + // TxPermittedValidator + permittedTx = await mockTx(6); + permittedValidator = new TxPermittedValidator(true); + + // SizeTxValidator + sizeTx = await mockTx(7); + sizeValidator = new SizeTxValidator(); + + // DoubleSpendTxValidator + doubleSpendTx = await mockTx(8); + gasTx = await mockTx(9); + + // Create real LMDB-backed world state with fee payer balance + const feePayerLeafSlot = await computeFeePayerBalanceLeafSlot(gasTx.data.feePayer); + const prefilledPublicData = [new PublicDataTreeLeaf(feePayerLeafSlot, new Fr(10n ** 18n))]; + worldStateService = await NativeWorldStateService.tmp(undefined, true, prefilledPublicData); + const merkleTree = worldStateService.getCommitted(); + + const nullifierSource: NullifierSource = { + nullifiersExist: async (nullifiers: Buffer[]) => { + const indices = await merkleTree.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, nullifiers); + return indices.map(index => index !== undefined); + }, + }; + doubleSpendValidator = new DoubleSpendTxValidator(nullifierSource); + + // GasTxValidator + gasValidator = new GasTxValidator( + new DatabasePublicStateSource(merkleTree), + ProtocolContractAddress.FeeJuice, + new GasFees(10, 10), + ); + + // PhasesTxValidator - private tx (early return) + phasesPrivateTx = await mockTxForRollup(10); + + // PhasesTxValidator - public tx with allowed setup + const allowedAddress = AztecAddress.fromNumber(999); + const allowedSelector = makeSelector(1); + phasesPublicTx = await mockTx(11, { numberOfNonRevertiblePublicCallRequests: 1 }); + await patchNonRevertibleFn(phasesPublicTx, 0, { address: allowedAddress, selector: allowedSelector }); + await phasesPublicTx.recomputeHash(); + + const contractDataSource: MockProxy = mock({ + getContract: mockFn().mockResolvedValue({ + currentContractClassId: Fr.random(), + originalContractClassId: Fr.random(), + }), + }); + phasesValidator = new PhasesTxValidator( + contractDataSource, + [{ address: allowedAddress, selector: allowedSelector }], + BigInt(Math.floor(Date.now() / 1000)), + ); + + // BlockHeaderTxValidator + blockHeaderTx = await mockTx(12); + blockHeaderTx.data.constants.anchorBlockHeader = worldStateService.getInitialHeader(); + await blockHeaderTx.recomputeHash(); + blockHeaderValidator = new BlockHeaderTxValidator(new ArchiveCache(merkleTree)); + + // DataTxValidator (stateless, single instance) + dataValidator = new DataTxValidator(); + + // --- Warmup: run each validator once to warm WASM/JIT --- + await ccLogFields.hash(); + await dataValidator.validateTx(privateTx); + await dataValidator.validateTx(publicTx); + await dataValidator.validateTx(ccLogTx); + await metadataValidator.validateTx(metadataTx); + await timestampValidator.validateTx(timestampTx); + await permittedValidator.validateTx(permittedTx); + await sizeValidator.validateTx(sizeTx); + await doubleSpendValidator.validateTx(doubleSpendTx); + await gasValidator.validateTx(gasTx); + await phasesValidator.validateTx(phasesPrivateTx); + await phasesValidator.validateTx(phasesPublicTx); + await blockHeaderValidator.validateTx(blockHeaderTx); + }); + + afterAll(async () => { + await worldStateService.close(); + + if (process.env.BENCH_OUTPUT) { + const data: any[] = []; + for (const { name, histogram, unit } of entries) { + data.push({ name: `TxValidator/${name}/avg`, value: histogram.mean, unit }); + data.push({ name: `TxValidator/${name}/p50`, value: histogram.percentile(50), unit }); + data.push({ name: `TxValidator/${name}/p95`, value: histogram.percentile(95), unit }); + } + + await fs.mkdir(path.dirname(process.env.BENCH_OUTPUT), { recursive: true }); + await fs.writeFile(process.env.BENCH_OUTPUT, JSON.stringify(data, null, 2)); + } else if (process.env.BENCH_OUTPUT_MD) { + await fs.mkdir(path.dirname(process.env.BENCH_OUTPUT_MD), { recursive: true }); + await using f = await fs.open(process.env.BENCH_OUTPUT_MD!, 'w'); + await f.write('|Validator|UNIT|MIN|AVG|P50|P95|MAX|\n'); + await f.write('|---------|----|---|---|---|---|---|\n'); + for (const { name, histogram, unit } of entries) { + await f.write( + `|${name}|${unit}|${histogram.min}|${histogram.mean}|${histogram.percentile(50)}|${histogram.percentile(95)}|${histogram.max}|\n`, + ); + } + } + }); + + it('poseidon2Hash of CONTRACT_CLASS_LOG_SIZE_IN_FIELDS fields', async () => { + const entry = makeEntry('poseidon2Hash-3023-fields', 'ms'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await ccLogFields.hash(); + entry.histogram.record(Math.max(1, Math.ceil(timer.ms()))); + } + }); + + it('DataTxValidator - private tx', async () => { + const entry = makeEntry('DataTxValidator-private/validateTx', 'ms'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await dataValidator.validateTx(privateTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.ms()))); + } + }); + + it('DataTxValidator - public tx', async () => { + const entry = makeEntry('DataTxValidator-public/validateTx', 'ms'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await dataValidator.validateTx(publicTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.ms()))); + } + }); + + it('DataTxValidator - tx with contract class log', async () => { + const entry = makeEntry('DataTxValidator-cc-log/validateTx', 'ms'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await dataValidator.validateTx(ccLogTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.ms()))); + } + }); + + it('MetadataTxValidator', async () => { + const entry = makeEntry('MetadataTxValidator/validateTx', 'us'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await metadataValidator.validateTx(metadataTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.us()))); + } + }); + + it('TimestampTxValidator', async () => { + const entry = makeEntry('TimestampTxValidator/validateTx', 'us'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await timestampValidator.validateTx(timestampTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.us()))); + } + }); + + it('TxPermittedValidator', async () => { + const entry = makeEntry('TxPermittedValidator/validateTx', 'us'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await permittedValidator.validateTx(permittedTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.us()))); + } + }); + + it('SizeTxValidator', async () => { + const entry = makeEntry('SizeTxValidator/validateTx', 'us'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await sizeValidator.validateTx(sizeTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.us()))); + } + }); + + it('DoubleSpendTxValidator', async () => { + const entry = makeEntry('DoubleSpendTxValidator/validateTx', 'us'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await doubleSpendValidator.validateTx(doubleSpendTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.us()))); + } + }); + + it('GasTxValidator', async () => { + const entry = makeEntry('GasTxValidator/validateTx', 'ms'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await gasValidator.validateTx(gasTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.ms()))); + } + }); + + it('PhasesTxValidator - private tx (early return)', async () => { + const entry = makeEntry('PhasesTxValidator-private/validateTx', 'us'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await phasesValidator.validateTx(phasesPrivateTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.us()))); + } + }); + + it('PhasesTxValidator - public tx with allowed setup', async () => { + const entry = makeEntry('PhasesTxValidator-public/validateTx', 'us'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await phasesValidator.validateTx(phasesPublicTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.us()))); + } + }); + + it('BlockHeaderTxValidator', async () => { + const entry = makeEntry('BlockHeaderTxValidator/validateTx', 'us'); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await blockHeaderValidator.validateTx(blockHeaderTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.us()))); + } + }); + + describe.each([100, 1_000, 10_000, 100_000])('DB fill rate: %d items', (dbSize: number) => { + let localWs: NativeWorldStateService; + let localFork: MerkleTreeWriteOperations; + let localBlockHeaderTx: Tx; + + beforeAll(async () => { + const primeTimer = new Timer(); + + // Create world state with fee payer balance only (initial tree size limits prefilled data) + const feePayerLeafSlot = await computeFeePayerBalanceLeafSlot(gasTx.data.feePayer); + localWs = await NativeWorldStateService.tmp(undefined, true, [ + new PublicDataTreeLeaf(feePayerLeafSlot, new Fr(10n ** 18n)), + ]); + localFork = await localWs.fork(); + + // Fill public data tree via fork + const publicDataLeaves = times(dbSize, i => + new PublicDataTreeLeaf(new Fr(BigInt(1000 + i)), new Fr(BigInt(i + 1))).toBuffer(), + ); + for (let start = 0; start < publicDataLeaves.length; start += MAX_NULLIFIERS_PER_TX) { + await localFork.batchInsert( + MerkleTreeId.PUBLIC_DATA_TREE, + publicDataLeaves.slice(start, start + MAX_NULLIFIERS_PER_TX), + 0, + ); + } + + // Fill nullifier tree in batches of MAX_NULLIFIERS_PER_TX, padded with Fr.ZERO + const nullifiers = times(dbSize, i => new Fr(BigInt(100000 + i))); + for (let start = 0; start < nullifiers.length; start += MAX_NULLIFIERS_PER_TX) { + const batch = nullifiers.slice(start, start + MAX_NULLIFIERS_PER_TX); + const padded = padArrayEnd(batch, Fr.ZERO, MAX_NULLIFIERS_PER_TX).map(n => n.toBuffer()); + await localFork.batchInsert(MerkleTreeId.NULLIFIER_TREE, padded, NULLIFIER_SUBTREE_HEIGHT); + } + + // Fill archive tree + await localFork.appendLeaves( + MerkleTreeId.ARCHIVE, + times(dbSize, i => new Fr(BigInt(200000 + i))), + ); + + // Record priming time + makeEntry(`DB-fill-${dbSize}/prime`, 'ms').histogram.record(Math.max(1, Math.ceil(primeTimer.ms()))); + + // BlockHeaderTx needs anchorBlockHeader matching the inner world state's initial header + localBlockHeaderTx = await mockTx(1000 + dbSize); + localBlockHeaderTx.data.constants.anchorBlockHeader = localWs.getInitialHeader(); + await localBlockHeaderTx.recomputeHash(); + }); + + afterAll(async () => { + await localFork?.close(); + await localWs?.close(); + }); + + it('DoubleSpendTxValidator', async () => { + const entry = makeEntry(`DoubleSpendTxValidator-db${dbSize}/validateTx`, 'us'); + const validator = new DoubleSpendTxValidator({ + nullifiersExist: async (ns: Buffer[]) => { + const indices = await localFork.findLeafIndices(MerkleTreeId.NULLIFIER_TREE, ns); + return indices.map(index => index !== undefined); + }, + }); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await validator.validateTx(doubleSpendTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.us()))); + } + }); + + it('GasTxValidator', async () => { + const entry = makeEntry(`GasTxValidator-db${dbSize}/validateTx`, 'ms'); + const validator = new GasTxValidator( + new DatabasePublicStateSource(localFork), + ProtocolContractAddress.FeeJuice, + new GasFees(10, 10), + ); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await validator.validateTx(gasTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.ms()))); + } + }); + + it('BlockHeaderTxValidator', async () => { + const entry = makeEntry(`BlockHeaderTxValidator-db${dbSize}/validateTx`, 'us'); + const validator = new BlockHeaderTxValidator(new ArchiveCache(localFork)); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await validator.validateTx(localBlockHeaderTx); + entry.histogram.record(Math.max(1, Math.ceil(timer.us()))); + } + }); + }); +}); From d94d912ed0f3ccf1e251af11415ce51c2815f7ea Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 11 Feb 2026 11:12:55 +0000 Subject: [PATCH 09/27] fix: cloudflare terraform API (#20387) Fix terraform API usage for R2 lifecycle rules --- spartan/terraform/cloudflare/main.tf | 39 +++++++++++++++++++--------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/spartan/terraform/cloudflare/main.tf b/spartan/terraform/cloudflare/main.tf index 4925a3ae535c..b1c12316ee9c 100644 --- a/spartan/terraform/cloudflare/main.tf +++ b/spartan/terraform/cloudflare/main.tf @@ -55,22 +55,37 @@ resource "cloudflare_r2_bucket_lifecycle" "cleanup" { rules = flatten([ for folder in local.top_level_folders : [ { - id = "delete-snapshots-${folder}" - enabled = true - conditions = { prefix = "${folder}/aztec" } - delete_objects_transition = { days = var.SNAPSHOT_RETENTION_DAYS } + id = "delete-snapshots-${folder}" + enabled = true + conditions = { prefix = "${folder}/aztec" } + delete_objects_transition = { + condition = { + max_age = var.SNAPSHOT_RETENTION_DAYS * 24 * 60 * 60 # Convert days to seconds + type = "Age" + } + } }, { - id = "delete-blobs-${folder}" - enabled = true - conditions = { prefix = "${folder}/blobs" } - delete_objects_transition = { days = var.BLOB_RETENTION_DAYS } + id = "delete-blobs-${folder}" + enabled = true + conditions = { prefix = "${folder}/blobs" } + delete_objects_transition = { + condition = { + max_age = var.BLOB_RETENTION_DAYS * 24 * 60 * 60 # Convert days to seconds + type = "Age" + } + } }, { - id = "delete-txs-${folder}" - enabled = true - conditions = { prefix = "${folder}/txs" } - delete_objects_transition = { days = var.TX_RETENTION_DAYS } + id = "delete-txs-${folder}" + enabled = true + conditions = { prefix = "${folder}/txs" } + delete_objects_transition = { + condition = { + max_age = var.TX_RETENTION_DAYS * 24 * 60 * 60 # Convert days to seconds + type = "Age" + } + } }, ] ]) From e99b3209f9648ec28a2a8c482d1237f0fb1724ed Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 11 Feb 2026 13:01:01 +0000 Subject: [PATCH 10/27] fix: HA e2e test order & retries (#20383) 2 fixes: - Run "should distribute work across multiple HA nodes" test last because it kills nodes - Update ci scripts to allow for option to kill docker-compose volumes so the test can be retried --- ci3/run_compose_test | 6 +- yarn-project/end-to-end/scripts/run_test.sh | 3 +- .../src/composed/ha/e2e_ha_full.test.ts | 230 +++++++++--------- .../end-to-end/src/fixtures/ha_setup.ts | 4 +- 4 files changed, 128 insertions(+), 115 deletions(-) diff --git a/ci3/run_compose_test b/ci3/run_compose_test index a303cca55852..105584084376 100755 --- a/ci3/run_compose_test +++ b/ci3/run_compose_test @@ -14,7 +14,11 @@ shutdown=0 trap 'shutdown=1' TERM INT down() { - docker compose -p "$name" down --remove-orphans --timeout 2 >/dev/null 2>&1 || true + local down_flags="--remove-orphans --timeout 2" + if [ "${REMOVE_COMPOSE_VOLUMES:-0}" -eq 1 ]; then + down_flags="-v $down_flags" + fi + docker compose -p "$name" down $down_flags >/dev/null 2>&1 || true } trap down EXIT diff --git a/yarn-project/end-to-end/scripts/run_test.sh b/yarn-project/end-to-end/scripts/run_test.sh index c9f2e2870bf8..f466b3d65a5e 100755 --- a/yarn-project/end-to-end/scripts/run_test.sh +++ b/yarn-project/end-to-end/scripts/run_test.sh @@ -24,6 +24,7 @@ case "$type" in TEST=$test exec run_compose_test $test end-to-end $PWD/web3signer ;; "ha") - TEST=$test exec run_compose_test $test end-to-end $PWD/ha + # Remove volumes on cleanup for HA tests to ensure clean database state on retries + TEST=$test REMOVE_COMPOSE_VOLUMES=1 exec run_compose_test $test end-to-end $PWD/ha ;; esac diff --git a/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts b/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts index d0cf585f608e..b00cebf7ac6f 100644 --- a/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts +++ b/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts @@ -49,7 +49,7 @@ const NODE_COUNT = 5; const VALIDATOR_COUNT = 4; const COMMITTEE_SIZE = 4; -describe.skip('HA Full Setup', () => { +describe('HA Full Setup', () => { jest.setTimeout(20 * 60 * 1000); // 20 minutes let logger: Logger; @@ -262,7 +262,12 @@ describe.skip('HA Full Setup', () => { afterEach(async () => { // Clean up database state between tests - await mainPool.query('DELETE FROM validator_duties'); + try { + await mainPool.query('DELETE FROM validator_duties'); + } catch (error) { + // Ignore cleanup errors (table might not exist on first run failure) + logger?.warn(`Failed to clean up validator_duties: ${error}`); + } }); it('should produce blocks with HA coordination and attestations', async () => { @@ -373,6 +378,117 @@ describe.skip('HA Full Setup', () => { } }); + it('should coordinate governance voting across HA nodes', async () => { + logger.info('Testing real governance voting with HA coordination'); + + const mockGovernancePayload = deployL1ContractsValues.l1ContractAddresses.governanceAddress; + logger.info(`Setting governance payload: ${mockGovernancePayload.toString()}`); + + // Configure all HA nodes to vote for this payload + for (let i = 0; i < NODE_COUNT; i++) { + await haNodeServices[i].setConfig({ + governanceProposerPayload: mockGovernancePayload, + }); + } + logger.info(`All ${NODE_COUNT} HA nodes configured to vote for governance payload`); + + // Send a transaction to trigger block building which will also trigger voting + logger.info('Sending transaction to trigger block building...'); + const deployer = new ContractDeployer(StatefulTestContractArtifact, wallet); + const receipt = await deployer.deploy(ownerAddress, ownerAddress, 42).send({ + from: ownerAddress, + contractAddressSalt: Fr.random(), + wait: { returnReceipt: true }, + }); + expect(receipt.blockNumber).toBeDefined(); + logger.info(`Transaction mined in block ${receipt.blockNumber}`); + + // Get the slot of the block that was just built + const [block] = await aztecNode.getCheckpointedBlocks(receipt.blockNumber!, 1); + if (!block) { + throw new Error(`Block ${receipt.blockNumber} not found`); + } + const blockSlot = block.block.header.globalVariables.slotNumber; + logger.info(`Block was built in slot ${blockSlot}`); + + // Compute round for governance voting from the block slot + const round = await governanceProposer.computeRound(blockSlot); + logger.info(`Block slot ${blockSlot}, governance round ${round}`); + + // Poll L1 for governance votes + logger.info('Polling L1 for governance votes...'); + const l1VoteCount = await retryUntil( + async () => { + const voteCount = Number( + await governanceProposer.getPayloadSignals( + deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(), + round, + mockGovernancePayload.toString(), + ), + ); + return voteCount > 0 ? voteCount : undefined; + }, + 'governance votes to appear on L1', + 6, // timeout in seconds (30 attempts * 200ms) + 0.2, // interval in seconds (200ms) + ); + logger.info(`Found ${l1VoteCount} governance vote(s) on L1 for payload ${mockGovernancePayload.toString()}`); + + // Verify votes were actually sent to L1 + expect(l1VoteCount).toBeGreaterThan(0); + logger.info(`Verified ${l1VoteCount} governance vote(s) successfully sent to L1`); + + // Get L1 round info to determine which slots have actually landed on L1. + // We anchor the comparison on L1's lastSignalSlot since: + // - The DB may have duties for future slots that haven't been published to L1 yet + // - L1 may have signals from earlier slots in the round before the governance payload was set in the DB + const roundInfo = await governanceProposer.getRoundInfo( + deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(), + round, + ); + const lastSignalSlot = Number(roundInfo.lastSignalSlot); + logger.info( + `L1 round ${round} info: lastSignalSlot=${lastSignalSlot}, l1VoteCount=${l1VoteCount}, payloadWithMostSignals=${roundInfo.payloadWithMostSignals}`, + ); + + // Query governance vote duties only for slots that have actually landed on L1 (up to lastSignalSlot) + const dbResult = await mainPool.query( + `SELECT * FROM validator_duties WHERE slot::numeric <= $1 AND duty_type = 'GOVERNANCE_VOTE' ORDER BY slot, started_at`, + [lastSignalSlot.toString()], + ); + const governanceVoteDuties = dbResult.rows; + logger.info( + `HA database shows ${governanceVoteDuties.length} governance vote duty(ies) up to slot ${lastSignalSlot}`, + ); + + if (governanceVoteDuties.length > 0) { + // Verify HA coordination: Only one duty per (slot, validator) should exist + const dutyKeys = governanceVoteDuties.map(row => `${row.slot}-${row.validator_address}`); + const uniqueDutyKeys = new Set(dutyKeys); + expect(uniqueDutyKeys.size).toBe(governanceVoteDuties.length); // No duplicate (slot, validator) pairs + + // All duties should be completed + for (const duty of governanceVoteDuties) { + logger.info( + ` Governance vote duty: slot ${duty.slot}, validator ${duty.validator_address}, node ${duty.node_id}, status ${duty.status}`, + ); + expect(duty.status).toBe(DutyStatus.SIGNED); + expect(duty.completed_at).toBeDefined(); + } + + // L1 votes should match the number of unique slots in the DB that have landed on L1 + const uniqueSlots = new Set(governanceVoteDuties.map(row => row.slot)); + logger.info( + `L1 vote count: ${l1VoteCount}, unique slots in DB with governance votes up to L1 lastSignalSlot: ${uniqueSlots.size} (slots: ${[...uniqueSlots].join(', ')})`, + ); + expect(l1VoteCount).toBe(uniqueSlots.size); + logger.info(`Verified L1 votes (${l1VoteCount}) === unique slots with votes (${uniqueSlots.size})`); + } + + logger.info('Governance voting with HA coordination and L1 verification complete'); + }); + + // NOTE: this test needs to run last it('should distribute work across multiple HA nodes', async () => { logger.info('Testing HA resilience by killing nodes after they produce blocks'); @@ -555,114 +671,4 @@ describe.skip('HA Full Setup', () => { } } }); - - it('should coordinate governance voting across HA nodes', async () => { - logger.info('Testing real governance voting with HA coordination'); - - const mockGovernancePayload = deployL1ContractsValues.l1ContractAddresses.governanceAddress; - logger.info(`Setting governance payload: ${mockGovernancePayload.toString()}`); - - // Configure all HA nodes to vote for this payload - for (let i = 0; i < NODE_COUNT; i++) { - await haNodeServices[i].setConfig({ - governanceProposerPayload: mockGovernancePayload, - }); - } - logger.info(`All ${NODE_COUNT} HA nodes configured to vote for governance payload`); - - // Send a transaction to trigger block building which will also trigger voting - logger.info('Sending transaction to trigger block building...'); - const deployer = new ContractDeployer(StatefulTestContractArtifact, wallet); - const receipt = await deployer.deploy(ownerAddress, ownerAddress, 42).send({ - from: ownerAddress, - contractAddressSalt: Fr.random(), - wait: { returnReceipt: true }, - }); - expect(receipt.blockNumber).toBeDefined(); - logger.info(`Transaction mined in block ${receipt.blockNumber}`); - - // Get the slot of the block that was just built - const [block] = await aztecNode.getCheckpointedBlocks(receipt.blockNumber!, 1); - if (!block) { - throw new Error(`Block ${receipt.blockNumber} not found`); - } - const blockSlot = block.block.header.globalVariables.slotNumber; - logger.info(`Block was built in slot ${blockSlot}`); - - // Compute round for governance voting from the block slot - const round = await governanceProposer.computeRound(blockSlot); - logger.info(`Block slot ${blockSlot}, governance round ${round}`); - - // Poll L1 for governance votes - logger.info('Polling L1 for governance votes...'); - const l1VoteCount = await retryUntil( - async () => { - const voteCount = Number( - await governanceProposer.getPayloadSignals( - deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(), - round, - mockGovernancePayload.toString(), - ), - ); - return voteCount > 0 ? voteCount : undefined; - }, - 'governance votes to appear on L1', - 6, // timeout in seconds (30 attempts * 200ms) - 0.2, // interval in seconds (200ms) - ); - logger.info(`Found ${l1VoteCount} governance vote(s) on L1 for payload ${mockGovernancePayload.toString()}`); - - // Verify votes were actually sent to L1 - expect(l1VoteCount).toBeGreaterThan(0); - logger.info(`Verified ${l1VoteCount} governance vote(s) successfully sent to L1`); - - // Get L1 round info to determine which slots have actually landed on L1. - // We anchor the comparison on L1's lastSignalSlot since: - // - The DB may have duties for future slots that haven't been published to L1 yet - // - L1 may have signals from earlier slots in the round before the governance payload was set in the DB - const roundInfo = await governanceProposer.getRoundInfo( - deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(), - round, - ); - const lastSignalSlot = Number(roundInfo.lastSignalSlot); - logger.info( - `L1 round ${round} info: lastSignalSlot=${lastSignalSlot}, l1VoteCount=${l1VoteCount}, payloadWithMostSignals=${roundInfo.payloadWithMostSignals}`, - ); - - // Query governance vote duties only for slots that have actually landed on L1 (up to lastSignalSlot) - const dbResult = await mainPool.query( - `SELECT * FROM validator_duties WHERE slot::numeric <= $1 AND duty_type = 'GOVERNANCE_VOTE' ORDER BY slot, started_at`, - [lastSignalSlot.toString()], - ); - const governanceVoteDuties = dbResult.rows; - logger.info( - `HA database shows ${governanceVoteDuties.length} governance vote duty(ies) up to slot ${lastSignalSlot}`, - ); - - if (governanceVoteDuties.length > 0) { - // Verify HA coordination: Only one duty per (slot, validator) should exist - const dutyKeys = governanceVoteDuties.map(row => `${row.slot}-${row.validator_address}`); - const uniqueDutyKeys = new Set(dutyKeys); - expect(uniqueDutyKeys.size).toBe(governanceVoteDuties.length); // No duplicate (slot, validator) pairs - - // All duties should be completed - for (const duty of governanceVoteDuties) { - logger.info( - ` Governance vote duty: slot ${duty.slot}, validator ${duty.validator_address}, node ${duty.node_id}, status ${duty.status}`, - ); - expect(duty.status).toBe(DutyStatus.SIGNED); - expect(duty.completed_at).toBeDefined(); - } - - // L1 votes should match the number of unique slots in the DB that have landed on L1 - const uniqueSlots = new Set(governanceVoteDuties.map(row => row.slot)); - logger.info( - `L1 vote count: ${l1VoteCount}, unique slots in DB with governance votes up to L1 lastSignalSlot: ${uniqueSlots.size} (slots: ${[...uniqueSlots].join(', ')})`, - ); - expect(l1VoteCount).toBe(uniqueSlots.size); - logger.info(`Verified L1 votes (${l1VoteCount}) === unique slots with votes (${uniqueSlots.size})`); - } - - logger.info('Governance voting with HA coordination and L1 verification complete'); - }); }); diff --git a/yarn-project/end-to-end/src/fixtures/ha_setup.ts b/yarn-project/end-to-end/src/fixtures/ha_setup.ts index 00d201851ca8..458c3342635d 100644 --- a/yarn-project/end-to-end/src/fixtures/ha_setup.ts +++ b/yarn-project/end-to-end/src/fixtures/ha_setup.ts @@ -69,8 +69,10 @@ export async function cleanupHADatabase(pool: Pool, logger?: Logger): Promise Date: Wed, 11 Feb 2026 13:57:32 +0000 Subject: [PATCH 11/27] fix: incorporate forge broadcast review feedback (#20390) --- .test_patterns.yml | 28 --------- l1-contracts/scripts/forge_broadcast.js | 79 +++++++++++-------------- 2 files changed, 33 insertions(+), 74 deletions(-) diff --git a/.test_patterns.yml b/.test_patterns.yml index fe848d520bf0..1acb5b490070 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -238,27 +238,6 @@ tests: owners: - *saleel - # Example http://ci.aztec-labs.com/18db8adf0db50928 - # This seems to be an error in acvm wasm/js - - regex: "run_compose_test [a-z]*-[a-z]* box boxes" - error_regex: "call_indirect to a null table entry" - owners: - - *adam - - # Example http://ci.aztec-labs.com/1752518c2215134c - # This seems to be a flake in how this is set up. TODO redo boxes. - - regex: "run_compose_test [a-z]*-[a-z]* box boxes" - error_regex: "expect.locator..toBeVisible.." - owners: - - *adam - - # Example http://ci.aztec-labs.com/cd76b19670ab613f - # related to wasm proving browser flakes potentially - - regex: "run_compose_test [a-z]*-[a-z]* box boxes" - error_regex: "Out of bounds memory access (evaluating 'this.exports" - owners: - - *adam - - regex: "run_compose_test vite-[a-z]* box boxes" error_regex: "Test timeout of [0-9]+ms exceeded." owners: @@ -330,13 +309,6 @@ tests: owners: - *palla - # http://ci.aztec-labs.com/963635a09039594f - # http://ci.aztec-labs.com/d3b12131764bce18 - - regex: "boxes/scripts/run_test.sh" - error_regex: "sending signal TERM to command" - owners: - - *adam - # http://ci.aztec-labs.com/e8228a36afda93b8 # Test passed but there was an error on stopping - regex: "playground/scripts/run_test.sh" diff --git a/l1-contracts/scripts/forge_broadcast.js b/l1-contracts/scripts/forge_broadcast.js index d8df4462617a..8d0e7e69d11d 100755 --- a/l1-contracts/scripts/forge_broadcast.js +++ b/l1-contracts/scripts/forge_broadcast.js @@ -34,8 +34,6 @@ import { spawn } from "node:child_process"; import { rmSync, writeSync } from "node:fs"; -import { request as httpRequest } from "node:http"; -import { request as httpsRequest } from "node:https"; // Chain IDs for timeout selection. const MAINNET_CHAIN_ID = 1; @@ -53,6 +51,11 @@ const MAX_RETRIES = parseInt( 10, ); +if (!Number.isSafeInteger(MAX_RETRIES)) { + process.stderr.write(`MAX_RETRIES is not a valid integer.\n`); + process.exit(1); +} + // Batch size of 8 prevents forge from hanging during broadcast. // See: https://github.com/foundry-rs/foundry/issues/6796 const BATCH_SIZE = 8; @@ -86,50 +89,29 @@ function extractVerifyFlag(args) { const RPC_TIMEOUT = 10_000; -/** JSON-RPC call using Node.js built-ins. Rejects on JSON-RPC errors and timeouts. */ -function rpcCall(rpcUrl, method, params) { - return new Promise((resolve, reject) => { - const url = new URL(rpcUrl); - const body = JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }); - const reqFn = url.protocol === "https:" ? httpsRequest : httpRequest; - - const timer = setTimeout(() => { - req.destroy(); - reject(new Error(`RPC call ${method} timed out after ${RPC_TIMEOUT}ms`)); - }, RPC_TIMEOUT); - - const req = reqFn( - url, - { method: "POST", headers: { "Content-Type": "application/json" } }, - (res) => { - let data = ""; - res.on("data", (chunk) => (data += chunk)); - res.on("end", () => { - clearTimeout(timer); - try { - const parsed = JSON.parse(data); - if (parsed.error) { - reject( - new Error( - `RPC error for ${method}: ${JSON.stringify(parsed.error)}`, - ), - ); - } else { - resolve(parsed.result); - } - } catch { - reject(new Error(`Bad RPC response: ${data.slice(0, 200)}`)); - } - }); - }, - ); - req.on("error", (err) => { - clearTimeout(timer); - reject(err); - }); - req.write(body); - req.end(); +/** JSON-RPC call using fetch. Rejects on JSON-RPC errors and timeouts. */ +async function rpcCall(rpcUrl, method, params) { + const body = JSON.stringify({ jsonrpc: "2.0", id: 1, method, params }); + const res = await fetch(rpcUrl, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body, + signal: AbortSignal.timeout(RPC_TIMEOUT), }); + if (!res.ok) { + throw new Error(`RPC HTTP ${res.status} for ${method}`); + } + const data = await res.text(); + let parsed; + try { + parsed = JSON.parse(data); + } catch { + throw new Error(`Bad RPC response for ${method}: ${data.slice(0, 200)}`); + } + if (parsed.error) { + throw new Error(`RPC error for ${method}: ${JSON.stringify(parsed.error)}`); + } + return parsed.result; } /** Detect if the RPC endpoint is an anvil dev node via web3_clientVersion. */ @@ -205,6 +187,11 @@ const TIMEOUT = process.env.FORGE_BROADCAST_TIMEOUT ? parseInt(process.env.FORGE_BROADCAST_TIMEOUT, 10) : getDefaultTimeout(chainId); +if (!Number.isSafeInteger(TIMEOUT)) { + process.stderr.write(`FORGE_BROADCAST_TIMEOUT is not a valid integer.\n`); + process.exit(1); +} + log( `chain_id=${chainId ?? "unknown"}, timeout=${TIMEOUT}s, max_retries=${MAX_RETRIES}, batch_size=${BATCH_SIZE}${wantsVerify ? ", verify=true (after broadcast)" : ""}`, ); @@ -301,7 +288,7 @@ for (let attempt = 1; attempt <= MAX_RETRIES; attempt++) { // - Forge computes new nonces from on-chain state // - New transactions replace any stuck ones with the same nonce // - The race condition is intermittent (~0.04%), so retries almost always succeed - rmSync("broadcast", { recursive: true, force: true }); + rmSync("broadcast", { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }); log( `Attempt ${attempt + 1}/${MAX_RETRIES + 1}: retrying from scratch (anvil)...`, From 420b14c3f0d15916725f89e13bbe378bf65211c5 Mon Sep 17 00:00:00 2001 From: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Date: Wed, 11 Feb 2026 14:02:53 +0000 Subject: [PATCH 12/27] refactor(p2p): rewrite FileStoreTxCollection with retry, backoff, and shared worker pool (#20317) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Rewrites `FileStoreTxCollection` with a shared worker pool, deadline-based expiry, and retry with round-robin and exponential backoff - Splits into two instances (fast/slow) with configurable worker counts and backoff settings via 6 new config fields - Extracts `getProofDeadlineForSlot` as a public free function from `SlowTxCollection` for reuse - Workers pull entries with priority (fewest attempts first), skip entries in backoff, and remove expired entries ## Test plan - All 10 `file_store_tx_collection.test.ts` tests pass (updated for new interface, added retry/backoff/deadline/clearPending tests) - All 22 `tx_collection.test.ts` tests pass (updated for two file store instances) - `yarn build` succeeds, `yarn format p2p` and `yarn lint p2p` clean - Tests use deterministic waits (`waitForSourceCalls`, `waitForTxsAdded`) instead of blind sleeps to avoid flakiness 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Alex Gherghisan Co-authored-by: Claude Opus 4.6 --- yarn-project/foundation/src/config/env_var.ts | 6 + .../p2p/src/services/tx_collection/config.ts | 42 +++ .../file_store_tx_collection.test.ts | 224 +++++++++++----- .../tx_collection/file_store_tx_collection.ts | 240 +++++++++++------- .../tx_collection/slow_tx_collection.ts | 13 +- .../tx_collection/tx_collection.test.ts | 3 +- .../services/tx_collection/tx_collection.ts | 52 +++- 7 files changed, 407 insertions(+), 173 deletions(-) diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index e5cc85f214de..7e92ce8255c6 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -251,6 +251,12 @@ export type EnvVar = | 'TX_COLLECTION_FILE_STORE_URLS' | 'TX_COLLECTION_FILE_STORE_SLOW_DELAY_MS' | 'TX_COLLECTION_FILE_STORE_FAST_DELAY_MS' + | 'TX_COLLECTION_FILE_STORE_FAST_WORKER_COUNT' + | 'TX_COLLECTION_FILE_STORE_SLOW_WORKER_COUNT' + | 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_BASE_MS' + | 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_BASE_MS' + | 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_MAX_MS' + | 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_MAX_MS' | 'TX_FILE_STORE_URL' | 'TX_FILE_STORE_UPLOAD_CONCURRENCY' | 'TX_FILE_STORE_MAX_QUEUE_SIZE' diff --git a/yarn-project/p2p/src/services/tx_collection/config.ts b/yarn-project/p2p/src/services/tx_collection/config.ts index 2c3d821bb440..f86950af78e8 100644 --- a/yarn-project/p2p/src/services/tx_collection/config.ts +++ b/yarn-project/p2p/src/services/tx_collection/config.ts @@ -37,6 +37,18 @@ export type TxCollectionConfig = { txCollectionFileStoreSlowDelayMs: number; /** Delay in ms before file store collection starts after fast collection is triggered */ txCollectionFileStoreFastDelayMs: number; + /** Number of concurrent workers for fast file store collection */ + txCollectionFileStoreFastWorkerCount: number; + /** Number of concurrent workers for slow file store collection */ + txCollectionFileStoreSlowWorkerCount: number; + /** Base backoff time in ms for fast file store collection retries */ + txCollectionFileStoreFastBackoffBaseMs: number; + /** Base backoff time in ms for slow file store collection retries */ + txCollectionFileStoreSlowBackoffBaseMs: number; + /** Max backoff time in ms for fast file store collection retries */ + txCollectionFileStoreFastBackoffMaxMs: number; + /** Max backoff time in ms for slow file store collection retries */ + txCollectionFileStoreSlowBackoffMaxMs: number; }; export const txCollectionConfigMappings: ConfigMappingsType = { @@ -121,4 +133,34 @@ export const txCollectionConfigMappings: ConfigMappingsType description: 'Delay before file store collection starts after fast collection', ...numberConfigHelper(2_000), }, + txCollectionFileStoreFastWorkerCount: { + env: 'TX_COLLECTION_FILE_STORE_FAST_WORKER_COUNT', + description: 'Number of concurrent workers for fast file store collection', + ...numberConfigHelper(5), + }, + txCollectionFileStoreSlowWorkerCount: { + env: 'TX_COLLECTION_FILE_STORE_SLOW_WORKER_COUNT', + description: 'Number of concurrent workers for slow file store collection', + ...numberConfigHelper(2), + }, + txCollectionFileStoreFastBackoffBaseMs: { + env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_BASE_MS', + description: 'Base backoff time in ms for fast file store collection retries', + ...numberConfigHelper(1_000), + }, + txCollectionFileStoreSlowBackoffBaseMs: { + env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_BASE_MS', + description: 'Base backoff time in ms for slow file store collection retries', + ...numberConfigHelper(5_000), + }, + txCollectionFileStoreFastBackoffMaxMs: { + env: 'TX_COLLECTION_FILE_STORE_FAST_BACKOFF_MAX_MS', + description: 'Max backoff time in ms for fast file store collection retries', + ...numberConfigHelper(5_000), + }, + txCollectionFileStoreSlowBackoffMaxMs: { + env: 'TX_COLLECTION_FILE_STORE_SLOW_BACKOFF_MAX_MS', + description: 'Max backoff time in ms for slow file store collection retries', + ...numberConfigHelper(30_000), + }, }; diff --git a/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.test.ts b/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.test.ts index fc3aaad52284..578a95dc0af9 100644 --- a/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.test.ts +++ b/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.test.ts @@ -1,6 +1,7 @@ import { BlockNumber } from '@aztec/foundation/branded-types'; import { createLogger } from '@aztec/foundation/log'; import { promiseWithResolvers } from '@aztec/foundation/promise'; +import { TestDateProvider } from '@aztec/foundation/timer'; import { L2Block } from '@aztec/stdlib/block'; import { Tx, TxHash } from '@aztec/stdlib/tx'; import { getTelemetryClient } from '@aztec/telemetry-client'; @@ -9,7 +10,7 @@ import { jest } from '@jest/globals'; import { type MockProxy, mock } from 'jest-mock-extended'; import type { TxPoolV2 } from '../../mem_pools/tx_pool_v2/interfaces.js'; -import { FileStoreTxCollection } from './file_store_tx_collection.js'; +import { type FileStoreCollectionConfig, FileStoreTxCollection } from './file_store_tx_collection.js'; import type { FileStoreTxSource } from './file_store_tx_source.js'; import { type TxAddContext, TxCollectionSink } from './tx_collection_sink.js'; @@ -19,6 +20,9 @@ describe('FileStoreTxCollection', () => { let txCollectionSink: TxCollectionSink; let txPool: MockProxy; let context: TxAddContext; + let dateProvider: TestDateProvider; + let deadline: Date; + let config: FileStoreCollectionConfig; let txs: Tx[]; let txHashes: TxHash[]; @@ -37,9 +41,9 @@ describe('FileStoreTxCollection', () => { }; const setFileStoreTxs = (source: MockProxy, txs: Tx[]) => { - source.getTxsByHash.mockImplementation(hashes => { - return Promise.resolve(hashes.map(h => txs.find(tx => tx.getTxHash().equals(h)))); - }); + source.getTxsByHash.mockImplementation(hashes => + Promise.resolve(hashes.map(h => txs.find(tx => tx.getTxHash().equals(h)))), + ); }; /** Waits for the sink to emit txs-added events for the expected number of txs. */ @@ -57,22 +61,44 @@ describe('FileStoreTxCollection', () => { return promise; }; + /** Waits until the total number of getTxsByHash calls across all sources reaches the expected count. */ + const waitForSourceCalls = async (sources: MockProxy[], totalCalls: number) => { + const start = Date.now(); + while (Date.now() - start < 60_000) { + const total = sources.reduce((sum, s) => sum + s.getTxsByHash.mock.calls.length, 0); + if (total >= totalCalls) { + return; + } + await new Promise(resolve => setTimeout(resolve, 5)); + } + const total = sources.reduce((sum, s) => sum + s.getTxsByHash.mock.calls.length, 0); + throw new Error(`Timed out waiting for ${totalCalls} source calls (got ${total})`); + }; + beforeEach(async () => { txPool = mock(); jest.spyOn(Math, 'random').mockReturnValue(0); + dateProvider = new TestDateProvider(); const log = createLogger('test'); txCollectionSink = new TxCollectionSink(txPool, getTelemetryClient(), log); fileStoreSources = [makeFileStoreSource('store1'), makeFileStoreSource('store2')]; - fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, log); + config = { + workerCount: 5, + backoffBaseMs: 1000, + backoffMaxMs: 5000, + }; + + fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, config, dateProvider, log); txs = await Promise.all([makeTx(), makeTx(), makeTx()]); txHashes = txs.map(tx => tx.getTxHash()); const block = await L2Block.random(BlockNumber(1)); context = { type: 'mined', block }; + deadline = new Date(dateProvider.now() + 60 * 60 * 1000); }); afterEach(async () => { @@ -80,121 +106,199 @@ describe('FileStoreTxCollection', () => { jest.restoreAllMocks(); }); - it('downloads txs immediately when startCollecting is called', async () => { + it('downloads txs when startCollecting is called', async () => { setFileStoreTxs(fileStoreSources[0], txs); fileStoreCollection.start(); - // Set up event listener before calling startCollecting const txsAddedPromise = waitForTxsAdded(txs.length); - - fileStoreCollection.startCollecting(txHashes, context); - - // Wait for all txs to be processed via events + fileStoreCollection.startCollecting(txHashes, context, deadline); await txsAddedPromise; expect(fileStoreSources[0].getTxsByHash).toHaveBeenCalled(); expect(txPool.addMinedTxs).toHaveBeenCalled(); }); - it('skips txs marked as found while queued', async () => { + it('skips txs marked as found', async () => { setFileStoreTxs(fileStoreSources[0], txs); fileStoreCollection.start(); - // Queue all txs, then mark the first as found before workers process it - fileStoreCollection.startCollecting(txHashes, context); + fileStoreCollection.startCollecting(txHashes, context, deadline); fileStoreCollection.foundTxs([txs[0]]); - // Set up event listener - only 2 txs should be downloaded const txsAddedPromise = waitForTxsAdded(2); - - // Wait for workers to process await txsAddedPromise; - // First tx should not have been requested from file store const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); const requestedHashes = allCalls.flat().flat(); expect(requestedHashes).not.toContainEqual(txHashes[0]); }); - it('stops tracking txs when foundTxs is called after queueing', async () => { - setFileStoreTxs(fileStoreSources[0], txs); - - fileStoreCollection.start(); - - // Queue all txs, then immediately mark first as found - fileStoreCollection.startCollecting(txHashes, context); - fileStoreCollection.foundTxs([txs[0]]); - - // Set up event listener - only 2 txs should be downloaded - const txsAddedPromise = waitForTxsAdded(2); - - // Wait for workers to process - await txsAddedPromise; - - // First tx should not have been requested from any file store - const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); - const requestedHashes = allCalls.flat().flat(); - expect(requestedHashes).not.toContainEqual(txHashes[0]); - - // Verify second and third tx were downloaded - expect(txPool.addMinedTxs).toHaveBeenCalled(); - }); - - it('tries multiple file stores when tx not found in first', async () => { + it('tries multiple file stores via round-robin', async () => { // Only second store has tx[0] setFileStoreTxs(fileStoreSources[1], [txs[0]]); - // Ensure we always start with source 0 so we can test the fallback to source 1 + // Pin random so we always start at source 0, ensuring we test the fallback to source 1 jest.spyOn(Math, 'random').mockReturnValue(0); fileStoreCollection.start(); - // Set up event listener const txsAddedPromise = waitForTxsAdded(1); - - fileStoreCollection.startCollecting([txHashes[0]], context); + fileStoreCollection.startCollecting([txHashes[0]], context, deadline); await txsAddedPromise; - // First store was tried but didn't have it + // Both stores should have been tried expect(fileStoreSources[0].getTxsByHash).toHaveBeenCalled(); - // Second store was tried and found it expect(fileStoreSources[1].getTxsByHash).toHaveBeenCalled(); expect(txPool.addMinedTxs).toHaveBeenCalled(); jest.restoreAllMocks(); }); - it('does not start workers if no file store sources are configured', async () => { + it('does not start workers if no file store sources are configured', () => { const log = createLogger('test'); - fileStoreCollection = new FileStoreTxCollection([], txCollectionSink, log); + fileStoreCollection = new FileStoreTxCollection([], txCollectionSink, config, dateProvider, log); fileStoreCollection.start(); - fileStoreCollection.startCollecting(txHashes, context); - - // Give some time for potential processing - await new Promise(resolve => setTimeout(resolve, 50)); + fileStoreCollection.startCollecting(txHashes, context, deadline); + // With no sources, start() is a no-op (no workers spawned) and startCollecting() returns + // immediately, so no calls should have been made synchronously. expect(fileStoreSources[0].getTxsByHash).not.toHaveBeenCalled(); }); it('does not re-queue txs that are already pending', async () => { - // Set txs on both sources so download count is deterministic regardless of random start index setFileStoreTxs(fileStoreSources[0], txs); setFileStoreTxs(fileStoreSources[1], txs); + // Use single worker for deterministic behavior + const log = createLogger('test'); + config = { workerCount: 1, backoffBaseMs: 1000, backoffMaxMs: 5000 }; + fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, config, dateProvider, log); + fileStoreCollection.start(); - // Set up event listener const txsAddedPromise = waitForTxsAdded(txs.length); - fileStoreCollection.startCollecting(txHashes, context); - fileStoreCollection.startCollecting(txHashes, context); // Duplicate call + fileStoreCollection.startCollecting(txHashes, context, deadline); + fileStoreCollection.startCollecting(txHashes, context, deadline); // Duplicate call await txsAddedPromise; - // Each tx should only be downloaded once (one source call per tx) + // With 1 worker processing sequentially, each tx should be found on the first source. + // Duplicate startCollecting should not create extra entries. const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); expect(allCalls.length).toBe(txHashes.length); }); + + it('retries across sources when tx is not found initially', async () => { + // Use a single worker to make behavior deterministic + const log = createLogger('test'); + config = { workerCount: 1, backoffBaseMs: 100, backoffMaxMs: 500 }; + fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, config, dateProvider, log); + + fileStoreCollection.start(); + + // Initially both sources return empty + fileStoreCollection.startCollecting([txHashes[0]], context, deadline); + + // Wait for first full cycle (2 sources = 2 calls) + await waitForSourceCalls(fileStoreSources, 2); + + // Now make second source return the tx + setFileStoreTxs(fileStoreSources[1], [txs[0]]); + + // Advance time past backoff so the worker retries + dateProvider.setTime(dateProvider.now() + 200); + + const txsAddedPromise = waitForTxsAdded(1); + await txsAddedPromise; + + expect(txPool.addMinedTxs).toHaveBeenCalled(); + }); + + it('expires entries past deadline', async () => { + const log = createLogger('test'); + config = { workerCount: 1, backoffBaseMs: 50, backoffMaxMs: 100 }; + fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, config, dateProvider, log); + + // Set a very short deadline + const shortDeadline = new Date(dateProvider.now() + 100); + + fileStoreCollection.start(); + fileStoreCollection.startCollecting([txHashes[0]], context, shortDeadline); + + // Wait for first full cycle (2 sources = 2 calls) + await waitForSourceCalls(fileStoreSources, 2); + + // Advance time past the deadline + dateProvider.setTime(dateProvider.now() + 200); + + // Clear mocks so we can distinguish new calls from old ones + jest.clearAllMocks(); + + // Add a new entry with a valid deadline and set up source to return it. + // This proves the worker is alive and the expired entry was cleaned up. + setFileStoreTxs(fileStoreSources[0], [txs[1]]); + const txsAddedPromise = waitForTxsAdded(1); + fileStoreCollection.startCollecting([txHashes[1]], context, deadline); + await txsAddedPromise; + + // Only txHashes[1] should have been requested after clearing mocks + const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); + const requestedHashes = allCalls.flat().flat(); + expect(requestedHashes).not.toContainEqual(txHashes[0]); + expect(requestedHashes).toContainEqual(txHashes[1]); + }); + + it('does not start collecting if deadline is in the past', () => { + const pastDeadline = new Date(dateProvider.now() - 1000); + + fileStoreCollection.start(); + fileStoreCollection.startCollecting(txHashes, context, pastDeadline); + + // startCollecting returns immediately without adding entries when deadline is past + expect(fileStoreSources[0].getTxsByHash).not.toHaveBeenCalled(); + }); + + it('foundTxs stops retry for found txs', async () => { + const log = createLogger('test'); + config = { workerCount: 1, backoffBaseMs: 50, backoffMaxMs: 100 }; + fileStoreCollection = new FileStoreTxCollection(fileStoreSources, txCollectionSink, config, dateProvider, log); + + setFileStoreTxs(fileStoreSources[0], [txs[1]]); + + fileStoreCollection.start(); + fileStoreCollection.startCollecting(txHashes, context, deadline); + + // Mark first tx as found + fileStoreCollection.foundTxs([txs[0]]); + + const txsAddedPromise = waitForTxsAdded(1); + await txsAddedPromise; + + // tx[0] should never have been attempted + const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); + const requestedHashes = allCalls.flat().flat(); + expect(requestedHashes).not.toContainEqual(txHashes[0]); + }); + + it('clearPending removes all entries', async () => { + fileStoreCollection.start(); + fileStoreCollection.startCollecting(txHashes, context, deadline); + fileStoreCollection.clearPending(); + + // Verify workers are alive but the cleared entries are gone by adding + // a new entry and confirming only it gets processed. + setFileStoreTxs(fileStoreSources[0], [txs[0]]); + const txsAddedPromise = waitForTxsAdded(1); + fileStoreCollection.startCollecting([txHashes[0]], context, deadline); + await txsAddedPromise; + + // Only the newly added tx[0] should have been requested, not all 3 original txs + const allCalls = fileStoreSources.flatMap(s => s.getTxsByHash.mock.calls); + const requestedHashes = allCalls.flat().flat(); + expect(requestedHashes).not.toContainEqual(txHashes[1]); + expect(requestedHashes).not.toContainEqual(txHashes[2]); + }); }); diff --git a/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.ts b/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.ts index 709238344a5d..baa5879ea46e 100644 --- a/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.ts +++ b/yarn-project/p2p/src/services/tx_collection/file_store_tx_collection.ts @@ -1,152 +1,198 @@ import { type Logger, createLogger } from '@aztec/foundation/log'; -import { FifoMemoryQueue } from '@aztec/foundation/queue'; +import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise'; +import { sleep } from '@aztec/foundation/sleep'; +import { DateProvider } from '@aztec/foundation/timer'; import { Tx, TxHash } from '@aztec/stdlib/tx'; import type { FileStoreTxSource } from './file_store_tx_source.js'; import type { TxAddContext, TxCollectionSink } from './tx_collection_sink.js'; -// Internal constants (not configurable by node operators) -const FILE_STORE_DOWNLOAD_CONCURRENCY = 5; // Max concurrent downloads +/** Configuration for a FileStoreTxCollection instance. */ +export type FileStoreCollectionConfig = { + workerCount: number; + backoffBaseMs: number; + backoffMaxMs: number; +}; + +type FileStoreTxEntry = { + txHash: string; + context: TxAddContext; + deadline: Date; + attempts: number; + lastAttemptTime: number; + nextSourceIndex: number; +}; /** * Collects txs from file stores as a fallback after P2P methods have been tried. - * Runs in parallel to slow/fast collection. The delay before starting file store - * collection is managed by the TxCollection orchestrator, not this class. + * Uses a shared worker pool that pulls entries with priority (fewest attempts first), + * retries with round-robin across sources, and applies exponential backoff between + * full cycles through all sources. */ export class FileStoreTxCollection { - /** Map from tx hash to add context for txs queued for download. */ - private pendingTxs = new Map(); + /** Map from tx hash string to entry for all pending downloads. */ + private entries = new Map(); - /** - * Tracks tx hashes found elsewhere, even before startCollecting is called. - * Needed because the orchestrator delays startCollecting via a real sleep, but foundTxs - * may arrive during that delay — before the hashes are added to pendingTxs. - */ - private foundTxHashes = new Set(); - - /** Queue of tx hashes to be downloaded. */ - private downloadQueue = new FifoMemoryQueue(); - - /** Worker promises for concurrent downloads. */ + /** Worker promises for the shared worker pool. */ private workers: Promise[] = []; - /** Whether the collection has been started. */ - private started = false; + /** Whether the worker pool is running. */ + private running = false; + + /** Signal used to wake sleeping workers when new entries arrive or stop is called. */ + private wakeSignal: PromiseWithResolvers; constructor( - private readonly fileStoreSources: FileStoreTxSource[], + private readonly sources: FileStoreTxSource[], private readonly txCollectionSink: TxCollectionSink, + private readonly config: FileStoreCollectionConfig, + private readonly dateProvider: DateProvider = new DateProvider(), private readonly log: Logger = createLogger('p2p:file_store_tx_collection'), - ) {} + ) { + this.wakeSignal = promiseWithResolvers(); + } - /** Starts the file store collection workers. */ - public start() { - if (this.fileStoreSources.length === 0) { - this.log.debug('No file store sources configured, skipping file store collection'); + /** Starts the shared worker pool. */ + public start(): void { + if (this.sources.length === 0) { + this.log.debug('No file store sources configured'); return; } - - this.started = true; - this.downloadQueue = new FifoMemoryQueue(); - - // Start concurrent download workers - for (let i = 0; i < FILE_STORE_DOWNLOAD_CONCURRENCY; i++) { - this.workers.push(this.downloadQueue.process(txHash => this.processDownload(txHash))); + this.running = true; + for (let i = 0; i < this.config.workerCount; i++) { + this.workers.push(this.workerLoop()); } - - this.log.info(`Started file store tx collection with ${this.fileStoreSources.length} sources`, { - sources: this.fileStoreSources.map(s => s.getInfo()), - concurrency: FILE_STORE_DOWNLOAD_CONCURRENCY, - }); } - /** Stops all collection activity. */ - public async stop() { - if (!this.started) { - return; - } - this.started = false; - this.downloadQueue.end(); + /** Stops all workers and clears state. */ + public async stop(): Promise { + this.running = false; + this.wake(); await Promise.all(this.workers); this.workers = []; - this.pendingTxs.clear(); - this.foundTxHashes.clear(); + this.entries.clear(); } - /** Remove the given tx hashes from pending. */ - public stopCollecting(txHashes: TxHash[]) { - for (const txHash of txHashes) { - const hashStr = txHash.toString(); - this.pendingTxs.delete(hashStr); + /** Adds entries to the shared map and wakes workers. */ + public startCollecting(txHashes: TxHash[], context: TxAddContext, deadline: Date): void { + if (this.sources.length === 0 || txHashes.length === 0) { + return; + } + if (+deadline <= this.dateProvider.now()) { + return; } - } - - /** Clears all pending state. Items already in the download queue will still be processed but won't be re-queued. */ - public clearPending() { - this.pendingTxs.clear(); - this.foundTxHashes.clear(); - } - /** Queue the given tx hashes for file store collection. */ - public startCollecting(txHashes: TxHash[], context: TxAddContext) { for (const txHash of txHashes) { const hashStr = txHash.toString(); - if (!this.pendingTxs.has(hashStr) && !this.foundTxHashes.has(hashStr)) { - this.pendingTxs.set(hashStr, context); - this.downloadQueue.put(txHash); + if (!this.entries.has(hashStr)) { + this.entries.set(hashStr, { + txHash: hashStr, + context, + deadline, + attempts: 0, + lastAttemptTime: 0, + nextSourceIndex: Math.floor(Math.random() * this.sources.length), + }); } } + this.wake(); } - /** Stop tracking txs that were found elsewhere. */ - public foundTxs(txs: Tx[]) { + /** Removes entries for txs that have been found elsewhere. */ + public foundTxs(txs: Tx[]): void { for (const tx of txs) { - const hashStr = tx.getTxHash().toString(); - this.pendingTxs.delete(hashStr); - this.foundTxHashes.add(hashStr); + this.entries.delete(tx.getTxHash().toString()); } } - /** Processes a single tx hash from the download queue. */ - private async processDownload(txHash: TxHash) { - const hashStr = txHash.toString(); - const context = this.pendingTxs.get(hashStr); - - // Skip if already found by another method - if (!context) { - return; - } - - await this.downloadTx(txHash, context); - this.pendingTxs.delete(hashStr); + /** Clears all pending entries. */ + public clearPending(): void { + this.entries.clear(); } - /** Attempt to download a tx from file stores (round-robin). */ - private async downloadTx(txHash: TxHash, context: TxAddContext) { - const startIndex = Math.floor(Math.random() * this.fileStoreSources.length); - for (let i = startIndex; i < startIndex + this.fileStoreSources.length; i++) { - const source = this.fileStoreSources[i % this.fileStoreSources.length]; + private async workerLoop(): Promise { + while (this.running) { + const action = this.getNextAction(); + if (action.type === 'sleep') { + await action.promise; + continue; + } + + const entry = action.entry; + const source = this.sources[entry.nextSourceIndex % this.sources.length]; + entry.nextSourceIndex++; + entry.attempts++; + entry.lastAttemptTime = this.dateProvider.now(); try { const result = await this.txCollectionSink.collect( hashes => source.getTxsByHash(hashes), - [txHash], - { - description: `file-store ${source.getInfo()}`, - method: 'file-store', - fileStore: source.getInfo(), - }, - context, + [TxHash.fromString(entry.txHash)], + { description: `file-store ${source.getInfo()}`, method: 'file-store', fileStore: source.getInfo() }, + entry.context, ); - if (result.txs.length > 0) { - return; + this.entries.delete(entry.txHash); } } catch (err) { - this.log.trace(`Failed to download tx ${txHash} from ${source.getInfo()}`, { err }); + this.log.trace(`Error downloading tx ${entry.txHash} from ${source.getInfo()}`, { err }); + } + } + } + + /** Single-pass scan: removes expired entries, finds the best ready entry, or computes sleep time. */ + private getNextAction(): { type: 'process'; entry: FileStoreTxEntry } | { type: 'sleep'; promise: Promise } { + const now = this.dateProvider.now(); + let best: FileStoreTxEntry | undefined; + let earliestReadyAt = Infinity; + + for (const [key, entry] of this.entries) { + if (+entry.deadline <= now) { + this.entries.delete(key); + continue; + } + const backoffMs = this.getBackoffMs(entry); + const readyAt = entry.lastAttemptTime + backoffMs; + if (readyAt > now) { + earliestReadyAt = Math.min(earliestReadyAt, readyAt); + continue; + } + if (!best || entry.attempts < best.attempts) { + best = entry; } } - this.log.trace(`Tx ${txHash} not found in any file store`); + if (best) { + return { type: 'process', entry: best }; + } + if (earliestReadyAt < Infinity) { + return { type: 'sleep', promise: this.sleepOrWake(earliestReadyAt - now) }; + } + return { type: 'sleep', promise: this.waitForWake() }; + } + + /** Computes backoff for an entry. Backoff applies after a full cycle through all sources. */ + private getBackoffMs(entry: FileStoreTxEntry): number { + const fullCycles = Math.floor(entry.attempts / this.sources.length); + if (fullCycles === 0) { + return 0; + } + return Math.min(this.config.backoffBaseMs * Math.pow(2, fullCycles - 1), this.config.backoffMaxMs); + } + + /** Resolves the current wake signal and creates a new one. */ + private wake(): void { + this.wakeSignal.resolve(); + this.wakeSignal = promiseWithResolvers(); + } + + /** Waits until the wake signal is resolved. */ + private async waitForWake(): Promise { + await this.wakeSignal.promise; + } + + /** Sleeps for the given duration or until the wake signal is resolved. */ + private async sleepOrWake(ms: number): Promise { + await Promise.race([sleep(ms), this.wakeSignal.promise]); } } diff --git a/yarn-project/p2p/src/services/tx_collection/slow_tx_collection.ts b/yarn-project/p2p/src/services/tx_collection/slow_tx_collection.ts index eb337e0a0778..6d264c0a8b5f 100644 --- a/yarn-project/p2p/src/services/tx_collection/slow_tx_collection.ts +++ b/yarn-project/p2p/src/services/tx_collection/slow_tx_collection.ts @@ -254,9 +254,14 @@ export class SlowTxCollection { /** Computes the proof submission deadline for a given slot, a tx mined in this slot is no longer interesting after this deadline */ private getDeadlineForSlot(slotNumber: SlotNumber): Date { - const epoch = getEpochAtSlot(slotNumber, this.constants); - const submissionEndEpoch = EpochNumber(epoch + this.constants.proofSubmissionEpochs); - const submissionEndTimestamp = getTimestampRangeForEpoch(submissionEndEpoch, this.constants)[1]; - return new Date(Number(submissionEndTimestamp) * 1000); + return getProofDeadlineForSlot(slotNumber, this.constants); } } + +/** Computes the proof submission deadline for a given slot. A tx mined in this slot is no longer interesting after this deadline. */ +export function getProofDeadlineForSlot(slotNumber: SlotNumber, constants: L1RollupConstants): Date { + const epoch = getEpochAtSlot(slotNumber, constants); + const submissionEndEpoch = EpochNumber(epoch + constants.proofSubmissionEpochs); + const submissionEndTimestamp = getTimestampRangeForEpoch(submissionEndEpoch, constants)[1]; + return new Date(Number(submissionEndTimestamp) * 1000); +} diff --git a/yarn-project/p2p/src/services/tx_collection/tx_collection.test.ts b/yarn-project/p2p/src/services/tx_collection/tx_collection.test.ts index abd6ae020af1..8cb04633dbb5 100644 --- a/yarn-project/p2p/src/services/tx_collection/tx_collection.test.ts +++ b/yarn-project/p2p/src/services/tx_collection/tx_collection.test.ts @@ -580,6 +580,7 @@ class TestFastTxCollection extends FastTxCollection { class TestTxCollection extends TxCollection { declare slowCollection: SlowTxCollection; declare fastCollection: TestFastTxCollection; - declare fileStoreCollection: TxCollection['fileStoreCollection']; + declare fileStoreSlowCollection: TxCollection['fileStoreSlowCollection']; + declare fileStoreFastCollection: TxCollection['fileStoreFastCollection']; declare handleTxsAddedToPool: TxPoolV2Events['txs-added']; } diff --git a/yarn-project/p2p/src/services/tx_collection/tx_collection.ts b/yarn-project/p2p/src/services/tx_collection/tx_collection.ts index 6d142fbba25e..eae579e2a555 100644 --- a/yarn-project/p2p/src/services/tx_collection/tx_collection.ts +++ b/yarn-project/p2p/src/services/tx_collection/tx_collection.ts @@ -18,7 +18,7 @@ import type { TxCollectionConfig } from './config.js'; import { FastTxCollection } from './fast_tx_collection.js'; import { FileStoreTxCollection } from './file_store_tx_collection.js'; import type { FileStoreTxSource } from './file_store_tx_source.js'; -import { SlowTxCollection } from './slow_tx_collection.js'; +import { SlowTxCollection, getProofDeadlineForSlot } from './slow_tx_collection.js'; import { type TxAddContext, TxCollectionSink } from './tx_collection_sink.js'; import type { TxSource } from './tx_source.js'; @@ -56,8 +56,11 @@ export class TxCollection { /** Fast collection methods */ protected readonly fastCollection: FastTxCollection; - /** File store collection */ - protected readonly fileStoreCollection: FileStoreTxCollection; + /** File store collection for slow (mined block) path */ + protected readonly fileStoreSlowCollection: FileStoreTxCollection; + + /** File store collection for fast (proposal/proving) path */ + protected readonly fileStoreFastCollection: FileStoreTxCollection; /** Loop for periodically reconciling found transactions from the tx pool in case we missed some */ private readonly reconcileFoundTxsLoop: RunningPromise; @@ -111,7 +114,28 @@ export class TxCollection { ); this.hasFileStoreSources = fileStoreSources.length > 0; - this.fileStoreCollection = new FileStoreTxCollection(fileStoreSources, this.txCollectionSink, this.log); + this.fileStoreSlowCollection = new FileStoreTxCollection( + fileStoreSources, + this.txCollectionSink, + { + workerCount: config.txCollectionFileStoreSlowWorkerCount, + backoffBaseMs: config.txCollectionFileStoreSlowBackoffBaseMs, + backoffMaxMs: config.txCollectionFileStoreSlowBackoffMaxMs, + }, + this.dateProvider, + this.log, + ); + this.fileStoreFastCollection = new FileStoreTxCollection( + fileStoreSources, + this.txCollectionSink, + { + workerCount: config.txCollectionFileStoreFastWorkerCount, + backoffBaseMs: config.txCollectionFileStoreFastBackoffBaseMs, + backoffMaxMs: config.txCollectionFileStoreFastBackoffMaxMs, + }, + this.dateProvider, + this.log, + ); this.reconcileFoundTxsLoop = new RunningPromise( () => this.reconcileFoundTxsWithPool(), @@ -137,7 +161,8 @@ export class TxCollection { public start(): Promise { this.started = true; this.slowCollection.start(); - this.fileStoreCollection.start(); + this.fileStoreSlowCollection.start(); + this.fileStoreFastCollection.start(); this.reconcileFoundTxsLoop.start(); // TODO(palla/txs): Collect mined unproven tx hashes for txs we dont have in the pool and populate missingTxs on startup @@ -150,7 +175,8 @@ export class TxCollection { await Promise.all([ this.slowCollection.stop(), this.fastCollection.stop(), - this.fileStoreCollection.stop(), + this.fileStoreSlowCollection.stop(), + this.fileStoreFastCollection.stop(), this.reconcileFoundTxsLoop.stop(), ]); @@ -175,6 +201,7 @@ export class TxCollection { // Delay file store collection to give P2P methods time to find txs first if (this.hasFileStoreSources) { const context: TxAddContext = { type: 'mined', block }; + const deadline = getProofDeadlineForSlot(block.header.getSlot(), this.constants); sleep(this.config.txCollectionFileStoreSlowDelayMs) .then(() => { if (this.started) { @@ -182,7 +209,7 @@ export class TxCollection { const stillMissing = new Set(this.slowCollection.getMissingTxHashes().map(h => h.toString())); const remaining = txHashes.filter(h => stillMissing.has(h.toString())); if (remaining.length > 0) { - this.fileStoreCollection.startCollecting(remaining, context); + this.fileStoreSlowCollection.startCollecting(remaining, context, deadline); } } }) @@ -223,7 +250,7 @@ export class TxCollection { sleep(this.config.txCollectionFileStoreFastDelayMs) .then(() => { if (this.started) { - this.fileStoreCollection.startCollecting(hashes, context); + this.fileStoreFastCollection.startCollecting(hashes, context, opts.deadline); } }) .catch(err => this.log.error('Error in file store fast delay', err)); @@ -245,7 +272,8 @@ export class TxCollection { private foundTxs(txs: Tx[]) { this.slowCollection.foundTxs(txs); this.fastCollection.foundTxs(txs); - this.fileStoreCollection.foundTxs(txs); + this.fileStoreSlowCollection.foundTxs(txs); + this.fileStoreFastCollection.foundTxs(txs); } /** @@ -255,7 +283,8 @@ export class TxCollection { public stopCollectingForBlocksUpTo(blockNumber: BlockNumber): void { this.slowCollection.stopCollectingForBlocksUpTo(blockNumber); this.fastCollection.stopCollectingForBlocksUpTo(blockNumber); - this.fileStoreCollection.clearPending(); + this.fileStoreSlowCollection.clearPending(); + this.fileStoreFastCollection.clearPending(); } /** @@ -265,7 +294,8 @@ export class TxCollection { public stopCollectingForBlocksAfter(blockNumber: BlockNumber): void { this.slowCollection.stopCollectingForBlocksAfter(blockNumber); this.fastCollection.stopCollectingForBlocksAfter(blockNumber); - this.fileStoreCollection.clearPending(); + this.fileStoreSlowCollection.clearPending(); + this.fileStoreFastCollection.clearPending(); } /** Every now and then, check if the pool has received one of the txs we are looking for, just to catch any race conditions */ From 461ca22e064e342a9e158ae48f27259696efaa3a Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 11 Feb 2026 11:23:58 -0300 Subject: [PATCH 13/27] chore(ci): run ci job on draft PRs (#20395) --- .github/workflows/ci3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci3.yml b/.github/workflows/ci3.yml index fd40ccf7a063..e4e4151126bc 100644 --- a/.github/workflows/ci3.yml +++ b/.github/workflows/ci3.yml @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest # exclusive with ci3-external.yml: never run on forks # (github.event.pull_request.head.repo.fork resolves to nil if not a pull request) - if: github.event.pull_request.head.repo.fork != true && github.event.pull_request.draft == false + if: github.event.pull_request.head.repo.fork != true environment: ${{ startsWith(github.ref, 'refs/tags/v') && 'master' || '' }} steps: - name: Checkout From dec2115ed253b28d044e73648bdce8debca093f2 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 11 Feb 2026 11:49:55 -0300 Subject: [PATCH 14/27] feat(bot): allow anchoring txs to proposed chain (#20392) Updates spartan scripts to config the PXE_SYNC_CHAIN_TIP variable that controls the chain tip used for anchoring txs in the bot. Also updates the bot so it can be configured to follow (as in wait until each tx in mined in) the "proposed" or "checkpointed" chain, rather than the "pending" chain, which is an outdated name. Still, for backwards compatibility, we still allow "pending" and map it to "checkpointed". Fixes A-539 --- .../aztec-bot/templates/env.configmap.yaml | 1 + spartan/aztec-bot/values.yaml | 1 + spartan/scripts/deploy_network.sh | 4 ++++ spartan/terraform/deploy-aztec-infra/main.tf | 2 ++ .../terraform/deploy-aztec-infra/variables.tf | 12 ++++++++++ yarn-project/aztec/src/cli/cmds/start_bot.ts | 7 ++++-- yarn-project/aztec/src/cli/util.ts | 7 ++++++ yarn-project/bot/src/base_bot.ts | 22 +++++-------------- yarn-project/bot/src/config.ts | 10 ++++++--- yarn-project/end-to-end/src/e2e_bot.test.ts | 10 ++++----- .../src/e2e_sequencer_config.test.ts | 2 +- .../src/spartan/gating-passive.test.ts | 2 +- .../end-to-end/src/spartan/utils/bot.ts | 2 +- 13 files changed, 53 insertions(+), 29 deletions(-) diff --git a/spartan/aztec-bot/templates/env.configmap.yaml b/spartan/aztec-bot/templates/env.configmap.yaml index 4e7b71230178..da4414c48818 100644 --- a/spartan/aztec-bot/templates/env.configmap.yaml +++ b/spartan/aztec-bot/templates/env.configmap.yaml @@ -9,6 +9,7 @@ data: BOT_PRIVATE_TRANSFERS_PER_TX: {{ .Values.bot.privateTransfersPerTx | quote }} BOT_PUBLIC_TRANSFERS_PER_TX: {{ .Values.bot.publicTransfersPerTx | quote }} BOT_FOLLOW_CHAIN: {{ .Values.bot.followChain | quote }} + PXE_SYNC_CHAIN_TIP: {{ .Values.bot.pxeSyncChainTip | quote }} BOT_NO_START: {{ .Values.bot.botNoStart | quote }} BOT_FEE_PAYMENT_METHOD: {{ .Values.bot.feePaymentMethod | quote }} BOT_AMM_TXS: {{ .Values.bot.ammTxs | quote }} diff --git a/spartan/aztec-bot/values.yaml b/spartan/aztec-bot/values.yaml index d310e507ed8e..10fecf8ed417 100644 --- a/spartan/aztec-bot/values.yaml +++ b/spartan/aztec-bot/values.yaml @@ -12,6 +12,7 @@ bot: publicTransfersPerTx: 1 # Do not wait for transactions followChain: "NONE" + pxeSyncChainTip: "checkpointed" botNoStart: false feePaymentMethod: "fee_juice" ammTxs: false diff --git a/spartan/scripts/deploy_network.sh b/spartan/scripts/deploy_network.sh index e6007c3fd3f4..beab348a291c 100755 --- a/spartan/scripts/deploy_network.sh +++ b/spartan/scripts/deploy_network.sh @@ -115,6 +115,8 @@ BOT_TRANSFERS_TX_INTERVAL_SECONDS=${BOT_TRANSFERS_TX_INTERVAL_SECONDS:-60} BOT_SWAPS_TX_INTERVAL_SECONDS=${BOT_SWAPS_TX_INTERVAL_SECONDS:-60} BOT_TRANSFERS_FOLLOW_CHAIN=${BOT_TRANSFERS_FOLLOW_CHAIN:-NONE} BOT_SWAPS_FOLLOW_CHAIN=${BOT_SWAPS_FOLLOW_CHAIN:-NONE} +BOT_TRANSFERS_PXE_SYNC_CHAIN_TIP=${BOT_TRANSFERS_PXE_SYNC_CHAIN_TIP:-checkpointed} +BOT_SWAPS_PXE_SYNC_CHAIN_TIP=${BOT_SWAPS_PXE_SYNC_CHAIN_TIP:-checkpointed} RPC_INGRESS_ENABLED=${RPC_INGRESS_ENABLED:-false} RPC_INGRESS_HOSTS=${RPC_INGRESS_HOSTS:-[]} @@ -506,6 +508,8 @@ BOT_SWAPS_MNEMONIC_START_INDEX = ${BOT_SWAPS_MNEMONIC_START_INDEX} BOT_SWAPS_REPLICAS = ${BOT_SWAPS_REPLICAS} BOT_SWAPS_TX_INTERVAL_SECONDS = ${BOT_SWAPS_TX_INTERVAL_SECONDS} BOT_SWAPS_FOLLOW_CHAIN = "${BOT_SWAPS_FOLLOW_CHAIN}" +BOT_TRANSFERS_PXE_SYNC_CHAIN_TIP = "${BOT_TRANSFERS_PXE_SYNC_CHAIN_TIP}" +BOT_SWAPS_PXE_SYNC_CHAIN_TIP = "${BOT_SWAPS_PXE_SYNC_CHAIN_TIP}" BOT_TRANSFERS_L2_PRIVATE_KEY = "${BOT_TRANSFERS_L2_PRIVATE_KEY:-0xcafe01}" BOT_SWAPS_L2_PRIVATE_KEY = "${BOT_SWAPS_L2_PRIVATE_KEY:-0xcafe02}" diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index 1222f613773f..d4518fc09787 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -576,6 +576,7 @@ locals { "bot.replicaCount" = var.BOT_TRANSFERS_REPLICAS "bot.txIntervalSeconds" = var.BOT_TRANSFERS_TX_INTERVAL_SECONDS "bot.followChain" = var.BOT_TRANSFERS_FOLLOW_CHAIN + "bot.pxeSyncChainTip" = var.BOT_TRANSFERS_PXE_SYNC_CHAIN_TIP "bot.botPrivateKey" = var.BOT_TRANSFERS_L2_PRIVATE_KEY "bot.nodeUrl" = local.internal_rpc_url "bot.mnemonic" = var.BOT_MNEMONIC @@ -599,6 +600,7 @@ locals { "bot.replicaCount" = var.BOT_SWAPS_REPLICAS "bot.txIntervalSeconds" = var.BOT_SWAPS_TX_INTERVAL_SECONDS "bot.followChain" = var.BOT_SWAPS_FOLLOW_CHAIN + "bot.pxeSyncChainTip" = var.BOT_SWAPS_PXE_SYNC_CHAIN_TIP "bot.botPrivateKey" = var.BOT_SWAPS_L2_PRIVATE_KEY "bot.nodeUrl" = local.internal_rpc_url "bot.mnemonic" = var.BOT_MNEMONIC diff --git a/spartan/terraform/deploy-aztec-infra/variables.tf b/spartan/terraform/deploy-aztec-infra/variables.tf index d5d412557818..798f88005737 100644 --- a/spartan/terraform/deploy-aztec-infra/variables.tf +++ b/spartan/terraform/deploy-aztec-infra/variables.tf @@ -519,6 +519,12 @@ variable "BOT_TRANSFERS_FOLLOW_CHAIN" { default = "PENDING" } +variable "BOT_TRANSFERS_PXE_SYNC_CHAIN_TIP" { + description = "Transfers bot PXE sync chain tip mode (e.g., checkpointed)" + type = string + default = "checkpointed" +} + variable "BOT_TRANSFERS_L2_PRIVATE_KEY" { description = "Private key for the transfers bot (hex string starting with 0x)" nullable = true @@ -549,6 +555,12 @@ variable "BOT_SWAPS_FOLLOW_CHAIN" { default = "PENDING" } +variable "BOT_SWAPS_PXE_SYNC_CHAIN_TIP" { + description = "AMM swaps bot PXE sync chain tip mode (e.g., checkpointed)" + type = string + default = "checkpointed" +} + variable "BOT_SWAPS_L2_PRIVATE_KEY" { description = "Private key for the AMM swaps bot (hex string starting with 0x)" type = string diff --git a/yarn-project/aztec/src/cli/cmds/start_bot.ts b/yarn-project/aztec/src/cli/cmds/start_bot.ts index 3d23797ed400..6263177786dc 100644 --- a/yarn-project/aztec/src/cli/cmds/start_bot.ts +++ b/yarn-project/aztec/src/cli/cmds/start_bot.ts @@ -12,7 +12,7 @@ import { } from '@aztec/telemetry-client'; import { TestWallet } from '@aztec/test-wallet/server'; -import { extractRelevantOptions } from '../util.js'; +import { extractRelevantOptions, stringifyConfig } from '../util.js'; import { getVersions } from '../versioning.js'; export async function startBot( @@ -38,10 +38,11 @@ export async function startBot( const aztecNode = createAztecNodeClient(config.nodeUrl, getVersions(), fetch); const pxeConfig = extractRelevantOptions(options, allPxeConfigMappings, 'pxe'); + userLog(`Creating bot test wallet with config ${stringifyConfig(pxeConfig)}`); const wallet = await TestWallet.create(aztecNode, pxeConfig); const telemetry = await initTelemetryClient(getTelemetryClientConfig()); - await addBot(options, signalHandlers, services, wallet, aztecNode, telemetry, undefined); + await addBot(options, signalHandlers, services, wallet, aztecNode, telemetry, undefined, userLog); } export async function addBot( @@ -52,8 +53,10 @@ export async function addBot( aztecNode: AztecNode, telemetry: TelemetryClient, aztecNodeAdmin?: AztecNodeAdmin, + userLog?: LogFn, ) { const config = extractRelevantOptions(options, botConfigMappings, 'bot'); + userLog?.(`Starting bot with config ${stringifyConfig(config)}`); const db = await (config.dataDirectory ? createStore('bot', BotStore.SCHEMA_VERSION, config) diff --git a/yarn-project/aztec/src/cli/util.ts b/yarn-project/aztec/src/cli/util.ts index d5063ba24de9..fe90d7f4b445 100644 --- a/yarn-project/aztec/src/cli/util.ts +++ b/yarn-project/aztec/src/cli/util.ts @@ -4,6 +4,7 @@ import type { ViemClient } from '@aztec/ethereum/types'; import type { ConfigMappingsType } from '@aztec/foundation/config'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; +import { jsonStringify } from '@aztec/foundation/json-rpc'; import { type LogFn, createLogger } from '@aztec/foundation/log'; import type { SharedNodeConfig } from '@aztec/node-lib/config'; import type { ProverConfig } from '@aztec/stdlib/interfaces/server'; @@ -388,3 +389,9 @@ export async function setupUpdateMonitor( checker.start(); } + +export function stringifyConfig(config: object): string { + return Object.entries(config) + .map(([key, value]) => `${key}=${jsonStringify(value)}`) + .join(' '); +} diff --git a/yarn-project/bot/src/base_bot.ts b/yarn-project/bot/src/base_bot.ts index 74184a072fc1..2ec0397f5902 100644 --- a/yarn-project/bot/src/base_bot.ts +++ b/yarn-project/bot/src/base_bot.ts @@ -1,13 +1,8 @@ import { AztecAddress } from '@aztec/aztec.js/addresses'; -import { - BatchCall, - ContractFunctionInteraction, - type SendInteractionOptions, - waitForProven, -} from '@aztec/aztec.js/contracts'; +import { BatchCall, ContractFunctionInteraction, type SendInteractionOptions } from '@aztec/aztec.js/contracts'; import { createLogger } from '@aztec/aztec.js/log'; import { waitForTx } from '@aztec/aztec.js/node'; -import { TxHash, TxReceipt } from '@aztec/aztec.js/tx'; +import { TxHash, TxReceipt, TxStatus } from '@aztec/aztec.js/tx'; import { Gas } from '@aztec/stdlib/gas'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; import type { TestWallet } from '@aztec/test-wallet/server'; @@ -29,8 +24,8 @@ export abstract class BaseBot { public async run(): Promise { this.attempts++; - const logCtx = { runId: Date.now() * 1000 + Math.floor(Math.random() * 1000) }; const { followChain, txMinedWaitSeconds } = this.config; + const logCtx = { runId: Date.now() * 1000 + Math.floor(Math.random() * 1000), followChain, txMinedWaitSeconds }; this.log.verbose(`Creating tx`, logCtx); const txHash = await this.createAndSendTx(logCtx); @@ -40,14 +35,9 @@ export abstract class BaseBot { return txHash; } - this.log.verbose( - `Awaiting tx ${txHash.toString()} to be on the ${followChain} chain (timeout ${txMinedWaitSeconds}s)`, - logCtx, - ); - const receipt = await waitForTx(this.node, txHash, { timeout: txMinedWaitSeconds }); - if (followChain === 'PROVEN') { - await waitForProven(this.node, receipt, { provenTimeout: txMinedWaitSeconds }); - } + const waitForStatus = TxStatus[followChain]; + this.log.verbose(`Awaiting tx ${txHash.toString()} to be on the ${followChain} chain`, logCtx); + const receipt = await waitForTx(this.node, txHash, { timeout: txMinedWaitSeconds, waitForStatus }); this.successes++; this.log.info( `Tx #${this.attempts} ${receipt.txHash} successfully mined in block ${receipt.blockNumber} (stats: ${this.successes}/${this.attempts} success)`, diff --git a/yarn-project/bot/src/config.ts b/yarn-project/bot/src/config.ts index ae73ce1f0f01..d02ef5ebb401 100644 --- a/yarn-project/bot/src/config.ts +++ b/yarn-project/bot/src/config.ts @@ -19,7 +19,7 @@ import type { ComponentsVersions } from '@aztec/stdlib/versioning'; import { z } from 'zod'; -const BotFollowChain = ['NONE', 'PENDING', 'PROVEN'] as const; +const BotFollowChain = ['NONE', 'PROPOSED', 'CHECKPOINTED', 'PROVEN'] as const; type BotFollowChain = (typeof BotFollowChain)[number]; export enum SupportedTokenContracts { @@ -213,10 +213,14 @@ export const botConfigMappings: ConfigMappingsType = { description: 'Which chain the bot follows', defaultValue: 'NONE', parseEnv(val) { - if (!(BotFollowChain as readonly string[]).includes(val.toUpperCase())) { + const upper = val.toUpperCase(); + if (upper === 'PENDING') { + return 'CHECKPOINTED'; + } + if (!(BotFollowChain as readonly string[]).includes(upper)) { throw new Error(`Invalid value for BOT_FOLLOW_CHAIN: ${val}`); } - return val as BotFollowChain; + return upper as BotFollowChain; }, }, maxPendingTxs: { diff --git a/yarn-project/end-to-end/src/e2e_bot.test.ts b/yarn-project/end-to-end/src/e2e_bot.test.ts index ffcef16930db..6648e886349b 100644 --- a/yarn-project/end-to-end/src/e2e_bot.test.ts +++ b/yarn-project/end-to-end/src/e2e_bot.test.ts @@ -44,7 +44,7 @@ describe('e2e_bot', () => { beforeAll(async () => { config = { ...getBotDefaultConfig(), - followChain: 'PENDING', + followChain: 'CHECKPOINTED', ammTxs: false, }; bot = await Bot.create(config, wallet, aztecNode, undefined, new BotStore(await openTmpStore('bot'))); @@ -109,7 +109,7 @@ describe('e2e_bot', () => { const config: BotConfig = { ...getBotDefaultConfig(), - followChain: 'PENDING', + followChain: 'CHECKPOINTED', ammTxs: false, // this bot has a well defined private key and salt @@ -147,7 +147,7 @@ describe('e2e_bot', () => { const config: BotConfig = { ...getBotDefaultConfig(), - followChain: 'PENDING', + followChain: 'CHECKPOINTED', ammTxs: false, // this bot has a well defined private key and salt @@ -185,7 +185,7 @@ describe('e2e_bot', () => { beforeAll(async () => { config = { ...getBotDefaultConfig(), - followChain: 'PENDING', + followChain: 'CHECKPOINTED', ammTxs: true, }; bot = await AmmBot.create(config, wallet, aztecNode, undefined, new BotStore(await openTmpStore('bot'))); @@ -215,7 +215,7 @@ describe('e2e_bot', () => { beforeAll(() => { config = { ...getBotDefaultConfig(), - followChain: 'PENDING', + followChain: 'CHECKPOINTED', ammTxs: false, senderPrivateKey: new SecretValue(Fr.random()), l1PrivateKey: new SecretValue(bufferToHex(getPrivateKeyFromIndex(8)!)), diff --git a/yarn-project/end-to-end/src/e2e_sequencer_config.test.ts b/yarn-project/end-to-end/src/e2e_sequencer_config.test.ts index 98258d0e5b10..f1ac27f7712a 100644 --- a/yarn-project/end-to-end/src/e2e_sequencer_config.test.ts +++ b/yarn-project/end-to-end/src/e2e_sequencer_config.test.ts @@ -39,7 +39,7 @@ describe('e2e_sequencer_config', () => { })); config = { ...getBotDefaultConfig(), - followChain: 'PENDING', + followChain: 'CHECKPOINTED', ammTxs: false, txMinedWaitSeconds: 12, }; diff --git a/yarn-project/end-to-end/src/spartan/gating-passive.test.ts b/yarn-project/end-to-end/src/spartan/gating-passive.test.ts index a122f1af5e11..4a8bf19ae4a7 100644 --- a/yarn-project/end-to-end/src/spartan/gating-passive.test.ts +++ b/yarn-project/end-to-end/src/spartan/gating-passive.test.ts @@ -112,7 +112,7 @@ describe('a test that passively observes the network in the presence of network logger: debugLogger, replicas: 1, txIntervalSeconds: 10, - followChain: 'PENDING', + followChain: 'CHECKPOINTED', }); }); diff --git a/yarn-project/end-to-end/src/spartan/utils/bot.ts b/yarn-project/end-to-end/src/spartan/utils/bot.ts index c841cad5a429..6314d2850292 100644 --- a/yarn-project/end-to-end/src/spartan/utils/bot.ts +++ b/yarn-project/end-to-end/src/spartan/utils/bot.ts @@ -34,7 +34,7 @@ export async function installTransferBot({ logger: log, replicas = 1, txIntervalSeconds = 10, - followChain = 'PENDING', + followChain = 'CHECKPOINTED', mnemonic = process.env.LABS_INFRA_MNEMONIC ?? 'test test test test test test test test test test test junk', mnemonicStartIndex, botPrivateKey = process.env.BOT_TRANSFERS_L2_PRIVATE_KEY ?? '0xcafe01', From ddc4a68f93bf56510c89e674b5aae85de1899f10 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 11 Feb 2026 11:57:55 -0300 Subject: [PATCH 15/27] chore: add setup-container script (#20309) Single bash command that installs all dependencies plus a network-isolated Claude in a local ubuntu lxc container. Put together using the devcontainer dockerfile and the official claude container dockerfile as guidelines. --- scripts/setup-container.sh | 505 +++++++++++++++++++++++++++++++++++++ 1 file changed, 505 insertions(+) create mode 100644 scripts/setup-container.sh diff --git a/scripts/setup-container.sh b/scripts/setup-container.sh new file mode 100644 index 000000000000..68a00a711f8a --- /dev/null +++ b/scripts/setup-container.sh @@ -0,0 +1,505 @@ +#!/bin/bash +set -euo pipefail + +# ============================================================================= +# Aztec Development Container Setup Script +# ============================================================================= +# This script sets up an Ubuntu container with all dependencies needed to build +# the Aztec project, plus Claude Code with restricted outbound network access. +# +# Last updated: 2026-01-30 +# +# Usage: sudo ./setup-container.sh +# ============================================================================= + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +log_info() { echo -e "${GREEN}[INFO]${NC} $1"; } +log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } +log_error() { echo -e "${RED}[ERROR]${NC} $1"; } + +# Check if running as root +if [ "$EUID" -ne 0 ]; then + log_error "Please run as root (sudo ./setup-container.sh)" + exit 1 +fi + +# Get the actual user if running via sudo +ACTUAL_USER=${SUDO_USER:-$USER} +ACTUAL_HOME=$(eval echo ~$ACTUAL_USER) + +export DEBIAN_FRONTEND=noninteractive + +# ============================================================================= +# SECTION 1: Base System Packages +# ============================================================================= +log_info "Installing base system packages..." + +apt-get update +apt-get install -y \ + build-essential \ + ca-certificates \ + bash \ + clang \ + cmake \ + make \ + ninja-build \ + git \ + curl \ + gnupg \ + gnupg2 \ + python3 \ + wget \ + time \ + jq \ + gawk \ + unzip \ + netcat-openbsd \ + parallel \ + xz-utils \ + lsof \ + xxd \ + zstd \ + lsb-release \ + software-properties-common \ + clang-16 \ + clang-format-16 \ + libc++-dev \ + libomp-dev \ + doxygen \ + libdw-dev \ + libelf-dev \ + pkg-config \ + libssl-dev \ + python3-clang \ + less \ + procps \ + sudo \ + fzf \ + zsh \ + man-db \ + nano \ + vim \ + iptables \ + ipset \ + iproute2 \ + dnsutils \ + aggregate + +apt-get -y autoremove +apt-get clean + +# ============================================================================= +# SECTION 2: Node.js 24 +# ============================================================================= +log_info "Installing Node.js 24..." + +curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg +echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_24.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list +apt-get update +apt-get install -y nodejs=24.12.0-1nodesource1 + +# ============================================================================= +# SECTION 3: Clang 18/20 +# ============================================================================= +log_info "Installing Clang 18 and 20..." + +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +./llvm.sh 18 all +./llvm.sh 20 all +rm llvm.sh + +# ============================================================================= +# SECTION 4: Rust +# ============================================================================= +log_info "Installing Rust..." + +export RUSTUP_HOME=/opt/rust/rustup +export CARGO_HOME=/opt/rust/cargo + +curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain 1.85.0 +source /opt/rust/cargo/env + +rustup target add wasm32-unknown-unknown wasm32-wasip1 + +# Make rust accessible to all users +chmod -R a+w /opt/rust + +# Add to system-wide profile +cat >> /etc/profile.d/rust.sh << 'EOF' +export RUSTUP_HOME=/opt/rust/rustup +export CARGO_HOME=/opt/rust/cargo +export PATH="/opt/rust/cargo/bin:$PATH" +EOF +chmod +x /etc/profile.d/rust.sh + +# ============================================================================= +# SECTION 5: wasi-sdk +# ============================================================================= +log_info "Installing wasi-sdk 27..." + +arch=$(uname -m) +if [ "$arch" = "aarch64" ]; then arch="arm64"; fi +wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-27/wasi-sdk-27.0-${arch}-linux.tar.gz +tar xvf wasi-sdk-27.0-${arch}-linux.tar.gz +mv wasi-sdk-27.0-${arch}-linux /opt/wasi-sdk +rm wasi-sdk-27.0-${arch}-linux.tar.gz + +# ============================================================================= +# SECTION 6: Foundry +# ============================================================================= +log_info "Installing Foundry v1.4.1..." + +export PATH="/opt/rust/cargo/bin:$PATH" +export FOUNDRY_BIN_DIR="/tmp/foundry-bin" +export RUSTFLAGS="-C target-cpu=generic" + +curl -L https://foundry.paradigm.xyz | bash +$HOME/.foundry/bin/foundryup -i v1.4.1 + +mkdir -p /opt/foundry/bin +for t in forge cast anvil chisel; do + mv $HOME/.foundry/bin/$t /opt/foundry/bin/$t + strip /opt/foundry/bin/$t +done +rm -rf $HOME/.foundry + +# Add to system-wide profile +cat >> /etc/profile.d/foundry.sh << 'EOF' +export PATH="/opt/foundry/bin:$PATH" +EOF +chmod +x /etc/profile.d/foundry.sh + +# ============================================================================= +# SECTION 7: Zig +# ============================================================================= +log_info "Installing Zig 0.15.1..." + +arch=$(uname -m) +zig_version=0.15.1 +curl -fsSL https://ziglang.org/download/${zig_version}/zig-${arch}-linux-${zig_version}.tar.xz | tar xJ +mv zig-${arch}-linux-${zig_version} /opt/zig +ln -sf /opt/zig/zig /usr/local/bin/zig + +# ============================================================================= +# SECTION 8: yq +# ============================================================================= +log_info "Installing yq v4..." + +curl -sL https://github.com/mikefarah/yq/releases/download/v4.42.1/yq_linux_$(dpkg --print-architecture) \ + -o /usr/local/bin/yq && chmod +x /usr/local/bin/yq + +# ============================================================================= +# SECTION 9: ldid (for macOS cross-compilation signing) +# ============================================================================= +log_info "Installing ldid..." + +curl -sL https://github.com/ProcursusTeam/ldid/releases/download/v2.1.5-procursus7/ldid_linux_x86_64 \ + -o /usr/local/bin/ldid && chmod +x /usr/local/bin/ldid + +# ============================================================================= +# SECTION 10: wasmtime +# ============================================================================= +log_info "Installing wasmtime..." + +curl -fsSL https://github.com/bytecodealliance/wasmtime/releases/download/v20.0.2/wasmtime-v20.0.2-$(uname -m)-linux.tar.xz | tar xJ +mv wasmtime-v20.0.2-$(uname -m)-linux/wasmtime /usr/local/bin +rm -rf wasmtime* + +# ============================================================================= +# SECTION 11: npm global packages +# ============================================================================= +log_info "Installing npm global packages (corepack, yarn, solhint)..." + +npm install --global corepack +corepack enable +corepack install --global yarn@4.5.2 +npm install --global solhint + +# ============================================================================= +# SECTION 12: GitHub CLI +# ============================================================================= +log_info "Installing GitHub CLI..." + +mkdir -p -m 755 /etc/apt/keyrings +wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg > /etc/apt/keyrings/githubcli-archive-keyring.gpg +chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg +echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null +apt-get update +apt-get install -y gh + +# ============================================================================= +# SECTION 13: Claude Code +# ============================================================================= +log_info "Installing Claude Code..." + +npm install -g @anthropic-ai/claude-code + +# ============================================================================= +# SECTION 14: Create Firewall Script +# ============================================================================= +log_info "Creating firewall initialization script..." + +cat > /usr/local/bin/init-firewall.sh << 'FIREWALL_EOF' +#!/bin/bash +set -euo pipefail +IFS=$'\n\t' + +echo "Initializing firewall with restricted outbound access..." + +# 1. Extract Docker DNS info BEFORE any flushing +DOCKER_DNS_RULES=$(iptables-save -t nat | grep "127\.0\.0\.11" || true) + +# Flush existing rules and delete existing ipsets +iptables -F +iptables -X +iptables -t nat -F +iptables -t nat -X +iptables -t mangle -F +iptables -t mangle -X +ipset destroy allowed-domains 2>/dev/null || true + +# 2. Selectively restore ONLY internal Docker DNS resolution +if [ -n "$DOCKER_DNS_RULES" ]; then + echo "Restoring Docker DNS rules..." + iptables -t nat -N DOCKER_OUTPUT 2>/dev/null || true + iptables -t nat -N DOCKER_POSTROUTING 2>/dev/null || true + echo "$DOCKER_DNS_RULES" | xargs -L 1 iptables -t nat +else + echo "No Docker DNS rules to restore" +fi + +# First allow DNS and localhost before any restrictions +# Allow outbound DNS +iptables -A OUTPUT -p udp --dport 53 -j ACCEPT +# Allow inbound DNS responses +iptables -A INPUT -p udp --sport 53 -j ACCEPT +# Allow outbound SSH +iptables -A OUTPUT -p tcp --dport 22 -j ACCEPT +# Allow inbound SSH responses +iptables -A INPUT -p tcp --sport 22 -m state --state ESTABLISHED -j ACCEPT +# Allow localhost +iptables -A INPUT -i lo -j ACCEPT +iptables -A OUTPUT -o lo -j ACCEPT + +# Create ipset with CIDR support +ipset create allowed-domains hash:net + +# Fetch GitHub meta information and aggregate + add their IP ranges +echo "Fetching GitHub IP ranges..." +gh_ranges=$(curl -s https://api.github.com/meta) +if [ -z "$gh_ranges" ]; then + echo "ERROR: Failed to fetch GitHub IP ranges" + exit 1 +fi + +if ! echo "$gh_ranges" | jq -e '.web and .api and .git' >/dev/null; then + echo "ERROR: GitHub API response missing required fields" + exit 1 +fi + +echo "Processing GitHub IPs..." +while read -r cidr; do + if [[ ! "$cidr" =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}/[0-9]{1,2}$ ]]; then + echo "ERROR: Invalid CIDR range from GitHub meta: $cidr" + exit 1 + fi + echo "Adding GitHub range $cidr" + ipset add allowed-domains "$cidr" +done < <(echo "$gh_ranges" | jq -r '(.web + .api + .git)[]' | aggregate -q) + +# Resolve and add other allowed domains +# Including Aztec CI server and standard Claude Code domains +for domain in \ + "ci.aztec-labs.com" \ + "registry.npmjs.org" \ + "api.anthropic.com" \ + "sentry.io" \ + "statsig.anthropic.com" \ + "statsig.com" \ + "marketplace.visualstudio.com" \ + "vscode.blob.core.windows.net" \ + "update.code.visualstudio.com"; do + echo "Resolving $domain..." + ips=$(dig +noall +answer A "$domain" | awk '$4 == "A" {print $5}') + if [ -z "$ips" ]; then + echo "WARNING: Failed to resolve $domain (may be expected if not using this service)" + continue + fi + + while read -r ip; do + if [[ ! "$ip" =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + echo "ERROR: Invalid IP from DNS for $domain: $ip" + exit 1 + fi + echo "Adding $ip for $domain" + ipset add allowed-domains "$ip" + done < <(echo "$ips") +done + +# Get host IP from default route +HOST_IP=$(ip route | grep default | cut -d" " -f3) +if [ -z "$HOST_IP" ]; then + echo "ERROR: Failed to detect host IP" + exit 1 +fi + +HOST_NETWORK=$(echo "$HOST_IP" | sed "s/\.[0-9]*$/.0\/24/") +echo "Host network detected as: $HOST_NETWORK" + +# Set up remaining iptables rules +iptables -A INPUT -s "$HOST_NETWORK" -j ACCEPT +iptables -A OUTPUT -d "$HOST_NETWORK" -j ACCEPT + +# Set default policies to DROP first +iptables -P INPUT DROP +iptables -P FORWARD DROP +iptables -P OUTPUT DROP + +# First allow established connections for already approved traffic +iptables -A INPUT -m state --state ESTABLISHED,RELATED -j ACCEPT +iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT + +# Then allow only specific outbound traffic to allowed domains +iptables -A OUTPUT -m set --match-set allowed-domains dst -j ACCEPT + +# Explicitly REJECT all other outbound traffic for immediate feedback +iptables -A OUTPUT -j REJECT --reject-with icmp-admin-prohibited + +echo "Firewall configuration complete" +echo "Verifying firewall rules..." + +# Verify blocked access +if curl --connect-timeout 5 https://example.com >/dev/null 2>&1; then + echo "ERROR: Firewall verification failed - was able to reach https://example.com" + exit 1 +else + echo "Firewall verification passed - unable to reach https://example.com as expected" +fi + +# Verify GitHub API access +if ! curl --connect-timeout 5 https://api.github.com/zen >/dev/null 2>&1; then + echo "ERROR: Firewall verification failed - unable to reach https://api.github.com" + exit 1 +else + echo "Firewall verification passed - able to reach https://api.github.com as expected" +fi + +echo "" +echo "=== Firewall Active ===" +echo "Allowed domains:" +echo " - GitHub (api, web, git)" +echo " - ci.aztec-labs.com" +echo " - registry.npmjs.org" +echo " - api.anthropic.com" +echo " - statsig.anthropic.com" +echo " - sentry.io" +echo " - VS Code marketplace" +echo "========================" +FIREWALL_EOF + +chmod +x /usr/local/bin/init-firewall.sh + +# Allow non-root users to run the firewall script +echo "$ACTUAL_USER ALL=(root) NOPASSWD: /usr/local/bin/init-firewall.sh" > /etc/sudoers.d/user-firewall +chmod 440 /etc/sudoers.d/user-firewall + +# ============================================================================= +# SECTION 15: Environment Setup +# ============================================================================= +log_info "Setting up environment..." + +# Create combined environment file +cat > /etc/profile.d/aztec-env.sh << 'EOF' +export RUSTUP_HOME=/opt/rust/rustup +export CARGO_HOME=/opt/rust/cargo +export PATH="/opt/rust/cargo/bin:/opt/foundry/bin:/opt/zig:$PATH" +export LANG=C.UTF-8 +export TERM=xterm-256color +EOF +chmod +x /etc/profile.d/aztec-env.sh + +# ============================================================================= +# SECTION 16: Verification +# ============================================================================= +log_info "Verifying installations..." + +# Source environment +source /etc/profile.d/aztec-env.sh +source /etc/profile.d/rust.sh +source /etc/profile.d/foundry.sh + +echo "" +echo "=== Installed Versions ===" + +echo -n "Node.js: " +node --version + +echo -n "npm: " +npm --version + +echo -n "Rust: " +rustc --version + +echo -n "Clang 20: " +clang++-20 --version | head -n1 + +echo -n "Zig: " +zig version + +echo -n "Foundry (forge): " +/opt/foundry/bin/forge --version + +echo -n "wasi-sdk: " +cat /opt/wasi-sdk/VERSION + +echo -n "yq: " +yq --version + +echo -n "GitHub CLI: " +gh --version | head -n1 + +echo -n "Claude Code: " +claude --version 2>/dev/null || echo "installed (run 'claude' to authenticate)" + +echo "" +echo "===========================" + +# ============================================================================= +# Final Instructions +# ============================================================================= +log_info "Setup complete!" + +cat << 'INSTRUCTIONS' + +=== NEXT STEPS === + +1. Source the environment (or log out and back in): + source /etc/profile.d/aztec-env.sh + +2. Authenticate Claude Code: + claude + +3. (Optional) Enable firewall to restrict outbound access: + sudo /usr/local/bin/init-firewall.sh + + This will restrict network access to only: + - GitHub (for git operations) + - ci.aztec-labs.com (Aztec CI) + - npm registry + - Anthropic API (for Claude) + - VS Code marketplace + +4. Bootstrap the Aztec project: + cd /path/to/aztec + ./bootstrap.sh check # Verify toolchains + ./bootstrap.sh # Full build + +=================== + +INSTRUCTIONS From d893143032154bac1dce0a505a9d8518312d49d5 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 11 Feb 2026 15:23:33 +0000 Subject: [PATCH 16/27] feat: build aztec-prover-agent with baked-in CRS (#20391) This PR builds a new image during release (tagged) builds specifically for the prover-agent with pre-baked CRS. This makes the image larger (by about 2GB) but will reduce the time spent on downloading the CRS at boot. Fix A-536 --- .github/workflows/deploy-network.yml | 2 + release-image/.gitignore | 1 + release-image/Dockerfile.prover-agent | 5 ++ .../Dockerfile.prover-agent.dockerignore | 2 + release-image/bootstrap.sh | 57 +++++++++++++++++++ .../aztec-node/templates/_pod-template.yaml | 4 +- spartan/aztec-node/values.yaml | 4 ++ spartan/scripts/deploy_network.sh | 1 + spartan/terraform/deploy-aztec-infra/main.tf | 8 +++ .../terraform/deploy-aztec-infra/variables.tf | 6 ++ 10 files changed, 88 insertions(+), 2 deletions(-) create mode 100644 release-image/Dockerfile.prover-agent create mode 100644 release-image/Dockerfile.prover-agent.dockerignore diff --git a/.github/workflows/deploy-network.yml b/.github/workflows/deploy-network.yml index 8ae5bf60d3de..7e8eb675081e 100644 --- a/.github/workflows/deploy-network.yml +++ b/.github/workflows/deploy-network.yml @@ -150,9 +150,11 @@ jobs: AZTEC_DOCKER_IMAGE: "aztecprotocol/aztec:${{ inputs.docker_image_tag || inputs.semver }}" CREATE_ROLLUP_CONTRACTS: ${{ inputs.deploy_contracts == true && 'true' || '' }} USE_NETWORK_CONFIG: ${{ inputs.deploy_contracts == true && 'false' || '' }} + PROVER_AGENT_DOCKER_IMAGE: "aztecprotocol/aztec-prover-agent:${{ inputs.docker_image_tag || inputs.semver }}" run: | echo "Deploying network: ${{ inputs.network }}" echo "Using image: $AZTEC_DOCKER_IMAGE" + echo "Using prover image: $PROVER_AGENT_DOCKER_IMAGE" echo "Using branch/ref: ${{ steps.checkout-ref.outputs.ref }}" cd spartan diff --git a/release-image/.gitignore b/release-image/.gitignore index 61295e1b932b..7fccdd711ed4 100644 --- a/release-image/.gitignore +++ b/release-image/.gitignore @@ -1 +1,2 @@ release-image-base +crs/ diff --git a/release-image/Dockerfile.prover-agent b/release-image/Dockerfile.prover-agent new file mode 100644 index 000000000000..c933f9f523c8 --- /dev/null +++ b/release-image/Dockerfile.prover-agent @@ -0,0 +1,5 @@ +ARG AZTEC_IMAGE_TAG +FROM aztecprotocol/aztec:${AZTEC_IMAGE_TAG} + +COPY --link crs/ /usr/src/crs/ +ENV CRS_PATH=/usr/src/crs diff --git a/release-image/Dockerfile.prover-agent.dockerignore b/release-image/Dockerfile.prover-agent.dockerignore new file mode 100644 index 000000000000..eb8f48b0b52c --- /dev/null +++ b/release-image/Dockerfile.prover-agent.dockerignore @@ -0,0 +1,2 @@ +* +!crs/ diff --git a/release-image/bootstrap.sh b/release-image/bootstrap.sh index 6096a5736980..7f71871d0099 100755 --- a/release-image/bootstrap.sh +++ b/release-image/bootstrap.sh @@ -3,6 +3,44 @@ source $(git rev-parse --show-toplevel)/ci3/source_bootstrap hash=$(cache_content_hash ^release-image/Dockerfile ^build-images/src/Dockerfile ^yarn-project/yarn.lock) +function prepare_crs { + echo_header "prepare crs for prover-agent image" + local crs_src=${CRS_PATH:-$HOME/.bb-crs} + + if [ ! -f "$crs_src/bn254_g1.dat" ]; then + # this assumes we pull the required number of points for proving the biggest circuit + echo "CRS not found at $crs_src, downloading..." + $root/barretenberg/scripts/download_bb_crs.sh + crs_src=$HOME/.bb-crs + fi + + mkdir -p crs + cp "$crs_src/bn254_g1.dat" crs/ + cp "$crs_src/bn254_g2.dat" crs/ + cp "$crs_src/grumpkin_g1.flat.dat" crs/ + # Normalize timestamps so COPY --link produces an identical layer across builds + for f in crs/*; do touch -t 197001010000 "$f"; done + echo "CRS files staged in crs/ ($(du -sh crs | cut -f1))" +} +export -f prepare_crs + +function build_prover_agent_image { + set -euo pipefail + local tag=$(git rev-parse HEAD) + + if ! docker image inspect aztecprotocol/aztec:$tag &>/dev/null; then + echo "Base image aztecprotocol/aztec:$tag not found. Run 'release-image/bootstrap.sh' first." + exit 1 + fi + + prepare_crs + echo_header "build prover-agent image" + docker build -f Dockerfile.prover-agent --build-arg AZTEC_IMAGE_TAG=$tag \ + -t aztecprotocol/aztec-prover-agent:$tag . + docker tag aztecprotocol/aztec-prover-agent:$tag aztecprotocol/aztec-prover-agent:latest +} +export -f build_prover_agent_image + function build_image { set -euo pipefail cd .. @@ -43,6 +81,10 @@ function build { fi denoise "build_image" + + if semver check "${REF_NAME:-}"; then + denoise "build_prover_agent_image" + fi } function test_cmds { @@ -64,6 +106,9 @@ function release { docker tag aztecprotocol/aztec:$COMMIT_HASH aztecprotocol/aztec:$tag-$(arch) do_or_dryrun docker push aztecprotocol/aztec:$tag-$(arch) + docker tag aztecprotocol/aztec-prover-agent:$COMMIT_HASH aztecprotocol/aztec-prover-agent:$tag-$(arch) + do_or_dryrun docker push aztecprotocol/aztec-prover-agent:$tag-$(arch) + # If doing a release in CI, update the remote manifest if we're the arm build. if [ "${DRY_RUN:-0}" == 0 ] && [ "$(arch)" == "arm64" ] && [ "${CI:-0}" -eq 1 ]; then # Wait for amd64 image to be available. @@ -77,6 +122,15 @@ function release { aztecprotocol/aztec:$tag-amd64 \ aztecprotocol/aztec:$tag-arm64 + while ! docker manifest inspect aztecprotocol/aztec-prover-agent:$tag-amd64 &>/dev/null; do + echo "Waiting for amd64 prover-agent image to be pushed..." + sleep 10 + done + + docker buildx imagetools create -t aztecprotocol/aztec-prover-agent:$tag \ + aztecprotocol/aztec-prover-agent:$tag-amd64 \ + aztecprotocol/aztec-prover-agent:$tag-arm64 + # We also release with our dist_tag, e.g. 'latest', 'staging' or 'nightly'. # docker buildx imagetools create -t aztecprotocol/aztec:$(dist_tag) \ # aztecprotocol/aztec:$tag-amd64 \ @@ -93,6 +147,7 @@ function push { fi echo $DOCKERHUB_PASSWORD | docker login -u ${DOCKERHUB_USERNAME:-aztecprotocolci} --password-stdin do_or_dryrun docker push aztecprotocol/aztec:$COMMIT_HASH + do_or_dryrun docker push aztecprotocol/aztec-prover-agent:$COMMIT_HASH } function push_pr { @@ -105,6 +160,8 @@ function push_pr { echo $DOCKERHUB_PASSWORD | docker login -u ${DOCKERHUB_USERNAME:-aztecprotocolci} --password-stdin docker tag aztecprotocol/aztec:$COMMIT_HASH aztecprotocol/aztecdev:$COMMIT_HASH do_or_dryrun docker push aztecprotocol/aztecdev:$COMMIT_HASH + docker tag aztecprotocol/aztec-prover-agent:$COMMIT_HASH aztecprotocol/aztec-prover-agent-dev:$COMMIT_HASH + do_or_dryrun docker push aztecprotocol/aztec-prover-agent-dev:$COMMIT_HASH } case "$cmd" in diff --git a/spartan/aztec-node/templates/_pod-template.yaml b/spartan/aztec-node/templates/_pod-template.yaml index b6989b8543ba..8d433f5577a3 100644 --- a/spartan/aztec-node/templates/_pod-template.yaml +++ b/spartan/aztec-node/templates/_pod-template.yaml @@ -1,6 +1,6 @@ {{- define "chart.podTemplate" }} -{{- $repo := .Values.global.aztecImage.repository | required ".Values.global.aztecImage.repository is required" }} -{{- $tag := .Values.global.aztecImage.tag | required ".Values.global.aztecImage.tag is required" }} +{{- $repo := .Values.node.image.repository | default .Values.global.aztecImage.repository | required ".Values.global.aztecImage.repository is required" }} +{{- $tag := .Values.node.image.tag | default .Values.global.aztecImage.tag | required ".Values.global.aztecImage.tag is required" }} {{- $image := printf "%s:%s" $repo $tag }} metadata: labels: diff --git a/spartan/aztec-node/values.yaml b/spartan/aztec-node/values.yaml index 8f1cb7af0ce2..735097b03781 100644 --- a/spartan/aztec-node/values.yaml +++ b/spartan/aztec-node/values.yaml @@ -81,6 +81,10 @@ node: - --node - --archiver + image: + repository: "" + tag: "" + env: {} envFrom: diff --git a/spartan/scripts/deploy_network.sh b/spartan/scripts/deploy_network.sh index beab348a291c..d9ef3fd2e13b 100755 --- a/spartan/scripts/deploy_network.sh +++ b/spartan/scripts/deploy_network.sh @@ -454,6 +454,7 @@ FULL_NODE_RESOURCE_PROFILE = "${FULL_NODE_RESOURCE_PROFILE}" ARCHIVE_RESOURCE_PROFILE = "${ARCHIVE_RESOURCE_PROFILE}" BLOB_SINK_RESOURCE_PROFILE = "${BLOB_SINK_RESOURCE_PROFILE}" AZTEC_DOCKER_IMAGE = "${AZTEC_DOCKER_IMAGE}" +PROVER_AGENT_DOCKER_IMAGE = "${PROVER_AGENT_DOCKER_IMAGE:-$AZTEC_DOCKER_IMAGE}" SPONSORED_FPC = ${SPONSORED_FPC} TEST_ACCOUNTS = ${TEST_ACCOUNTS} L1_CHAIN_ID = "${ETHEREUM_CHAIN_ID}" diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index d4518fc09787..8390addb91e4 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -84,6 +84,11 @@ locals { tag = split(":", var.AZTEC_DOCKER_IMAGE)[1] } + prover_agent_image = var.PROVER_AGENT_DOCKER_IMAGE != "" ? { + repository = split(":", var.PROVER_AGENT_DOCKER_IMAGE)[0] + tag = split(":", var.PROVER_AGENT_DOCKER_IMAGE)[1] + } : local.aztec_image + # Detect local kind context (e.g., "kind-kind") to gate Service types is_kind = can(regex("^kind", var.K8S_CLUSTER_CONTEXT)) @@ -328,6 +333,9 @@ locals { "broker.node.logLevel" = var.LOG_LEVEL "broker.node.env.BOOTSTRAP_NODES" = "asdf" "broker.node.env.PROVER_BROKER_DEBUG_REPLAY_ENABLED" = var.PROVER_BROKER_DEBUG_REPLAY_ENABLED + "agent.node.image.repository" = local.prover_agent_image.repository + "agent.node.image.tag" = local.prover_agent_image.tag + "agent.node.env.CRS_PATH" = "/usr/src/crs" "agent.node.proverRealProofs" = var.PROVER_REAL_PROOFS "agent.node.env.PROVER_AGENT_POLL_INTERVAL_MS" = var.PROVER_AGENT_POLL_INTERVAL_MS "agent.replicaCount" = var.PROVER_REPLICAS diff --git a/spartan/terraform/deploy-aztec-infra/variables.tf b/spartan/terraform/deploy-aztec-infra/variables.tf index 798f88005737..b8821306bde2 100644 --- a/spartan/terraform/deploy-aztec-infra/variables.tf +++ b/spartan/terraform/deploy-aztec-infra/variables.tf @@ -105,6 +105,12 @@ variable "AZTEC_DOCKER_IMAGE" { default = "aztecprotocol/aztec:staging" } +variable "PROVER_AGENT_DOCKER_IMAGE" { + description = "Docker image for prover agents (includes baked-in CRS). Defaults to AZTEC_DOCKER_IMAGE." + type = string + default = "" +} + variable "VALIDATOR_VALUES" { description = "The values file to apply" type = string From 2c21cbf0498fe289abd6146d03c54b8f4fc2910f Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 11 Feb 2026 15:27:17 +0000 Subject: [PATCH 17/27] fix!: change protocol contracts deployer to be the contract address --- .../protocol-contracts/src/make_protocol_contract.ts | 2 +- .../protocol-contracts/src/scripts/generate_data.ts | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/yarn-project/protocol-contracts/src/make_protocol_contract.ts b/yarn-project/protocol-contracts/src/make_protocol_contract.ts index ab18dcaf1b1d..14c3e0be7dbf 100644 --- a/yarn-project/protocol-contracts/src/make_protocol_contract.ts +++ b/yarn-project/protocol-contracts/src/make_protocol_contract.ts @@ -16,7 +16,7 @@ export async function makeProtocolContract( const salt = ProtocolContractSalt[name]; // TODO(@spalladino): This computes the contract class from the artifact twice. const contractClass = await getContractClassFromArtifact(artifact); - const instance = await getContractInstanceFromInstantiationParams(artifact, { salt }); + const instance = await getContractInstanceFromInstantiationParams(artifact, { salt, deployer: address }); return { instance: { ...instance, address }, contractClass, diff --git a/yarn-project/protocol-contracts/src/scripts/generate_data.ts b/yarn-project/protocol-contracts/src/scripts/generate_data.ts index 3e27ba86f4b5..e07a1f967cf8 100644 --- a/yarn-project/protocol-contracts/src/scripts/generate_data.ts +++ b/yarn-project/protocol-contracts/src/scripts/generate_data.ts @@ -63,8 +63,8 @@ async function copyArtifact(srcName: string, destName: string) { return artifact; } -async function computeAddress(artifact: NoirCompiledContract) { - const instance = await getContractInstanceFromInstantiationParams(loadContractArtifact(artifact), { salt }); +async function computeAddress(artifact: NoirCompiledContract, deployer: AztecAddress) { + const instance = await getContractInstanceFromInstantiationParams(loadContractArtifact(artifact), { salt, deployer }); return instance.address; } @@ -179,7 +179,7 @@ async function main() { const destName = destNames[i]; const artifact = await copyArtifact(srcName, destName); await generateDeclarationFile(destName); - derivedAddresses.push(await computeAddress(artifact)); + derivedAddresses.push(await computeAddress(artifact, AztecAddress.fromBigInt(contractAddressMapping[destName]))); } await generateOutputFile(destNames, derivedAddresses); From 2a20970e431ac035df0217662e240f25e76c7acc Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 11 Feb 2026 16:01:09 +0000 Subject: [PATCH 18/27] Remove duplicate code --- .../mem_pools/tx_pool_v2/tx_pool_indices.ts | 25 +++++++------------ 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts index 45bcc5b17774..b4f302001dcf 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts @@ -72,7 +72,7 @@ export class TxPoolIndices { * Iterates pending transaction hashes in priority order. * @param order - 'desc' for highest priority first, 'asc' for lowest priority first */ - *iteratePendingByPriority(order: 'asc' | 'desc'): Generator { + *iteratePendingByPriority(order: 'asc' | 'desc', filter?: (hash: string) => boolean): Generator { // Use compareFee from tx_metadata, swap args for descending order const feeCompareFn = order === 'desc' ? (a: bigint, b: bigint) => compareFee(b, a) : compareFee; const hashCompareFn = order === 'desc' ? (a: string, b: string) => compareTxHash(b, a) : compareTxHash; @@ -84,7 +84,9 @@ export class TxPoolIndices { // Use compareTxHash from tx_metadata, swap args for descending order const sortedHashes = [...hashesAtFee].sort(hashCompareFn); for (const hash of sortedHashes) { - yield hash; + if (filter === undefined || filter(hash)) { + yield hash; + } } } } @@ -95,21 +97,12 @@ export class TxPoolIndices { * @param maxReceivedAt - Only yield txs with receivedAt <= this value */ *iterateEligiblePendingByPriority(order: 'asc' | 'desc', maxReceivedAt: number): Generator { - const feeCompareFn = order === 'desc' ? (a: bigint, b: bigint) => compareFee(b, a) : compareFee; - const hashCompareFn = order === 'desc' ? (a: string, b: string) => compareTxHash(b, a) : compareTxHash; + const filter = (hash: string) => { + const meta = this.#metadata.get(hash); + return meta !== undefined && meta.receivedAt <= maxReceivedAt; + }; - const sortedFees = [...this.#pendingByPriority.keys()].sort(feeCompareFn); - - for (const fee of sortedFees) { - const hashesAtFee = this.#pendingByPriority.get(fee)!; - const sortedHashes = [...hashesAtFee].sort(hashCompareFn); - for (const hash of sortedHashes) { - const meta = this.#metadata.get(hash); - if (meta && meta.receivedAt <= maxReceivedAt) { - yield hash; - } - } - } + return this.iteratePendingByPriority(order, filter); } /** Iterates all metadata entries */ From a9d269b08d7397f33bf6acac6fc075e87fed0e67 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 11 Feb 2026 16:13:44 +0000 Subject: [PATCH 19/27] Fix --- yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts index b4f302001dcf..cf8291a17bad 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_indices.ts @@ -102,7 +102,7 @@ export class TxPoolIndices { return meta !== undefined && meta.receivedAt <= maxReceivedAt; }; - return this.iteratePendingByPriority(order, filter); + yield* this.iteratePendingByPriority(order, filter); } /** Iterates all metadata entries */ From edb69916fd3e7ffc39d16115eb3ba3b422fc5fec Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 11 Feb 2026 17:32:52 +0000 Subject: [PATCH 20/27] Try and fix test --- playground/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/playground/docker-compose.yml b/playground/docker-compose.yml index 339f3c5f6093..ead515cbcf93 100644 --- a/playground/docker-compose.yml +++ b/playground/docker-compose.yml @@ -26,6 +26,7 @@ services: SEQ_TX_POLLING_INTERVAL_MS: 50 WS_BLOCK_CHECK_INTERVAL_MS: 50 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 + P2P_MIN_TX_POOL_AGE_MS: 0 healthcheck: test: ['CMD', 'curl', '-fSs', 'http://127.0.0.1:8080/status'] interval: 3s From 45664e74b2ef27121954cca778d64c50a2ca15a4 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Wed, 11 Feb 2026 18:09:23 +0000 Subject: [PATCH 21/27] More test fixes --- yarn-project/end-to-end/scripts/docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/end-to-end/scripts/docker-compose.yml b/yarn-project/end-to-end/scripts/docker-compose.yml index bb926391df9d..528efb33a286 100644 --- a/yarn-project/end-to-end/scripts/docker-compose.yml +++ b/yarn-project/end-to-end/scripts/docker-compose.yml @@ -27,6 +27,7 @@ services: SEQ_POLLING_INTERVAL_MS: 500 WS_BLOCK_CHECK_INTERVAL_MS: 500 ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500 + P2P_MIN_TX_POOL_AGE_MS: 0 HARDWARE_CONCURRENCY: ${HARDWARE_CONCURRENCY:-} end-to-end: From 59d2acd64b416a435ab66016175cc41533e97cd7 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 11 Feb 2026 16:36:50 -0300 Subject: [PATCH 22/27] refactor(sentinel): update validator statuses to checkpoint-based naming (#20372) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary - Decouples block and checkpoint concepts in the sentinel for multi-block-per-slot support - Renames `block-mined`/`block-proposed` to `checkpoint-mined`/`checkpoint-proposed` - Splits `block-missed` into `blocks-missed` (no block proposals at all) and `checkpoint-missed` (blocks proposed but checkpoint not attested) - Adds `hasBlockProposalsForSlot` to P2P attestation pool and client to distinguish the two missed cases - Updates `ValidatorStatusType` from `'block' | 'attestation'` to `'proposer' | 'attestation'` - Bumps sentinel store schema version (2 → 3) with new serialization mappings - Updates all tests and slasher README documentation ## Test plan - Sentinel unit tests pass (36/36) - Store unit tests pass (8/8) - RPC interface tests pass (60/60) - E2E sentinel tests updated for new status names 🤖 Generated with [Claude Code](https://claude.com/claude-code) Fixes A-527 Co-authored-by: Claude Opus 4.6 --- .../docs/resources/migration_notes.md | 20 +++++ .../aztec-node/src/sentinel/sentinel.test.ts | 76 ++++++++++++------- .../aztec-node/src/sentinel/sentinel.ts | 58 +++++++++----- .../aztec-node/src/sentinel/store.test.ts | 39 +++++----- yarn-project/aztec-node/src/sentinel/store.ts | 24 +++--- yarn-project/end-to-end/src/e2e_bot.test.ts | 2 +- ...tiple_validators_sentinel.parallel.test.ts | 14 +++- .../src/e2e_p2p/validators_sentinel.test.ts | 4 +- yarn-project/p2p/src/client/interface.ts | 3 + yarn-project/p2p/src/client/p2p_client.ts | 4 + .../attestation_pool/attestation_pool.ts | 8 ++ .../p2p/src/test-helpers/testbench-utils.ts | 4 + yarn-project/slasher/README.md | 23 +++--- .../stdlib/src/interfaces/aztec-node.test.ts | 4 +- yarn-project/stdlib/src/validators/schemas.ts | 9 ++- yarn-project/stdlib/src/validators/types.ts | 9 ++- .../txe/src/state_machine/dummy_p2p_client.ts | 4 + 17 files changed, 206 insertions(+), 99 deletions(-) diff --git a/docs/docs-developers/docs/resources/migration_notes.md b/docs/docs-developers/docs/resources/migration_notes.md index addcb336e5e8..ea2afbf175da 100644 --- a/docs/docs-developers/docs/resources/migration_notes.md +++ b/docs/docs-developers/docs/resources/migration_notes.md @@ -97,6 +97,26 @@ Additionally, `debug_log_format_slice` has been removed. Use `debug_log_format` This has been done as usage of Noir slices is discouraged and the function was unused in the aztec codebase. +### [AztecNode] Sentinel validator status values renamed + +The `ValidatorStatusInSlot` values returned by `getValidatorsStats` and `getValidatorStats` have been updated to reflect the multi-block-per-slot model, where blocks and checkpoints are distinct concepts: + +```diff +- 'block-mined' ++ 'checkpoint-mined' + +- 'block-proposed' ++ 'checkpoint-proposed' + +- 'block-missed' ++ 'checkpoint-missed' // blocks were proposed but checkpoint was not attested ++ 'blocks-missed' // no block proposals were sent at all +``` + +The `attestation-sent` and `attestation-missed` values are unchanged but now explicitly refer to checkpoint attestations. + +The `ValidatorStatusType` used for categorizing statuses has also changed from `'block' | 'attestation'` to `'proposer' | 'attestation'`. + ### [aztec.js] `getDecodedPublicEvents` renamed to `getPublicEvents` with new signature The `getDecodedPublicEvents` function has been renamed to `getPublicEvents` and now uses a filter object instead of positional parameters: diff --git a/yarn-project/aztec-node/src/sentinel/sentinel.test.ts b/yarn-project/aztec-node/src/sentinel/sentinel.test.ts index 9a939a74b2b8..23bd50f032cd 100644 --- a/yarn-project/aztec-node/src/sentinel/sentinel.test.ts +++ b/yarn-project/aztec-node/src/sentinel/sentinel.test.ts @@ -117,25 +117,33 @@ describe('sentinel', () => { p2p.getCheckpointAttestationsForSlot.mockResolvedValue(attestations); }); - it('flags block as mined', async () => { + it('flags checkpoint as mined', async () => { // Create a checkpoint with a block at the target slot and emit chain-checkpointed event const checkpoint = await Checkpoint.random(CheckpointNumber(1), { numBlocks: 1, slotNumber: slot }); await emitCheckpointEvent(checkpoint); const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee); - expect(activity[proposer.toString()]).toEqual('block-mined'); + expect(activity[proposer.toString()]).toEqual('checkpoint-mined'); }); - it('flags block as proposed when it is not mined but there are attestations', async () => { + it('flags checkpoint as proposed when it is not mined but there are attestations', async () => { p2p.getCheckpointAttestationsForSlot.mockResolvedValue(attestations); const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee); - expect(activity[proposer.toString()]).toEqual('block-proposed'); + expect(activity[proposer.toString()]).toEqual('checkpoint-proposed'); }); - it('flags block as missed when there are no attestations', async () => { + it('flags as blocks-missed when there are no attestations and no block proposals', async () => { p2p.getCheckpointAttestationsForSlot.mockResolvedValue([]); + p2p.hasBlockProposalsForSlot.mockResolvedValue(false); const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee); - expect(activity[proposer.toString()]).toEqual('block-missed'); + expect(activity[proposer.toString()]).toEqual('blocks-missed'); + }); + + it('flags as checkpoint-missed when there are no attestations but block proposals exist', async () => { + p2p.getCheckpointAttestationsForSlot.mockResolvedValue([]); + p2p.hasBlockProposalsForSlot.mockResolvedValue(true); + const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee); + expect(activity[proposer.toString()]).toEqual('checkpoint-missed'); }); it('identifies attestors from p2p and archiver', async () => { @@ -208,8 +216,8 @@ describe('sentinel', () => { const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee); - // Validators 0 and 1 should be marked as having sent attestations (proposer is validator 0, so block-mined) - expect(activity[committee[0].toString()]).toEqual('block-mined'); + // Validators 0 and 1 should be marked as having sent attestations (proposer is validator 0, so checkpoint-mined) + expect(activity[committee[0].toString()]).toEqual('checkpoint-mined'); expect(activity[committee[1].toString()]).toEqual('attestation-sent'); // Validators 2 and 3 should be marked as having missed attestations (not counted as sent despite placeholders) @@ -242,9 +250,21 @@ describe('sentinel', () => { it('does not tag attestors as missed if there was no block and no attestations', async () => { p2p.getCheckpointAttestationsForSlot.mockResolvedValue([]); + p2p.hasBlockProposalsForSlot.mockResolvedValue(false); + + const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee); + expect(activity[proposer.toString()]).toEqual('blocks-missed'); + expect(activity[committee[1].toString()]).not.toBeDefined(); + expect(activity[committee[2].toString()]).not.toBeDefined(); + expect(activity[committee[3].toString()]).not.toBeDefined(); + }); + + it('does not tag attestors as missed if blocks were proposed but checkpoint was missed', async () => { + p2p.getCheckpointAttestationsForSlot.mockResolvedValue([]); + p2p.hasBlockProposalsForSlot.mockResolvedValue(true); const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee); - expect(activity[proposer.toString()]).toEqual('block-missed'); + expect(activity[proposer.toString()]).toEqual('checkpoint-missed'); expect(activity[committee[1].toString()]).not.toBeDefined(); expect(activity[committee[2].toString()]).not.toBeDefined(); expect(activity[committee[3].toString()]).not.toBeDefined(); @@ -260,10 +280,10 @@ describe('sentinel', () => { it('computes stats correctly', () => { const stats = sentinel.computeStatsForValidator(validator, [ - { slot: SlotNumber(1), status: 'block-mined' }, - { slot: SlotNumber(2), status: 'block-proposed' }, - { slot: SlotNumber(3), status: 'block-missed' }, - { slot: SlotNumber(4), status: 'block-missed' }, + { slot: SlotNumber(1), status: 'checkpoint-mined' }, + { slot: SlotNumber(2), status: 'checkpoint-proposed' }, + { slot: SlotNumber(3), status: 'checkpoint-missed' }, + { slot: SlotNumber(4), status: 'blocks-missed' }, { slot: SlotNumber(5), status: 'attestation-sent' }, { slot: SlotNumber(6), status: 'attestation-missed' }, ]); @@ -282,10 +302,10 @@ describe('sentinel', () => { it('resets streaks correctly', () => { const stats = sentinel.computeStatsForValidator(validator, [ - { slot: SlotNumber(1), status: 'block-mined' }, - { slot: SlotNumber(2), status: 'block-missed' }, - { slot: SlotNumber(3), status: 'block-mined' }, - { slot: SlotNumber(4), status: 'block-missed' }, + { slot: SlotNumber(1), status: 'checkpoint-mined' }, + { slot: SlotNumber(2), status: 'blocks-missed' }, + { slot: SlotNumber(3), status: 'checkpoint-mined' }, + { slot: SlotNumber(4), status: 'blocks-missed' }, { slot: SlotNumber(5), status: 'attestation-sent' }, { slot: SlotNumber(6), status: 'attestation-missed' }, { slot: SlotNumber(7), status: 'attestation-sent' }, @@ -303,7 +323,7 @@ describe('sentinel', () => { }); it('considers only latest slots', () => { - const history = times(20, i => ({ slot: SlotNumber(i), status: 'block-missed' }) as const); + const history = times(20, i => ({ slot: SlotNumber(i), status: 'blocks-missed' }) as const); const stats = sentinel.computeStatsForValidator(validator, history, SlotNumber(15)); expect(stats.address.toString()).toEqual(validator); @@ -312,7 +332,7 @@ describe('sentinel', () => { }); it('filters history by toSlot parameter', () => { - const history = times(20, i => ({ slot: SlotNumber(i), status: 'block-missed' }) as const); + const history = times(20, i => ({ slot: SlotNumber(i), status: 'blocks-missed' }) as const); const stats = sentinel.computeStatsForValidator(validator, history, SlotNumber(5), SlotNumber(10)); expect(stats.address.toString()).toEqual(validator); @@ -329,12 +349,12 @@ describe('sentinel', () => { validator = EthAddress.random(); jest.spyOn(store, 'getHistoryLength').mockReturnValue(10); jest.spyOn(store, 'getHistory').mockResolvedValue([ - { slot: SlotNumber(1), status: 'block-mined' }, + { slot: SlotNumber(1), status: 'checkpoint-mined' }, { slot: SlotNumber(2), status: 'attestation-sent' }, ]); jest.spyOn(store, 'getHistories').mockResolvedValue({ [validator.toString()]: [ - { slot: SlotNumber(1), status: 'block-mined' }, + { slot: SlotNumber(1), status: 'checkpoint-mined' }, { slot: SlotNumber(2), status: 'attestation-sent' }, ], }); @@ -369,7 +389,7 @@ describe('sentinel', () => { it('should return expected mocked data structure', async () => { const mockHistory: ValidatorStatusHistory = [ - { slot: SlotNumber(1), status: 'block-mined' }, + { slot: SlotNumber(1), status: 'checkpoint-mined' }, { slot: SlotNumber(2), status: 'attestation-sent' }, ]; const mockProvenPerformance = [ @@ -405,7 +425,7 @@ describe('sentinel', () => { }); it('should call computeStatsForValidator with correct parameters', async () => { - const mockHistory: ValidatorStatusHistory = [{ slot: SlotNumber(5), status: 'block-mined' }]; + const mockHistory: ValidatorStatusHistory = [{ slot: SlotNumber(5), status: 'checkpoint-mined' }]; jest.spyOn(store, 'getHistory').mockResolvedValue(mockHistory); jest.spyOn(store, 'getProvenPerformance').mockResolvedValue([]); const computeStatsSpy = jest.spyOn(sentinel, 'computeStatsForValidator').mockReturnValue({ @@ -422,7 +442,7 @@ describe('sentinel', () => { }); it('should use default slot range when not provided', async () => { - const mockHistory: ValidatorStatusHistory = [{ slot: SlotNumber(5), status: 'block-mined' }]; + const mockHistory: ValidatorStatusHistory = [{ slot: SlotNumber(5), status: 'checkpoint-mined' }]; jest.spyOn(store, 'getHistory').mockResolvedValue(mockHistory); jest.spyOn(store, 'getProvenPerformance').mockResolvedValue([]); const computeStatsSpy = jest.spyOn(sentinel, 'computeStatsForValidator').mockReturnValue({ @@ -444,7 +464,7 @@ describe('sentinel', () => { }); it('should not produce negative slot numbers when historyLength exceeds lastProcessedSlot', async () => { - const mockHistory: ValidatorStatusHistory = [{ slot: SlotNumber(2), status: 'block-mined' }]; + const mockHistory: ValidatorStatusHistory = [{ slot: SlotNumber(2), status: 'checkpoint-mined' }]; jest.spyOn(store, 'getHistory').mockResolvedValue(mockHistory); jest.spyOn(store, 'getProvenPerformance').mockResolvedValue([]); jest.spyOn(store, 'getHistoryLength').mockReturnValue(1000); // Large history length @@ -472,7 +492,7 @@ describe('sentinel', () => { }); it('should return proven performance data from store', async () => { - const mockHistory: ValidatorStatusHistory = [{ slot: SlotNumber(1), status: 'block-mined' }]; + const mockHistory: ValidatorStatusHistory = [{ slot: SlotNumber(1), status: 'checkpoint-mined' }]; const mockProvenPerformance = [ { epoch: EpochNumber(5), missed: 3, total: 12 }, { epoch: EpochNumber(6), missed: 0, total: 15 }, @@ -505,7 +525,7 @@ describe('sentinel', () => { it('should not produce negative slot numbers when historyLength exceeds lastProcessedSlot', async () => { const validator = EthAddress.random(); const mockHistories = { - [validator.toString()]: [{ slot: SlotNumber(2), status: 'block-mined' as const }], + [validator.toString()]: [{ slot: SlotNumber(2), status: 'checkpoint-mined' as const }], }; jest.spyOn(store, 'getHistories').mockResolvedValue(mockHistories); jest.spyOn(store, 'getHistoryLength').mockReturnValue(1000); // Large history length @@ -528,7 +548,7 @@ describe('sentinel', () => { const validator = EthAddress.random(); const mockHistories = { [validator.toString()]: [ - { slot: SlotNumber(95), status: 'block-mined' as const }, + { slot: SlotNumber(95), status: 'checkpoint-mined' as const }, { slot: SlotNumber(100), status: 'attestation-sent' as const }, ], }; diff --git a/yarn-project/aztec-node/src/sentinel/sentinel.ts b/yarn-project/aztec-node/src/sentinel/sentinel.ts index 18c6be46eb62..44e4dc80d022 100644 --- a/yarn-project/aztec-node/src/sentinel/sentinel.ts +++ b/yarn-project/aztec-node/src/sentinel/sentinel.ts @@ -36,6 +36,17 @@ import EventEmitter from 'node:events'; import { SentinelStore } from './store.js'; +/** Maps a validator status to its category: proposer or attestation. */ +function statusToCategory(status: ValidatorStatusInSlot): ValidatorStatusType { + switch (status) { + case 'attestation-sent': + case 'attestation-missed': + return 'attestation'; + default: + return 'proposer'; + } +} + export class Sentinel extends (EventEmitter as new () => WatcherEmitter) implements L2BlockStreamEventHandler, Watcher { protected runningPromise: RunningPromise; protected blockStream!: L2BlockStream; @@ -336,16 +347,16 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme // Check if there is an L2 block in L1 for this L2 slot - // Here we get all attestations for the block mined at the given slot, - // or all attestations for all proposals in the slot if no block was mined. + // Here we get all checkpoint attestations for the checkpoint at the given slot, + // or all checkpoint attestations for all proposals in the slot if no checkpoint was mined. // We gather from both p2p (contains the ones seen on the p2p layer) and archiver - // (contains the ones synced from mined blocks, which we may have missed from p2p). - const block = this.slotNumberToCheckpoint.get(slot); - const p2pAttested = await this.p2p.getCheckpointAttestationsForSlot(slot, block?.archive); + // (contains the ones synced from mined checkpoints, which we may have missed from p2p). + const checkpoint = this.slotNumberToCheckpoint.get(slot); + const p2pAttested = await this.p2p.getCheckpointAttestationsForSlot(slot, checkpoint?.archive); // Filter out attestations with invalid signatures const p2pAttestors = p2pAttested.map(a => a.getSender()).filter((s): s is EthAddress => s !== undefined); const attestors = new Set( - [...p2pAttestors.map(a => a.toString()), ...(block?.attestors.map(a => a.toString()) ?? [])].filter( + [...p2pAttestors.map(a => a.toString()), ...(checkpoint?.attestors.map(a => a.toString()) ?? [])].filter( addr => proposer.toString() !== addr, // Exclude the proposer from the attestors ), ); @@ -356,20 +367,29 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme // But we'll leave that corner case out to reduce pressure on the node. // TODO(palla/slash): This breaks if a given node has more than one validator in the current committee, // since they will attest to their own proposal it even if it's not re-executable. - const blockStatus = block ? 'mined' : attestors.size > 0 ? 'proposed' : 'missed'; - this.logger.debug(`Block for slot ${slot} was ${blockStatus}`, { ...block, slot }); + let status: 'checkpoint-mined' | 'checkpoint-proposed' | 'checkpoint-missed' | 'blocks-missed'; + if (checkpoint) { + status = 'checkpoint-mined'; + } else if (attestors.size > 0) { + status = 'checkpoint-proposed'; + } else { + // No checkpoint on L1 and no checkpoint attestations seen. Check if block proposals were sent for this slot. + const hasBlockProposals = await this.p2p.hasBlockProposalsForSlot(slot); + status = hasBlockProposals ? 'checkpoint-missed' : 'blocks-missed'; + } + this.logger.debug(`Checkpoint status for slot ${slot}: ${status}`, { ...checkpoint, slot }); - // Get attestors that failed their duties for this block, but only if there was a block proposed + // Get attestors that failed their checkpoint attestation duties, but only if there was a checkpoint proposed or mined const missedAttestors = new Set( - blockStatus === 'missed' + status === 'blocks-missed' || status === 'checkpoint-missed' ? [] : committee.filter(v => !attestors.has(v.toString()) && !proposer.equals(v)).map(v => v.toString()), ); this.logger.debug(`Retrieved ${attestors.size} attestors out of ${committee.length} for slot ${slot}`, { - blockStatus, + status, proposer: proposer.toString(), - ...block, + ...checkpoint, slot, attestors: [...attestors], missedAttestors: [...missedAttestors], @@ -379,7 +399,7 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme // Compute the status for each validator in the committee const statusFor = (who: `0x${string}`): ValidatorStatusInSlot | undefined => { if (who === proposer.toString()) { - return `block-${blockStatus}`; + return status; } else if (attestors.has(who)) { return 'attestation-sent'; } else if (missedAttestors.has(who)) { @@ -472,14 +492,16 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme ): ValidatorStats { let history = fromSlot ? allHistory.filter(h => BigInt(h.slot) >= fromSlot) : allHistory; history = toSlot ? history.filter(h => BigInt(h.slot) <= toSlot) : history; - const lastProposal = history.filter(h => h.status === 'block-proposed' || h.status === 'block-mined').at(-1); + const lastProposal = history + .filter(h => h.status === 'checkpoint-proposed' || h.status === 'checkpoint-mined') + .at(-1); const lastAttestation = history.filter(h => h.status === 'attestation-sent').at(-1); return { address: EthAddress.fromString(address), lastProposal: this.computeFromSlot(lastProposal?.slot), lastAttestation: this.computeFromSlot(lastAttestation?.slot), totalSlots: history.length, - missedProposals: this.computeMissed(history, 'block', ['block-missed']), + missedProposals: this.computeMissed(history, 'proposer', ['checkpoint-missed', 'blocks-missed']), missedAttestations: this.computeMissed(history, 'attestation', ['attestation-missed']), history, }; @@ -487,10 +509,12 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme protected computeMissed( history: ValidatorStatusHistory, - computeOverPrefix: ValidatorStatusType | undefined, + computeOverCategory: ValidatorStatusType | undefined, filter: ValidatorStatusInSlot[], ) { - const relevantHistory = history.filter(h => !computeOverPrefix || h.status.startsWith(computeOverPrefix)); + const relevantHistory = history.filter( + h => !computeOverCategory || statusToCategory(h.status) === computeOverCategory, + ); const filteredHistory = relevantHistory.filter(h => filter.includes(h.status)); return { currentStreak: countWhile([...relevantHistory].reverse(), h => filter.includes(h.status)), diff --git a/yarn-project/aztec-node/src/sentinel/store.test.ts b/yarn-project/aztec-node/src/sentinel/store.test.ts index 8525b428b23d..0c7babb7ce99 100644 --- a/yarn-project/aztec-node/src/sentinel/store.test.ts +++ b/yarn-project/aztec-node/src/sentinel/store.test.ts @@ -27,11 +27,12 @@ describe('sentinel-store', () => { it('inserts new validators with all statuses', async () => { const slot = SlotNumber(1); - const validators: `0x${string}`[] = times(5, () => EthAddress.random().toString()); + const validators: `0x${string}`[] = times(6, () => EthAddress.random().toString()); const statuses: ValidatorStatusInSlot[] = [ - 'block-mined', - 'block-proposed', - 'block-missed', + 'checkpoint-mined', + 'checkpoint-proposed', + 'checkpoint-missed', + 'blocks-missed', 'attestation-sent', 'attestation-missed', ]; @@ -58,15 +59,15 @@ describe('sentinel-store', () => { // Insert existing validators with initial statuses await store.updateValidators( SlotNumber(1), - Object.fromEntries(existingValidators.map(v => [v, 'block-mined'] as const)), + Object.fromEntries(existingValidators.map(v => [v, 'checkpoint-mined'] as const)), ); // Insert new validators with their statuses, and append history to existing ones await store.updateValidators( SlotNumber(2), Object.fromEntries([ - ...newValidators.map(v => [v, 'block-proposed'] as const), - ...existingValidators.map(v => [v, 'block-missed'] as const), + ...newValidators.map(v => [v, 'checkpoint-proposed'] as const), + ...existingValidators.map(v => [v, 'checkpoint-missed'] as const), ]), ); @@ -74,17 +75,17 @@ describe('sentinel-store', () => { expect(Object.keys(histories)).toHaveLength(4); expect(histories[existingValidators[0]]).toEqual([ - { slot: SlotNumber(1), status: 'block-mined' }, - { slot: SlotNumber(2), status: 'block-missed' }, + { slot: SlotNumber(1), status: 'checkpoint-mined' }, + { slot: SlotNumber(2), status: 'checkpoint-missed' }, ]); expect(histories[existingValidators[1]]).toEqual([ - { slot: SlotNumber(1), status: 'block-mined' }, - { slot: SlotNumber(2), status: 'block-missed' }, + { slot: SlotNumber(1), status: 'checkpoint-mined' }, + { slot: SlotNumber(2), status: 'checkpoint-missed' }, ]); - expect(histories[newValidators[0]]).toEqual([{ slot: SlotNumber(2), status: 'block-proposed' }]); - expect(histories[newValidators[1]]).toEqual([{ slot: SlotNumber(2), status: 'block-proposed' }]); + expect(histories[newValidators[0]]).toEqual([{ slot: SlotNumber(2), status: 'checkpoint-proposed' }]); + expect(histories[newValidators[1]]).toEqual([{ slot: SlotNumber(2), status: 'checkpoint-proposed' }]); }); it('trims history to the specified length', async () => { @@ -92,16 +93,16 @@ describe('sentinel-store', () => { const validator = EthAddress.random().toString(); for (let i = 0; i < 10; i++) { - await store.updateValidators(SlotNumber(slot + i), { [validator]: 'block-mined' }); + await store.updateValidators(SlotNumber(slot + i), { [validator]: 'checkpoint-mined' }); } const histories = await store.getHistories(); expect(histories[validator]).toHaveLength(historyLength); expect(histories[validator]).toEqual([ - { slot: SlotNumber(7), status: 'block-mined' }, - { slot: SlotNumber(8), status: 'block-mined' }, - { slot: SlotNumber(9), status: 'block-mined' }, - { slot: SlotNumber(10), status: 'block-mined' }, + { slot: SlotNumber(7), status: 'checkpoint-mined' }, + { slot: SlotNumber(8), status: 'checkpoint-mined' }, + { slot: SlotNumber(9), status: 'checkpoint-mined' }, + { slot: SlotNumber(10), status: 'checkpoint-mined' }, ]); }); @@ -207,6 +208,6 @@ describe('sentinel-store', () => { await expect( store.updateProvenPerformance(EpochNumber(1), { [validator]: { missed: 2, total: 10 } }), ).rejects.toThrow(); - await expect(store.updateValidators(SlotNumber(1), { [validator]: 'block-mined' })).rejects.toThrow(); + await expect(store.updateValidators(SlotNumber(1), { [validator]: 'checkpoint-mined' })).rejects.toThrow(); }); }); diff --git a/yarn-project/aztec-node/src/sentinel/store.ts b/yarn-project/aztec-node/src/sentinel/store.ts index 7d2c83138f88..c06be27cb179 100644 --- a/yarn-project/aztec-node/src/sentinel/store.ts +++ b/yarn-project/aztec-node/src/sentinel/store.ts @@ -9,7 +9,7 @@ import type { } from '@aztec/stdlib/validators'; export class SentinelStore { - public static readonly SCHEMA_VERSION = 2; + public static readonly SCHEMA_VERSION = 3; // a map from validator address to their ValidatorStatusHistory private readonly historyMap: AztecAsyncMap<`0x${string}`, Buffer>; @@ -86,11 +86,7 @@ export class SentinelStore { }); } - private async pushValidatorStatusForSlot( - who: EthAddress, - slot: SlotNumber, - status: 'block-mined' | 'block-proposed' | 'block-missed' | 'attestation-sent' | 'attestation-missed', - ) { + private async pushValidatorStatusForSlot(who: EthAddress, slot: SlotNumber, status: ValidatorStatusInSlot) { await this.store.transactionAsync(async () => { const currentHistory = (await this.getHistory(who)) ?? []; const newHistory = [...currentHistory, { slot, status }].slice(-this.config.historyLength); @@ -149,16 +145,18 @@ export class SentinelStore { private statusToNumber(status: ValidatorStatusInSlot): number { switch (status) { - case 'block-mined': + case 'checkpoint-mined': return 1; - case 'block-proposed': + case 'checkpoint-proposed': return 2; - case 'block-missed': + case 'checkpoint-missed': return 3; case 'attestation-sent': return 4; case 'attestation-missed': return 5; + case 'blocks-missed': + return 6; default: { const _exhaustive: never = status; throw new Error(`Unknown status: ${status}`); @@ -169,15 +167,17 @@ export class SentinelStore { private statusFromNumber(status: number): ValidatorStatusInSlot { switch (status) { case 1: - return 'block-mined'; + return 'checkpoint-mined'; case 2: - return 'block-proposed'; + return 'checkpoint-proposed'; case 3: - return 'block-missed'; + return 'checkpoint-missed'; case 4: return 'attestation-sent'; case 5: return 'attestation-missed'; + case 6: + return 'blocks-missed'; default: throw new Error(`Unknown status: ${status}`); } diff --git a/yarn-project/end-to-end/src/e2e_bot.test.ts b/yarn-project/end-to-end/src/e2e_bot.test.ts index 08d546dbb133..8628a7ebdeb9 100644 --- a/yarn-project/end-to-end/src/e2e_bot.test.ts +++ b/yarn-project/end-to-end/src/e2e_bot.test.ts @@ -219,7 +219,7 @@ describe('e2e_bot', () => { beforeAll(() => { config = { ...getBotDefaultConfig(), - followChain: 'CHECKPOINTED', + followChain: 'PROPOSED', ammTxs: false, senderPrivateKey: new SecretValue(Fr.random()), l1PrivateKey: new SecretValue(bufferToHex(getPrivateKeyFromIndex(8)!)), diff --git a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts index 94e33fc0d53e..7dc521775561 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts @@ -186,14 +186,18 @@ describe('e2e_p2p_multiple_validators_sentinel', () => { const validatorStats = stats.stats[validator.toString().toLowerCase()]; const history = validatorStats?.history.filter(h => h.slot > initialSlot && h.slot <= slotForSentinel) ?? []; t.logger.info(`Asserting stats for online validator ${validator}`, { history }); - expect(history.filter(h => h.status === 'attestation-missed' || h.status === 'block-missed')).toBeEmpty(); + expect( + history.filter( + h => h.status === 'attestation-missed' || h.status === 'blocks-missed' || h.status === 'checkpoint-missed', + ), + ).toBeEmpty(); } // At least one of the first node validators must have been seen as proposer const firstNodeBlockProposedHistory = firstNodeValidators .flatMap(v => stats.stats[v.toString().toLowerCase()].history) .filter(h => h.slot > initialSlot && h.slot <= slotForSentinel) - .filter(h => h.status === 'block-proposed'); + .filter(h => h.status === 'checkpoint-proposed'); expect(firstNodeBlockProposedHistory).not.toBeEmpty(); // And all of the proposers for the offline node must be seen as missed attestation or proposal @@ -201,7 +205,11 @@ describe('e2e_p2p_multiple_validators_sentinel', () => { const validatorStats = stats.stats[validator.toString().toLowerCase()]; const history = validatorStats.history?.filter(h => h.slot > initialSlot && h.slot <= slotForSentinel) ?? []; t.logger.info(`Asserting stats for offline validator ${validator}`, { history }); - expect(history.filter(h => h.status === 'attestation-missed' || h.status === 'block-missed')).not.toBeEmpty(); + expect( + history.filter( + h => h.status === 'attestation-missed' || h.status === 'blocks-missed' || h.status === 'checkpoint-missed', + ), + ).not.toBeEmpty(); } }); }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/validators_sentinel.test.ts b/yarn-project/end-to-end/src/e2e_p2p/validators_sentinel.test.ts index 0b62ad197753..3a4e47afe387 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/validators_sentinel.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/validators_sentinel.test.ts @@ -103,7 +103,7 @@ describe('e2e_p2p_validators_sentinel', () => { initialSlot && lastProcessedSlot && lastProcessedSlot - initialSlot >= blockCount - 1 && - Object.values(stats).some(stat => stat.history.some(h => h.status === 'block-mined')) && + Object.values(stats).some(stat => stat.history.some(h => h.status === 'checkpoint-mined')) && Object.values(stats).some(stat => stat.history.some(h => h.status === 'attestation-sent')) && stats[offlineValidator.toString().toLowerCase()] && stats[offlineValidator.toString().toLowerCase()].history.length > 0 && @@ -132,7 +132,7 @@ describe('e2e_p2p_validators_sentinel', () => { it('collects stats on a block builder', () => { const [proposerValidator, proposerStats] = Object.entries(stats.stats).find(([_, v]) => - v?.history?.some(h => h.status === 'block-mined'), + v?.history?.some(h => h.status === 'checkpoint-mined'), )!; t.logger.info(`Asserting stats for proposer validator ${proposerValidator}`); expect(proposerStats).toBeDefined(); diff --git a/yarn-project/p2p/src/client/interface.ts b/yarn-project/p2p/src/client/interface.ts index 73d0d870e887..f70316f88715 100644 --- a/yarn-project/p2p/src/client/interface.ts +++ b/yarn-project/p2p/src/client/interface.ts @@ -236,6 +236,9 @@ export type P2P = P2PApiFull & handleAuthRequestFromPeer(authRequest: AuthRequest, peerId: PeerId): Promise; + /** Checks if any block proposals exist for the given slot. */ + hasBlockProposalsForSlot(slot: SlotNumber): Promise; + /** If node running this P2P stack is validator, passes in validator address to P2P layer */ registerThisValidatorAddresses(address: EthAddress[]): void; }; diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index 966d11119782..588eccbc2269 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -404,6 +404,10 @@ export class P2PClient return this.attestationPool.addOwnCheckpointAttestations(attestations); } + public hasBlockProposalsForSlot(slot: SlotNumber): Promise { + return this.attestationPool.hasBlockProposalsForSlot(slot); + } + // REVIEW: https://github.com/AztecProtocol/aztec-packages/issues/7963 // ^ This pattern is not my favorite (md) public registerBlockProposalHandler(handler: P2PBlockReceivedCallback): void { diff --git a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts index 7f4626a035c7..827d91ce84ea 100644 --- a/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts +++ b/yarn-project/p2p/src/mem_pools/attestation_pool/attestation_pool.ts @@ -43,6 +43,7 @@ export type AttestationPoolApi = Pick< | 'deleteOlderThan' | 'getCheckpointAttestationsForSlot' | 'getCheckpointAttestationsForSlotAndProposal' + | 'hasBlockProposalsForSlot' | 'isEmpty' >; @@ -254,6 +255,13 @@ export class AttestationPool { return undefined; } + /** Checks if any block proposals exist for a given slot (at index 0). */ + public async hasBlockProposalsForSlot(slot: SlotNumber): Promise { + const positionKey = this.getBlockPositionKey(slot, 0); + const count = await this.blockProposalsForSlotAndIndex.getValueCountAsync(positionKey); + return count > 0; + } + /** * Attempts to add a checkpoint proposal to the pool. * diff --git a/yarn-project/p2p/src/test-helpers/testbench-utils.ts b/yarn-project/p2p/src/test-helpers/testbench-utils.ts index c8c81dc7b997..d0b8ce352fde 100644 --- a/yarn-project/p2p/src/test-helpers/testbench-utils.ts +++ b/yarn-project/p2p/src/test-helpers/testbench-utils.ts @@ -256,6 +256,10 @@ export class InMemoryAttestationPool { return Promise.resolve({ added: true, alreadyExists: false, count: 1 }); } + hasBlockProposalsForSlot(_slot: SlotNumber): Promise { + return Promise.resolve(false); + } + isEmpty(): Promise { return Promise.resolve(this.proposals.size === 0); } diff --git a/yarn-project/slasher/README.md b/yarn-project/slasher/README.md index 3de956862489..dbd4454cf1eb 100644 --- a/yarn-project/slasher/README.md +++ b/yarn-project/slasher/README.md @@ -202,21 +202,24 @@ Details about specific offenses in the system: Inactivity slashing is one of the most critical, since it allows purging validators that are not fulfilling their duties, which could potentially bring the chain to a halt. This slashing must be aggressive enough to balance out the rate of the entry queue, in case the queue is filled with inactive validators. Furthermore, if enough inactive validators join the system, it may become impossible to gather enough quorum to pass any governance proposal. -Inactivity slashing is handled by the `Sentinel` which monitors performance of all validators slot-by-slot. After each slot, the sentinel assigns one of the following to the block proposer for the slot: -- `block-mined` if the block was added to L1 -- `block-proposed` if the block received at least one attestation, but didn't make it to L1 -- `block-missed` if the block received no attestations (note that we cannot rely on the P2P proposal alone since it may be invalid, unless we reexecute it) +Inactivity slashing is handled by the `Sentinel` which monitors performance of all validators slot-by-slot. With the multiple-blocks-per-slot model, block proposals and checkpoints are distinct concepts: proposers build multiple blocks per slot, but attestations are only for checkpoints. After each slot, the sentinel assigns one of the following to the proposer for the slot: +- `checkpoint-mined` if the checkpoint was added to L1 +- `checkpoint-proposed` if the checkpoint received at least one attestation, but didn't make it to L1 +- `checkpoint-missed` if blocks were proposed but the checkpoint received no attestations +- `blocks-missed` if no block proposals were sent for this slot at all -And assigns one of the following to each validator: -- `attestation-sent` if there was a `block-proposed` or `block-mined` and an attestation from this validator was seen on either on L1 or on the P2P network -- `attestation-missed` if there was a `block-proposed` or `block-mined` but no attestation was seen -- none if the slot was a `block-missed` +And assigns one of the following to each validator (these refer to checkpoint attestations): +- `attestation-sent` if there was a `checkpoint-proposed` or `checkpoint-mined` and a checkpoint attestation from this validator was seen on either on L1 or on the P2P network +- `attestation-missed` if there was a `checkpoint-proposed` or `checkpoint-mined` but no checkpoint attestation was seen +- none if the slot was a `blocks-missed` + +Both `blocks-missed` and `checkpoint-missed` count as proposer inactivity. Once an epoch is proven, the sentinel computes the _proven performance_ for the epoch for each validator. Note that we wait until the epoch is proven so we know that the data for all blocks in the epoch was available, and validators who did not attest were effectively inactive. Then, for each validator such that: ``` -total_failures = count(block-missed) + count(attestation-missed) -total = count(block-*) + count(attestation-*) +total_failures = count(blocks-missed) + count(checkpoint-missed) + count(attestation-missed) +total = count(checkpoint-*) + count(blocks-*) + count(attestation-*) total_failures / total >= slash_inactivity_target_percentage ``` diff --git a/yarn-project/stdlib/src/interfaces/aztec-node.test.ts b/yarn-project/stdlib/src/interfaces/aztec-node.test.ts index 139b195a95d5..2243efd963dc 100644 --- a/yarn-project/stdlib/src/interfaces/aztec-node.test.ts +++ b/yarn-project/stdlib/src/interfaces/aztec-node.test.ts @@ -360,7 +360,7 @@ describe('AztecNodeApiSchema', () => { count: 1, total: 1, }, - history: [{ slot: SlotNumber(1), status: 'block-mined' }], + history: [{ slot: SlotNumber(1), status: 'checkpoint-mined' }], }, }, lastProcessedSlot: SlotNumber(20), @@ -407,7 +407,7 @@ describe('AztecNodeApiSchema', () => { totalSlots: 5, missedAttestations: { currentStreak: 0, count: 0, total: 1 }, missedProposals: { currentStreak: 0, count: 0, total: 1 }, - history: [{ slot: SlotNumber(1), status: 'block-mined' }], + history: [{ slot: SlotNumber(1), status: 'checkpoint-mined' }], }, allTimeProvenPerformance: [], lastProcessedSlot: SlotNumber(10), diff --git a/yarn-project/stdlib/src/validators/schemas.ts b/yarn-project/stdlib/src/validators/schemas.ts index 6deb8c61fde4..1201aecfd05b 100644 --- a/yarn-project/stdlib/src/validators/schemas.ts +++ b/yarn-project/stdlib/src/validators/schemas.ts @@ -12,7 +12,14 @@ import type { } from './types.js'; export const ValidatorStatusInSlotSchema = zodFor()( - z.enum(['block-mined', 'block-proposed', 'block-missed', 'attestation-sent', 'attestation-missed']), + z.enum([ + 'checkpoint-mined', + 'checkpoint-proposed', + 'checkpoint-missed', + 'blocks-missed', + 'attestation-sent', + 'attestation-missed', + ]), ); export const ValidatorStatusHistorySchema = zodFor()( diff --git a/yarn-project/stdlib/src/validators/types.ts b/yarn-project/stdlib/src/validators/types.ts index 4e4309dc2866..bf005e4537d1 100644 --- a/yarn-project/stdlib/src/validators/types.ts +++ b/yarn-project/stdlib/src/validators/types.ts @@ -1,12 +1,13 @@ import type { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import type { EthAddress } from '@aztec/foundation/eth-address'; -export type ValidatorStatusType = 'block' | 'attestation'; +export type ValidatorStatusType = 'proposer' | 'attestation'; export type ValidatorStatusInSlot = - | 'block-mined' - | 'block-proposed' - | 'block-missed' + | 'checkpoint-mined' + | 'checkpoint-proposed' + | 'checkpoint-missed' + | 'blocks-missed' | 'attestation-sent' | 'attestation-missed'; diff --git a/yarn-project/txe/src/state_machine/dummy_p2p_client.ts b/yarn-project/txe/src/state_machine/dummy_p2p_client.ts index 8d4fcd249100..fa6615cc8b13 100644 --- a/yarn-project/txe/src/state_machine/dummy_p2p_client.ts +++ b/yarn-project/txe/src/state_machine/dummy_p2p_client.ts @@ -216,4 +216,8 @@ export class DummyP2P implements P2P { public registerDuplicateAttestationCallback(_callback: P2PDuplicateAttestationCallback): void { throw new Error('DummyP2P does not implement "registerDuplicateAttestationCallback"'); } + + public hasBlockProposalsForSlot(_slot: SlotNumber): Promise { + throw new Error('DummyP2P does not implement "hasBlockProposalsForSlot"'); + } } From 10fac722a10d06c8ea2416f65f800afeb402509b Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 12 Feb 2026 10:51:11 +0000 Subject: [PATCH 23/27] chore: log tx hash (#20413) Log tx hashes when removing txs from the mempool --- .../src/mem_pools/tx_pool_v2/deleted_pool.ts | 7 +++++-- .../fee_payer_balance_eviction_rule.ts | 3 +++ .../eviction/invalid_txs_after_mining_rule.ts | 2 +- .../eviction/invalid_txs_after_reorg_rule.ts | 2 +- .../eviction/low_priority_eviction_rule.ts | 4 +++- .../mem_pools/tx_pool_v2/tx_pool_v2_impl.ts | 18 ++++++++++++++---- 6 files changed, 27 insertions(+), 9 deletions(-) diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts index e4eea8793967..9227e0b32809 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/deleted_pool.ts @@ -142,7 +142,7 @@ export class DeletedPool { } if (count > 0) { - this.#log.debug(`Marked ${count} transactions from pruned blocks`); + this.#log.info(`Marked ${count} transactions from pruned blocks`); } } @@ -237,7 +237,9 @@ export class DeletedPool { await this.#txsDB.delete(txHash); } - this.#log.debug(`Finalized ${toHardDelete.length} txs from pruned blocks at block ${finalizedBlockNumber}`); + this.#log.debug(`Finalized ${toHardDelete.length} txs from pruned blocks at block ${finalizedBlockNumber}`, { + txHashes: toHardDelete, + }); return toHardDelete; } @@ -269,6 +271,7 @@ export class DeletedPool { this.#log.debug( `Cleaned up ${toHardDelete.length} slot-deleted txs from slot ${previousSlot} (now slot ${currentSlot})`, + { txHashes: toHardDelete }, ); } diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts index 7cd1b48a4a08..32a5db700677 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/fee_payer_balance_eviction_rule.ts @@ -68,6 +68,9 @@ export class FeePayerBalanceEvictionRule implements EvictionRule { if (txsToEvict.length > 0) { await pool.deleteTxs(txsToEvict); + this.log.verbose(`Evicted ${txsToEvict.length} txs due to insufficient fee payer balance`, { + txHashes: txsToEvict, + }); } return { diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts index 593898ef9182..8c39393f48af 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_mining_rule.ts @@ -54,7 +54,7 @@ export class InvalidTxsAfterMiningRule implements EvictionRule { await pool.deleteTxs(txsToEvict); } - this.log.debug(`Evicted ${txsToEvict.length} invalid txs after block mined`); + this.log.debug(`Evicted ${txsToEvict.length} invalid txs after block mined`, { txHashes: txsToEvict }); return { reason: 'block_mined_invalid_txs', diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts index 7e4630e55269..f76c23eb7b1e 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/invalid_txs_after_reorg_rule.ts @@ -81,7 +81,7 @@ export class InvalidTxsAfterReorgRule implements EvictionRule { this.log.verbose(`Kept ${keptCount} txs that did not reference pruned blocks`); } - this.log.debug(`Evicted ${txsToEvict.length} invalid txs after reorg`); + this.log.info(`Evicted ${txsToEvict.length} invalid txs after reorg`, { txHashes: txsToEvict }); return { reason: 'reorg_invalid_txs', diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts index c9a015abe3ad..c9854aeb6a17 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/eviction/low_priority_eviction_rule.ts @@ -60,7 +60,9 @@ export class LowPriorityEvictionRule implements EvictionRule { const numNewTxsEvicted = context.newTxHashes.filter(newTxHash => txsToEvict.includes(newTxHash)).length; - this.log.verbose(`Evicted ${txsToEvict.length} low priority txs, including ${numNewTxsEvicted} newly added txs`); + this.log.verbose(`Evicted ${txsToEvict.length} low priority txs, including ${numNewTxsEvicted} newly added txs`, { + txHashes: txsToEvict, + }); return { reason: 'low_priority', diff --git a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts index bc07801d8aaa..31979f3ea1a1 100644 --- a/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts +++ b/yarn-project/p2p/src/mem_pools/tx_pool_v2/tx_pool_v2_impl.ts @@ -168,7 +168,7 @@ export class TxPoolV2Impl { await this.#txsDB.delete(txHashStr); } }); - this.#log.info(`Deleted ${toDelete.length} invalid/rejected transactions on startup`); + this.#log.info(`Deleted ${toDelete.length} invalid/rejected transactions on startup`, { txHashes: toDelete }); } async addPendingTxs(txs: Tx[], opts: { source?: string }): Promise { @@ -259,7 +259,10 @@ export class TxPoolV2Impl { // Evict conflicts for (const evictHashStr of preAddResult.txHashesToEvict) { await this.#deleteTx(evictHashStr); - this.#log.debug(`Evicted tx ${evictHashStr} due to higher-fee tx ${txHashStr}`); + this.#log.debug(`Evicted tx ${evictHashStr} due to higher-fee tx ${txHashStr}`, { + evictedTxHash: evictHashStr, + replacementTxHash: txHashStr, + }); if (acceptedPending.has(evictHashStr)) { // Evicted tx was from this batch - mark as ignored in result acceptedPending.delete(evictHashStr); @@ -471,6 +474,11 @@ export class TxPoolV2Impl { // Step 7: Delete invalid and evicted txs await this.#deleteTxsBatch([...invalid, ...toEvict]); + this.#log.info( + `Handled prune to block ${latestBlock.number}: ${valid.length} txs restored to pending, ${invalid.length} invalid, ${toEvict.length} evicted due to nullifier conflicts`, + { txHashesRestored: valid.map(m => m.txHash), txHashesInvalid: invalid, txHashesEvicted: toEvict }, + ); + // Step 8: Run eviction rules for ALL pending txs (not just restored ones) // This handles cases like existing pending txs with invalid fee payer balances await this.#evictionManager.evictAfterChainPrune(latestBlock.number); @@ -480,7 +488,7 @@ export class TxPoolV2Impl { // Delete failed txs await this.#deleteTxsBatch(txHashes.map(h => h.toString())); - this.#log.info(`Deleted ${txHashes.length} failed txs`); + this.#log.info(`Deleted ${txHashes.length} failed txs`, { txHashes: txHashes.map(h => h.toString()) }); } async handleFinalizedBlock(block: BlockHeader): Promise { @@ -512,7 +520,9 @@ export class TxPoolV2Impl { } if (minedTxsToFinalize.length > 0) { - this.#log.info(`Finalized ${minedTxsToFinalize.length} mined txs from blocks up to ${blockNumber}`); + this.#log.info(`Finalized ${minedTxsToFinalize.length} mined txs from blocks up to ${blockNumber}`, { + txHashes: minedTxsToFinalize, + }); } } From 7b6dc47e2f91d25f136741f7feb3b86b2e994f5d Mon Sep 17 00:00:00 2001 From: spypsy <6403450+spypsy@users.noreply.github.com> Date: Thu, 12 Feb 2026 10:56:39 +0000 Subject: [PATCH 24/27] chore: update l1 fee analysis to measure blob count in L1 blocks related to A-351. Adds some more metrics for fee analysis to count number of blobs that were in the finalized L1 block. This will help get better stats on inclusion rate per strategy during high blob congestion --- .../metrics/grafana/dashboards/l1_fees.json | 339 +++++++++++++++++- .../sequencer-client/src/sequencer/metrics.ts | 24 ++ yarn-project/telemetry-client/src/metrics.ts | 17 + 3 files changed, 375 insertions(+), 5 deletions(-) diff --git a/spartan/metrics/grafana/dashboards/l1_fees.json b/spartan/metrics/grafana/dashboards/l1_fees.json index 36d13035cfea..33747c63c2ed 100644 --- a/spartan/metrics/grafana/dashboards/l1_fees.json +++ b/spartan/metrics/grafana/dashboards/l1_fees.json @@ -985,13 +985,342 @@ "title": "Blob Transaction Counts", "type": "timeseries" }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "Total number of blobs in pending and included blocks", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "blobs" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Pending" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Included" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 37 + }, + "id": 33, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "expr": "sum(increase(aztec_fisherman_fee_analysis_pending_blob_count_sum{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) / sum(increase(aztec_fisherman_fee_analysis_pending_blob_count_count{k8s_namespace_name=\"$namespace\"}[$__rate_interval]))", + "legendFormat": "Pending", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "expr": "sum(increase(aztec_fisherman_fee_analysis_included_blob_count_sum{k8s_namespace_name=\"$namespace\"}[$__rate_interval])) / sum(increase(aztec_fisherman_fee_analysis_included_blob_count_count{k8s_namespace_name=\"$namespace\"}[$__rate_interval]))", + "legendFormat": "Included", + "range": true, + "refId": "B" + } + ], + "title": "Total Blob Counts", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "Percentage of blocks that reached 100% blob capacity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "yellow", + "value": 0.5 + }, + { + "color": "red", + "value": 0.8 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 37 + }, + "id": 34, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "expr": "sum(increase(aztec_fisherman_fee_analysis_block_blobs_full{k8s_namespace_name=\"$namespace\",aztec_ok=\"true\"}[$__range])) / sum(increase(aztec_fisherman_fee_analysis_block_blobs_full{k8s_namespace_name=\"$namespace\"}[$__range]))", + "legendFormat": "Blocks Full Rate", + "range": true, + "refId": "A" + } + ], + "title": "Block Blob Capacity Full Rate", + "type": "gauge" + }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 37 + "y": 45 + }, + "id": 60, + "panels": [], + "title": "Full Block Analysis", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "description": "Inclusion rate by strategy for blocks that reached 100% blob capacity", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 61, + "options": { + "legend": { + "calcs": [ + "last", + "mean" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "12.3.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${data_source}" + }, + "editorMode": "code", + "expr": "sum by(aztec_fisherman_strategy_id) (\n increase(aztec_fisherman_fee_analysis_would_be_included{k8s_namespace_name=\"$namespace\",aztec_ok=\"true\"}[$__rate_interval])\n * on() group_left()\n (aztec_fisherman_fee_analysis_block_blobs_full{k8s_namespace_name=\"$namespace\",aztec_ok=\"true\"} > 0)\n) / sum by(aztec_fisherman_strategy_id) (\n increase(aztec_fisherman_fee_analysis_would_be_included{k8s_namespace_name=\"$namespace\"}[$__rate_interval])\n * on() group_left()\n (aztec_fisherman_fee_analysis_block_blobs_full{k8s_namespace_name=\"$namespace\"} > 0)\n)", + "legendFormat": "{{aztec_fisherman_strategy_id}}", + "range": true, + "refId": "A" + } + ], + "title": "Inclusion Rate by Strategy (Full Blocks Only)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 55 }, "id": 50, "panels": [], @@ -1044,7 +1373,7 @@ "h": 10, "w": 12, "x": 0, - "y": 38 + "y": 56 }, "id": 51, "options": { @@ -1149,7 +1478,7 @@ "h": 10, "w": 12, "x": 12, - "y": 38 + "y": 56 }, "id": 52, "options": { @@ -1214,7 +1543,7 @@ "h": 1, "w": 24, "x": 0, - "y": 48 + "y": 66 }, "id": 40, "panels": [], @@ -1342,7 +1671,7 @@ "h": 9, "w": 24, "x": 0, - "y": 49 + "y": 67 }, "id": 41, "options": { diff --git a/yarn-project/sequencer-client/src/sequencer/metrics.ts b/yarn-project/sequencer-client/src/sequencer/metrics.ts index 9f8c11ae8742..2758074d3bdd 100644 --- a/yarn-project/sequencer-client/src/sequencer/metrics.ts +++ b/yarn-project/sequencer-client/src/sequencer/metrics.ts @@ -51,6 +51,9 @@ export class SequencerMetrics { private fishermanTimeBeforeBlock: Histogram; private fishermanPendingBlobTxCount: Histogram; private fishermanIncludedBlobTxCount: Histogram; + private fishermanPendingBlobCount: Histogram; + private fishermanIncludedBlobCount: Histogram; + private fishermanBlockBlobsFull: UpDownCounter; private fishermanCalculatedPriorityFee: Histogram; private fishermanPriorityFeeDelta: Histogram; private fishermanEstimatedCost: Histogram; @@ -161,6 +164,18 @@ export class SequencerMetrics { this.fishermanMinedBlobTxTotalCost = this.meter.createHistogram( Metrics.FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST, ); + + this.fishermanPendingBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_COUNT); + + this.fishermanIncludedBlobCount = this.meter.createHistogram(Metrics.FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_COUNT); + + this.fishermanBlockBlobsFull = createUpDownCounterWithDefault( + this.meter, + Metrics.FISHERMAN_FEE_ANALYSIS_BLOCK_BLOBS_FULL, + { + [Attributes.OK]: [true, false], + }, + ); } public recordRequiredAttestations(requiredAttestationsCount: number, allowanceMs: number) { @@ -281,10 +296,12 @@ export class SequencerMetrics { // Record pending block snapshot data (once per strategy for comparison) this.fishermanPendingBlobTxCount.record(analysis.pendingSnapshot.pendingBlobTxCount, strategyAttributes); + this.fishermanPendingBlobCount.record(analysis.pendingSnapshot.pendingBlobCount, strategyAttributes); // Record mined block data if available if (analysis.minedBlock) { this.fishermanIncludedBlobTxCount.record(analysis.minedBlock.includedBlobTxCount, strategyAttributes); + this.fishermanIncludedBlobCount.record(analysis.minedBlock.includedBlobCount, strategyAttributes); // Record actual fees from blob transactions in the mined block for (const blobTx of analysis.minedBlock.includedBlobTxs) { @@ -318,6 +335,13 @@ export class SequencerMetrics { if (analysis.analysis) { this.fishermanTimeBeforeBlock.record(Math.ceil(analysis.analysis.timeBeforeBlockMs), strategyAttributes); + // Record whether the block reached 100% blob capacity + if (analysis.analysis.blockBlobsFull) { + this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: true }); + } else { + this.fishermanBlockBlobsFull.add(1, { ...strategyAttributes, [Attributes.OK]: false }); + } + // Record strategy-specific inclusion result if (strategyResult.wouldBeIncluded !== undefined) { if (strategyResult.wouldBeIncluded) { diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts index eaaf772e54e5..fa482558bdb4 100644 --- a/yarn-project/telemetry-client/src/metrics.ts +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -474,6 +474,23 @@ export const FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST: MetricDefinition = unit: 'eth', valueType: ValueType.DOUBLE, }; +export const FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_COUNT: MetricDefinition = { + name: 'aztec.fisherman.fee_analysis.pending_blob_count', + description: 'Total number of blobs in pending blob transactions', + unit: 'blobs', + valueType: ValueType.INT, +}; +export const FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_COUNT: MetricDefinition = { + name: 'aztec.fisherman.fee_analysis.included_blob_count', + description: 'Total number of blobs included in the mined block', + unit: 'blobs', + valueType: ValueType.INT, +}; +export const FISHERMAN_FEE_ANALYSIS_BLOCK_BLOBS_FULL: MetricDefinition = { + name: 'aztec.fisherman.fee_analysis.block_blobs_full', + description: 'Whether the mined block reached 100% blob capacity', + valueType: ValueType.INT, +}; export const VALIDATOR_INVALID_ATTESTATION_RECEIVED_COUNT: MetricDefinition = { name: 'aztec.validator.invalid_attestation_received_count', From 647ce4807fb90e97d5fdf0c1572b743bc3087bee Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 12 Feb 2026 13:17:38 +0000 Subject: [PATCH 25/27] chore: set up tx file store in next-net (#20418) Modifies the filestore to upload tx objects to `txs/aztec--` (just like blobs and snapshots) and sets up next-net to upload/download from file store. Fix A-505 --- spartan/environments/next-net.env | 5 ++- spartan/scripts/deploy_network.sh | 13 +++++++ spartan/scripts/setup_gcp_secrets.sh | 8 ++++ spartan/terraform/deploy-aztec-infra/main.tf | 7 ++++ .../terraform/deploy-aztec-infra/variables.tf | 19 +++++++++ yarn-project/p2p/src/client/factory.ts | 12 +++++- .../tx_collection/file_store_tx_source.ts | 9 +++-- .../tx_file_store/tx_file_store.test.ts | 39 +++++++++++-------- .../services/tx_file_store/tx_file_store.ts | 8 ++-- 9 files changed, 95 insertions(+), 25 deletions(-) diff --git a/spartan/environments/next-net.env b/spartan/environments/next-net.env index 96e43e756467..33708eed55db 100644 --- a/spartan/environments/next-net.env +++ b/spartan/environments/next-net.env @@ -18,6 +18,9 @@ ETHERSCAN_API_KEY=REPLACE_WITH_GCP_SECRET DEPLOY_INTERNAL_BOOTNODE=true STORE_SNAPSHOT_URL= BLOB_BUCKET_DIRECTORY=${BLOB_BUCKET_DIRECTORY:-next-net/blobs} +TX_FILE_STORE_ENABLED=true +TX_FILE_STORE_BUCKET_DIRECTORY=${TX_FILE_STORE_BUCKET_DIRECTORY:-next-net/txs} +TX_COLLECTION_FILE_STORE_URLS="https://aztec-labs-snapshots.com/${TX_FILE_STORE_BUCKET_DIRECTORY}" R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET PROVER_FAILED_PROOF_STORE=gs://aztec-develop/next-net/failed-proofs @@ -56,4 +59,4 @@ RPC_INGRESS_STATIC_IP_NAME=nextnet-rpc-ip RPC_INGRESS_SSL_CERT_NAMES='["nextnet-rpc-cert"]' VALIDATOR_HA_REPLICAS=1 -VALIDATOR_RESOURCE_PROFILE="prod-spot" \ No newline at end of file +VALIDATOR_RESOURCE_PROFILE="prod-spot" diff --git a/spartan/scripts/deploy_network.sh b/spartan/scripts/deploy_network.sh index d9ef3fd2e13b..9f38cdd4dbc9 100755 --- a/spartan/scripts/deploy_network.sh +++ b/spartan/scripts/deploy_network.sh @@ -162,6 +162,16 @@ else BLOB_FILE_STORE_UPLOAD_URL_TF="null" fi +# TX filestore configuration +TX_FILE_STORE_ENABLED=${TX_FILE_STORE_ENABLED:-false} +TX_FILE_STORE_URL_TF="" +if [[ -n "${TX_FILE_STORE_URL:-}" ]]; then + TX_FILE_STORE_URL_TF="\"$TX_FILE_STORE_URL\"" +else + TX_FILE_STORE_URL_TF="null" +fi +TX_COLLECTION_FILE_STORE_URLS=${TX_COLLECTION_FILE_STORE_URLS:-} + P2P_GOSSIPSUB_D=${P2P_GOSSIPSUB_D:-6} P2P_GOSSIPSUB_DLO=${P2P_GOSSIPSUB_DLO:-4} P2P_GOSSIPSUB_DHI=${P2P_GOSSIPSUB_DHI:-12} @@ -546,6 +556,9 @@ PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAGE = ${PROVER_L1_PRIORITY_FEE_BUMP_PERCENTAG PROVER_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE = ${PROVER_L1_PRIORITY_FEE_RETRY_BUMP_PERCENTAGE:-null} BLOB_ALLOW_EMPTY_SOURCES = ${BLOB_ALLOW_EMPTY_SOURCES:-false} BLOB_FILE_STORE_UPLOAD_URL = ${BLOB_FILE_STORE_UPLOAD_URL_TF} +TX_FILE_STORE_ENABLED = ${TX_FILE_STORE_ENABLED} +TX_FILE_STORE_URL = ${TX_FILE_STORE_URL_TF} +TX_COLLECTION_FILE_STORE_URLS = "${TX_COLLECTION_FILE_STORE_URLS}" DEBUG_P2P_INSTRUMENT_MESSAGES = ${DEBUG_P2P_INSTRUMENT_MESSAGES:-false} PROVER_AGENT_INCLUDE_METRICS = "${PROVER_AGENT_INCLUDE_METRICS-null}" diff --git a/spartan/scripts/setup_gcp_secrets.sh b/spartan/scripts/setup_gcp_secrets.sh index fb4073388386..2bde3c4e4b15 100755 --- a/spartan/scripts/setup_gcp_secrets.sh +++ b/spartan/scripts/setup_gcp_secrets.sh @@ -157,4 +157,12 @@ if [[ -n "${BLOB_BUCKET_DIRECTORY:-}" ]]; then export BLOB_FILE_STORE_UPLOAD_URL="s3://testnet-bucket/${BLOB_BUCKET_DIRECTORY}/?endpoint=https://${r2_account_id}.r2.cloudflarestorage.com" fi +# Construct TX_FILE_STORE_URL from the r2-account-id secret and TX_FILE_STORE_BUCKET_DIRECTORY +if [[ -n "${TX_FILE_STORE_BUCKET_DIRECTORY:-}" ]]; then + secret_file=$(get_secret "r2-account-id") + mask_secret_value "TX_FILE_STORE_URL" "$secret_file" + r2_account_id=$(cat "$secret_file") + export TX_FILE_STORE_URL="s3://testnet-bucket/${TX_FILE_STORE_BUCKET_DIRECTORY}/?endpoint=https://${r2_account_id}.r2.cloudflarestorage.com" +fi + echo "Successfully set up GCP secrets for $NETWORK" diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index 8390addb91e4..2e36227b6d2f 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -219,6 +219,7 @@ locals { "validator.node.env.P2P_DROP_TX" = var.P2P_DROP_TX "validator.node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE "validator.node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "validator.node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS } # Note: nonsensitive() is required here because helm_releases is used in for_each, @@ -357,6 +358,7 @@ locals { "node.node.env.P2P_DROP_TX" = var.P2P_DROP_TX "node.node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE "node.node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "node.node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS "node.service.p2p.nodePortEnabled" = var.P2P_NODEPORT_ENABLED "node.service.p2p.announcePort" = local.p2p_port_prover "node.service.p2p.port" = local.p2p_port_prover @@ -436,6 +438,9 @@ locals { "node.env.P2P_DROP_TX" = var.P2P_DROP_TX "node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE "node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "node.env.TX_FILE_STORE_ENABLED" = var.TX_FILE_STORE_ENABLED + "node.env.TX_FILE_STORE_URL" = var.TX_FILE_STORE_URL + "node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS }, # Only set RPC mnemonic config in fisherman mode) var.FISHERMAN_MODE ? { @@ -490,6 +495,7 @@ locals { "node.env.P2P_DROP_TX" = var.P2P_DROP_TX "node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE "node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS } boot_node_host_path = "node.env.BOOT_NODE_HOST" bootstrap_nodes_path = "node.env.BOOTSTRAP_NODES" @@ -529,6 +535,7 @@ locals { "node.env.P2P_DROP_TX" = var.P2P_DROP_TX "node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE "node.env.WS_NUM_HISTORIC_BLOCKS" = var.WS_NUM_HISTORIC_BLOCKS + "node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS } boot_node_host_path = "node.env.BOOT_NODE_HOST" bootstrap_nodes_path = "node.env.BOOTSTRAP_NODES" diff --git a/spartan/terraform/deploy-aztec-infra/variables.tf b/spartan/terraform/deploy-aztec-infra/variables.tf index b8821306bde2..1f38933f523b 100644 --- a/spartan/terraform/deploy-aztec-infra/variables.tf +++ b/spartan/terraform/deploy-aztec-infra/variables.tf @@ -656,6 +656,25 @@ variable "BLOB_FILE_STORE_UPLOAD_URL" { default = null } +variable "TX_FILE_STORE_ENABLED" { + description = "Whether to enable uploading transactions to file storage" + type = bool + default = false +} + +variable "TX_FILE_STORE_URL" { + description = "URL for uploading transactions (e.g., s3://bucket/path/, gs://bucket/path/)" + type = string + nullable = true + default = null +} + +variable "TX_COLLECTION_FILE_STORE_URLS" { + description = "Comma-separated URLs for reading transactions from file storage" + type = string + default = "" +} + variable "PROVER_AGENT_POLL_INTERVAL_MS" { description = "Interval in milliseconds between prover agent polls" type = number diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index 30299c55aada..1a4a813c5fa5 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -76,6 +76,9 @@ export async function createP2PClient( const attestationStore = await createStore(P2P_ATTESTATION_STORE_NAME, 1, config, bindings); const l1Constants = await archiver.getL1Constants(); + const rollupAddress = inputConfig.l1Contracts.rollupAddress.toString().toLowerCase().replace(/^0x/, ''); + const txFileStoreBasePath = `aztec-${inputConfig.l1ChainId}-${inputConfig.rollupVersion}-0x${rollupAddress}`; + /** Validator factory for pool re-validation (double-spend + block header only). */ const createPoolTxValidator = async () => { await worldStateSynchronizer.syncImmediate(); @@ -154,6 +157,7 @@ export async function createP2PClient( const fileStoreSources = await createFileStoreTxSources( config.txCollectionFileStoreUrls, + txFileStoreBasePath, logger.createChild('file-store-tx-source'), ); if (fileStoreSources.length > 0) { @@ -174,7 +178,13 @@ export async function createP2PClient( logger.createChild('tx-collection'), ); - const txFileStore = await TxFileStore.create(mempools.txPool, config, logger.createChild('tx-file-store'), telemetry); + const txFileStore = await TxFileStore.create( + mempools.txPool, + config, + txFileStoreBasePath, + logger.createChild('tx-file-store'), + telemetry, + ); return new P2PClient( clientType, diff --git a/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts b/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts index b88f6b028ede..ec8381d2d6cf 100644 --- a/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts +++ b/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts @@ -9,6 +9,7 @@ export class FileStoreTxSource implements TxSource { private constructor( private readonly fileStore: ReadOnlyFileStore, private readonly baseUrl: string, + private readonly basePath: string, private readonly log: Logger, ) {} @@ -20,6 +21,7 @@ export class FileStoreTxSource implements TxSource { */ public static async create( url: string, + basePath: string, log: Logger = createLogger('p2p:file_store_tx_source'), ): Promise { try { @@ -28,7 +30,7 @@ export class FileStoreTxSource implements TxSource { log.warn(`Failed to create file store for URL: ${url}`); return undefined; } - return new FileStoreTxSource(fileStore, url, log); + return new FileStoreTxSource(fileStore, url, basePath, log); } catch (err) { log.warn(`Error creating file store for URL: ${url}`, { error: err }); return undefined; @@ -42,7 +44,7 @@ export class FileStoreTxSource implements TxSource { public getTxsByHash(txHashes: TxHash[]): Promise<(Tx | undefined)[]> { return Promise.all( txHashes.map(async txHash => { - const path = `txs/${txHash.toString()}.bin`; + const path = `${this.basePath}/txs/${txHash.toString()}.bin`; try { const buffer = await this.fileStore.read(path); return Tx.fromBuffer(buffer); @@ -63,8 +65,9 @@ export class FileStoreTxSource implements TxSource { */ export async function createFileStoreTxSources( urls: string[], + basePath: string, log: Logger = createLogger('p2p:file_store_tx_source'), ): Promise { - const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, log))); + const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, basePath, log))); return sources.filter((s): s is FileStoreTxSource => s !== undefined); } diff --git a/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts b/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts index 3a35574afc7f..e14f47a2b6dd 100644 --- a/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts +++ b/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts @@ -19,6 +19,7 @@ describe('TxFileStore', () => { let config: TxFileStoreConfig; let txFileStore: TxFileStore | undefined; const log = createLogger('test:tx_file_store'); + const basePath = 'aztec-1-1-0x1234'; const makeTx = async () => { const tx = Tx.random(); @@ -29,7 +30,7 @@ describe('TxFileStore', () => { /** Counts files in the txs subdirectory of the temp directory. */ async function countUploadedFiles(): Promise { try { - const files = await readdir(join(tmpDir, 'txs')); + const files = await readdir(join(tmpDir, basePath, 'txs')); return files.length; } catch { return 0; @@ -43,7 +44,7 @@ describe('TxFileStore', () => { beforeEach(async () => { // Clean up any files from previous test try { - await rm(join(tmpDir, 'txs'), { recursive: true, force: true }); + await rm(join(tmpDir, basePath), { recursive: true, force: true }); } catch { // Directory might not exist } @@ -73,25 +74,25 @@ describe('TxFileStore', () => { describe('create', () => { it('returns undefined when disabled', async () => { config.txFileStoreEnabled = false; - const result = await TxFileStore.create(txPool, config, log, undefined, fileStore); + const result = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); expect(result).toBeUndefined(); }); it('returns undefined when upload URL is not configured', async () => { config.txFileStoreUrl = undefined; - const result = await TxFileStore.create(txPool, config, log, undefined, fileStore); + const result = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); expect(result).toBeUndefined(); }); it('creates file store when enabled and configured', async () => { - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); expect(txFileStore).toBeDefined(); }); }); describe('start/stop', () => { it('subscribes to txs-added event on start', async () => { - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); txFileStore!.start(); const spy = jest.spyOn(fileStore, 'save'); @@ -101,13 +102,15 @@ describe('TxFileStore', () => { await txFileStore!.flush(); - expect(spy).toHaveBeenCalledWith(`txs/${tx.getTxHash().toString()}.bin`, tx.toBuffer(), { compress: false }); + expect(spy).toHaveBeenCalledWith(`${basePath}/txs/${tx.getTxHash().toString()}.bin`, tx.toBuffer(), { + compress: false, + }); spy.mockRestore(); }); it('unsubscribes from txs-added event on stop', async () => { - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); txFileStore!.start(); const spy = jest.spyOn(fileStore, 'save'); @@ -134,7 +137,7 @@ describe('TxFileStore', () => { describe('tx upload', () => { it('uploads tx when txs-added event fires', async () => { - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); txFileStore!.start(); const spy = jest.spyOn(fileStore, 'save'); @@ -144,13 +147,15 @@ describe('TxFileStore', () => { await txFileStore!.flush(); - expect(spy).toHaveBeenCalledWith(`txs/${tx.getTxHash().toString()}.bin`, tx.toBuffer(), { compress: false }); + expect(spy).toHaveBeenCalledWith(`${basePath}/txs/${tx.getTxHash().toString()}.bin`, tx.toBuffer(), { + compress: false, + }); spy.mockRestore(); }); it('uploads multiple txs', async () => { - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); txFileStore!.start(); const spy = jest.spyOn(fileStore, 'save'); @@ -169,7 +174,7 @@ describe('TxFileStore', () => { it('respects concurrency limit', async () => { config.txFileStoreUploadConcurrency = 10; config.txFileStoreMaxQueueSize = 100; // Increase to accommodate 20 txs - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); txFileStore!.start(); let activeCalls = 0; @@ -204,7 +209,7 @@ describe('TxFileStore', () => { }); it('skips duplicate tx uploads', async () => { - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); txFileStore!.start(); const spy = jest.spyOn(fileStore, 'save'); @@ -227,7 +232,7 @@ describe('TxFileStore', () => { it('drops oldest txs when queue exceeds max size', async () => { config.txFileStoreUploadConcurrency = 1; config.txFileStoreMaxQueueSize = 2; - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); txFileStore!.start(); const spy = jest.spyOn(fileStore, 'save'); @@ -251,7 +256,7 @@ describe('TxFileStore', () => { describe('error handling', () => { it('retries on transient failures', async () => { - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); txFileStore!.start(); const originalSave = fileStore.save.bind(fileStore); @@ -276,7 +281,7 @@ describe('TxFileStore', () => { it('continues processing after exhausting retries', async () => { // Use concurrency=1 to ensure sequential processing for predictable retry behavior config.txFileStoreUploadConcurrency = 1; - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); txFileStore!.start(); const originalSave = fileStore.save.bind(fileStore); @@ -305,7 +310,7 @@ describe('TxFileStore', () => { describe('getPendingUploadCount', () => { it('returns correct count of pending uploads', async () => { - txFileStore = await TxFileStore.create(txPool, config, log, undefined, fileStore); + txFileStore = await TxFileStore.create(txPool, config, basePath, log, undefined, fileStore); txFileStore!.start(); expect(txFileStore!.getPendingUploadCount()).toBe(0); diff --git a/yarn-project/p2p/src/services/tx_file_store/tx_file_store.ts b/yarn-project/p2p/src/services/tx_file_store/tx_file_store.ts index 13ea96d8621f..672bdd6d40cc 100644 --- a/yarn-project/p2p/src/services/tx_file_store/tx_file_store.ts +++ b/yarn-project/p2p/src/services/tx_file_store/tx_file_store.ts @@ -31,6 +31,7 @@ export class TxFileStore { private readonly config: TxFileStoreConfig, private readonly instrumentation: TxFileStoreInstrumentation, private readonly log: Logger, + private readonly basePath: string, ) { this.handleTxsAdded = (args: { txs: Tx[]; source?: string }) => { this.enqueueTxs(args.txs); @@ -50,6 +51,7 @@ export class TxFileStore { static async create( txPool: TxPoolV2, config: TxFileStoreConfig, + basePath: string, log: Logger = createLogger('p2p:tx_file_store'), telemetry: TelemetryClient = getTelemetryClient(), fileStoreOverride?: FileStore, @@ -71,8 +73,8 @@ export class TxFileStore { } const instrumentation = new TxFileStoreInstrumentation(telemetry, 'TxFileStore'); - log.info('Created tx file store', { url: config.txFileStoreUrl }); - return new TxFileStore(fileStore, txPool, config, instrumentation, log); + log.info('Created tx file store', { url: config.txFileStoreUrl, basePath }); + return new TxFileStore(fileStore, txPool, config, instrumentation, log, basePath); } /** Starts listening to TxPool events and uploading txs. */ @@ -122,7 +124,7 @@ export class TxFileStore { private async uploadTx(tx: Tx): Promise { const txHash = tx.getTxHash().toString(); - const path = `txs/${txHash}.bin`; + const path = `${this.basePath}/txs/${txHash}.bin`; const timer = new Timer(); if (this.recentUploads.has(txHash)) { From 5102ec2d28a4a9306a69fa9a4c0bdfea38e09df2 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 12 Feb 2026 14:02:06 +0000 Subject: [PATCH 26/27] chore(ci): gate draft PRs from CI, allow override with ci-draft label (#20426) Co-authored-by: Claude Opus 4.6 --- .github/workflows/ci3.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci3.yml b/.github/workflows/ci3.yml index 526ba37fc85f..a706e33fd021 100644 --- a/.github/workflows/ci3.yml +++ b/.github/workflows/ci3.yml @@ -41,7 +41,7 @@ jobs: runs-on: ubuntu-latest # exclusive with ci3-external.yml: never run on forks # (github.event.pull_request.head.repo.fork resolves to nil if not a pull request) - if: github.event.pull_request.head.repo.fork != true + if: github.event.pull_request.head.repo.fork != true && (github.event.pull_request.draft == false || contains(github.event.pull_request.labels.*.name, 'ci-draft')) environment: ${{ startsWith(github.ref, 'refs/tags/v') && 'master' || '' }} steps: - name: Checkout From a59ceac9cd6e23f1abb9995e8e6c0648608df1a4 Mon Sep 17 00:00:00 2001 From: ludamad Date: Thu, 12 Feb 2026 14:13:06 +0000 Subject: [PATCH 27/27] fix: stabilize writing_an_account_contract.test.ts (#20420) --- yarn-project/end-to-end/src/fixtures/setup.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/yarn-project/end-to-end/src/fixtures/setup.ts b/yarn-project/end-to-end/src/fixtures/setup.ts index 5d496a313e47..3413bb82fc64 100644 --- a/yarn-project/end-to-end/src/fixtures/setup.ts +++ b/yarn-project/end-to-end/src/fixtures/setup.ts @@ -415,9 +415,14 @@ export async function setup( if (enableAutomine) { await ethCheatCodes.setAutomine(false); await ethCheatCodes.setIntervalMining(config.ethereumSlotDuration); - dateProvider.setTime((await ethCheatCodes.timestamp()) * 1000); } + // Always sync dateProvider to L1 time after deploying L1 contracts, regardless of mining mode. + // In compose mode, L1 time may have drifted ahead of system time due to the local-network watcher + // warping time forward on each filled slot. Without this sync, the sequencer computes the wrong + // slot from its dateProvider and cannot propose blocks. + dateProvider.setTime((await ethCheatCodes.timestamp()) * 1000); + if (opts.l2StartTime) { await ethCheatCodes.warp(opts.l2StartTime, { resetBlockInterval: true }); }