AztecProtocol · PhilWindle · Jun 9, 2026 · Jun 9, 2026
diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_optimistic_proving.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_optimistic_proving.parallel.test.ts
@@ -55,6 +55,23 @@ describe('e2e_epochs/epochs_optimistic_proving', () => {
     return BlockNumber(cp.startBlock + cp.blockCount - 1);
   };
 
+  /**
+   * Returns the canonical checkpoint numbers that fall within `epoch`, considering checkpoints
+   * `1..upTo`. Retries until the archiver has indexed the whole range so the count is stable.
+   */
+  const checkpointsInEpoch = async (epoch: EpochNumber, upTo: CheckpointNumber): Promise<CheckpointNumber[]> => {
+    const cps = await retryUntil(
+      async () => {
+        const all = await node.getCheckpoints(CheckpointNumber(1), Number(upTo));
+        return all.length >= Number(upTo) ? all : undefined;
+      },
+      `archiver indexes checkpoints up to ${upTo}`,
+      30,
+      0.2,
+    );
+    return cps.filter(cp => getEpochAtSlot(cp.header.slotNumber, test.constants) === epoch).map(cp => cp.number);
+  };
+
   /**
    * Background sampler proving the prover-node works an epoch *optimistically* — i.e. it
    * spawns a checkpoint's sub-tree before the epoch is over on L1, not just after the
@@ -463,13 +480,28 @@ describe('e2e_epochs/epochs_optimistic_proving', () => {
     });
 
     it('removes a checkpoint mid-epoch via reorg and proves with survivors', async () => {
+      // Anchor on a freshly-started epoch so the checkpoints we reorg over (and the survivor)
+      // are guaranteed to live in the same epoch. Without this, setup landing near an epoch
+      // boundary could leave the survivor in the previous epoch, passing the test without
+      // actually exercising in-epoch checkpoint removal (see #22990).
+      await test.waitUntilNextEpochStarts();
+
       // Wait for 2 checkpoints mid-epoch.
       const initialCheckpoint = (await test.monitor.run(true)).checkpointNumber;
       const midCheckpoint = CheckpointNumber(initialCheckpoint + 2);
       await test.waitUntilCheckpointNumber(midCheckpoint, L2_SLOT_DURATION_IN_S * 6);
       const checkpointBeforeReorg = test.monitor.checkpointNumber;
       logger.info(`Reached checkpoint ${checkpointBeforeReorg}`);
 
+      // Capture the epoch we're reorging within so we can assert the survivor stays in it.
+      const epochBeforeReorg = await epochOfCheckpoint(checkpointBeforeReorg);
+
+      // (1) The epoch must hold multiple checkpoints, with checkpointBeforeReorg as its latest —
+      // otherwise removing the last one wouldn't leave any in-epoch survivors to prove with.
+      const epochCheckpointsBeforeReorg = await checkpointsInEpoch(epochBeforeReorg, checkpointBeforeReorg);
+      expect(epochCheckpointsBeforeReorg.length).toBeGreaterThanOrEqual(2);
+      expect(epochCheckpointsBeforeReorg.at(-1)).toEqual(checkpointBeforeReorg);
+
       // Stop block production so no replacement is proposed.
       await context.aztecNodeAdmin!.setConfig({ skipPublishingCheckpointsPercent: 100 });
 
@@ -478,7 +510,8 @@ describe('e2e_epochs/epochs_optimistic_proving', () => {
       await context.cheatCodes.eth.reorgWithReplacement(1);
 
       const afterReorgCheckpoint = (await test.monitor.run(true)).checkpointNumber;
-      expect(afterReorgCheckpoint).toBeLessThan(checkpointBeforeReorg);
+      // (2) The reorg removed exactly the last checkpoint, leaving N-1.
+      expect(afterReorgCheckpoint).toEqual(CheckpointNumber(checkpointBeforeReorg - 1));
       logger.info(`After reorg: checkpoint ${afterReorgCheckpoint} (was ${checkpointBeforeReorg})`);
 
       // Verify node detects the reorg.
@@ -489,12 +522,21 @@ describe('e2e_epochs/epochs_optimistic_proving', () => {
         0.5,
       );
 
-      // Wait for the epoch to end and proof to land with the surviving checkpoints.
-      // Use the surviving checkpoint to look up which epoch we're in.
+      // The survivor must still be in the epoch we reorged within — otherwise the reorg removed
+      // the only in-epoch checkpoint and the test isn't exercising mid-epoch removal.
       const currentEpoch = await epochOfCheckpoint(afterReorgCheckpoint);
+      expect(currentEpoch).toEqual(epochBeforeReorg);
+
+      // The epoch now holds exactly N-1 checkpoints — the survivors of the removal.
+      const survivingCheckpoints = await checkpointsInEpoch(epochBeforeReorg, afterReorgCheckpoint);
+      expect(survivingCheckpoints.length).toEqual(epochCheckpointsBeforeReorg.length - 1);
+      expect(survivingCheckpoints.at(-1)).toEqual(afterReorgCheckpoint);
+
+      // Wait for the epoch to end and proof to land with the surviving checkpoints.
       await test.waitUntilEpochStarts(currentEpoch + 1);
       const epochEndCheckpoint = (await test.monitor.run(true)).checkpointNumber;
 
+      // (3) The epoch proved up to and including the last surviving checkpoint (the (N-1)th).
       expect(epochEndCheckpoint).toEqual(afterReorgCheckpoint);
 
       await test.waitUntilProvenCheckpointNumber(epochEndCheckpoint, 240);

diff --git a/yarn-project/prover-client/src/orchestrator/checkpoint-sub-tree-orchestrator.ts b/yarn-project/prover-client/src/orchestrator/checkpoint-sub-tree-orchestrator.ts
@@ -8,6 +8,7 @@ import {
 } from '@aztec/constants';
 import { BlockNumber, type EpochNumber } from '@aztec/foundation/branded-types';
 import { Fr } from '@aztec/foundation/curves/bn254';
+import { AbortError } from '@aztec/foundation/error';
 import type { LoggerBindings } from '@aztec/foundation/log';
 import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise';
 import type { SerialQueue } from '@aztec/foundation/queue';
@@ -83,6 +84,17 @@ export type SubTreeResult = {
 
 type TreeSnapshots = Map<MerkleTreeId, AppendOnlyTreeSnapshot>;
 
+/**
+ * Base rollup hints as produced before proving: `PrivateBaseRollupHints` / `PublicBaseRollupHints`
+ * deliberately carry no recursive proof or verification key. The proof + VK are supplied later, when
+ * `TxProvingState.getBaseRollupTypeAndInputs` wraps these hints into the "with proof + VK" types —
+ * `PrivateTxBaseRollupPrivateInputs` (a `ChonkProofData`) or `PublicTxBaseRollupPrivateInputs` (a
+ * chonk-verifier proof + AVM proof). Those proofs are *required constructor arguments* of the wrapper
+ * types, so the only way to obtain a provable input is to populate them — they cannot be silently
+ * omitted. Naming the proof-less hints type here makes that boundary explicit at `prepareBaseRollupInputs`.
+ */
+type BaseRollupHintsWithoutProofAndVK = BaseRollupHints;
+
 /**
  * Orchestrates block-level proving for a single checkpoint, stopping at the boundary
  * where checkpoint root rollup would otherwise begin. Used by the per-checkpoint
@@ -457,6 +469,9 @@ export class CheckpointSubTreeOrchestrator extends ProvingScheduler {
   public cancel() {
     this.cancelled = true;
     this.resetSchedulerState(this.cancelJobsOnStop);
+    // Reject the proving state (and hence subTreeResult) so anyone awaiting the sub-tree result
+    // is released rather than hanging — matching TopTreeOrchestrator.cancel().
+    this.provingState?.cancel();
 
     for (const [blockNumber, db] of this.dbs.entries()) {
       void db.close().catch(err => this.logger.error(`Error closing db for block ${blockNumber}`, err));
@@ -556,9 +571,11 @@ export class CheckpointSubTreeOrchestrator extends ProvingScheduler {
     newL1ToL2MessageTreeSnapshot: AppendOnlyTreeSnapshot,
     startSpongeBlob: SpongeBlob,
     db: MerkleTreeWriteOperations,
-  ): Promise<[BaseRollupHints, TreeSnapshots]> {
-    // We build the base rollup inputs using a mock proof and verification key.
-    // These will be overwritten later once we have proven the chonk verifier circuit and any public kernels.
+  ): Promise<[BaseRollupHintsWithoutProofAndVK, TreeSnapshots]> {
+    // These hints deliberately carry no recursive proof or verification key — see
+    // BaseRollupHintsWithoutProofAndVK. The tx's proof + VK are attached later in
+    // TxProvingState.getBaseRollupTypeAndInputs from the proven chonk-verifier / kernel / AVM
+    // proofs, which are required there and so cannot be silently omitted.
     const start = performance.now();
     const hints = await insertSideEffectsAndBuildBaseRollupHints(
       tx,
@@ -660,9 +677,16 @@ export class CheckpointSubTreeOrchestrator extends ProvingScheduler {
         this.epochNumber,
       ),
     );
-    void promise.then(handleResult).catch(() => {
-      // The cache self-cleans on rejection; a future call (replacement sub-tree
-      // for this tx) will see the miss and re-enqueue. No action needed here.
+    void promise.then(handleResult).catch(err => {
+      // The cache self-cleans on rejection, so a replacement sub-tree for this tx will see the
+      // miss and re-enqueue. But if this proving state is still active, the failure must abort
+      // it: otherwise the base rollup for this tx is never enqueued and the checkpoint (and
+      // epoch) orchestrators hang forever waiting for a proof that will never arrive.
+      if (err instanceof AbortError || !provingState.verifyState()) {
+        return;
+      }
+      this.logger.error(`Chonk verifier proof failed for tx ${txHash}`, err);
+      provingState.reject(`Chonk verifier proof failed for tx ${txHash}: ${err}`);
     });
   }
 

diff --git a/yarn-project/prover-client/src/orchestrator/proving-scheduler.ts b/yarn-project/prover-client/src/orchestrator/proving-scheduler.ts
@@ -19,10 +19,16 @@ export interface ProvingStateLike {
  * Common scheduling infrastructure shared by every orchestrator that drives broker
  * proving jobs:
  *
- *   - A shared `SerialQueue` (`deferredJobQueue`) acting as the enqueue-throttle. The
- *     queue is owned by the `ProverClient` and shared across every orchestrator (every
- *     sub-tree and top-tree across every concurrent epoch session), so the total rate
- *     of job submission to the broker is bounded once, not once-per-orchestrator.
+ *   - A shared `SerialQueue` (`deferredJobQueue`) that serialises the *act of handing a
+ *     job to the broker*, one initiation per event-loop tick. Each queue task kicks off
+ *     `safeJob` (which submits to the broker) without awaiting it, then yields with
+ *     `sleep(0)`; the next task therefore runs on the following macrotask. This does NOT
+ *     cap how many broker jobs are concurrently in flight — the broker's own queue absorbs
+ *     that. What it bounds is the burst rate: a sub-tree that synchronously discovers
+ *     thousands of ready jobs can't flood the broker (and monopolise the event loop) in a
+ *     single tick. The queue is owned by the `ProverClient` and shared across every
+ *     orchestrator (every sub-tree and top-tree across every concurrent epoch session), so
+ *     this pacing is applied once globally rather than once-per-orchestrator.
  *   - A list of `AbortController`s (`pendingProvingJobs`) so a `cancel()` can abort
  *     in-flight broker jobs when needed.
  *   - A `deferredProving<T>(state, request, callback, isCancelled?)` method that wraps
@@ -143,9 +149,11 @@ export abstract class ProvingScheduler {
     };
 
     void this.deferredJobQueue.put(async () => {
+      // Kick off the broker submission without awaiting it — awaiting here would serialise all
+      // proving (one job at a time) and kill parallelism. The `sleep(0)` yields the event loop
+      // so the next queued job initiates on the following macrotask, pacing bursts rather than
+      // bounding in-flight broker concurrency (the broker's own queue handles that).
       void safeJob();
-      // Yield to the macrotask queue so Node has a chance to interleave other work
-      // between enqueues.
       await sleep(0);
     });
   }

diff --git a/yarn-project/prover-node/src/config.ts b/yarn-project/prover-node/src/config.ts
@@ -68,7 +68,8 @@ export const specificProverNodeConfigMappings: ConfigMappingsType<SpecificProver
     defaultValue: undefined,
   },
   proverNodeEpochProvingDelayMs: {
-    description: 'Optional delay in milliseconds to wait before proving a new epoch',
+    description:
+      'Optional delay in milliseconds to wait for late-arriving events (e.g. reorgs) to settle before starting top-tree proving for an epoch',
     defaultValue: undefined,
   },
   txGatheringIntervalMs: {

diff --git a/yarn-project/prover-node/src/proof-publishing-service.ts b/yarn-project/prover-node/src/proof-publishing-service.ts
@@ -333,6 +333,8 @@ export class ProofPublishingService {
       proof: candidate.proof,
       batchedBlobInputs: candidate.batchedBlobInputs,
       attestations: candidate.attestations,
+      // Stop the L1 tx retrying past the candidate's submission-window deadline.
+      deadline: candidate.deadline,
     };
 
     if (this.deps.config.skipSubmitProof) {

diff --git a/yarn-project/prover-node/src/prover-node-publisher.test.ts b/yarn-project/prover-node/src/prover-node-publisher.test.ts
@@ -188,82 +188,6 @@ describe('prover-node-publisher', () => {
     },
   );
 
-  it('waits until the proven checkpoint reaches the checkpoint before the proof start', async () => {
-    const checkpoints = Array.from({ length: 100 }, () => RootRollupPublicInputs.random());
-    const fromCheckpoint = CheckpointNumber(33);
-    const toCheckpoint = CheckpointNumber(64);
-
-    rollup.getTips
-      .mockResolvedValueOnce({
-        pending: CheckpointNumber(65),
-        proven: CheckpointNumber(31),
-      })
-      .mockResolvedValueOnce({
-        pending: CheckpointNumber(65),
-        proven: CheckpointNumber(32),
-      })
-      .mockResolvedValue({
-        pending: CheckpointNumber(65),
-        proven: CheckpointNumber(32),
-      });
-    rollup.getRollupConstants.mockResolvedValue({
-      l1StartBlock: 0n,
-      l1GenesisTime: BigInt(Math.floor(Date.now() / 1000)),
-      slotDuration: 1,
-      epochDuration: 1,
-      proofSubmissionEpochs: 100,
-      targetCommitteeSize: 48,
-      rollupManaLimit: Number.MAX_SAFE_INTEGER,
-    });
-
-    rollup.getCheckpoint.mockImplementation((checkpointNumber: CheckpointNumber) =>
-      Promise.resolve({
-        archive: checkpoints[checkpointNumber - 1].endArchiveRoot,
-        attestationsHash: Buffer32.ZERO,
-        payloadDigest: Buffer32.ZERO,
-        headerHash: Buffer32.ZERO,
-        blobCommitmentsHash: Buffer32.ZERO,
-        outHash: '0x',
-        slotNumber: SlotNumber(0),
-        feeHeader: {
-          excessMana: 0n,
-          manaUsed: 0n,
-          ethPerFeeAsset: 0n,
-          congestionCost: 0n,
-          proverCost: 0n,
-        },
-      }),
-    );
-
-    const ourPublicInputs = RootRollupPublicInputs.random();
-    ourPublicInputs.previousArchiveRoot = checkpoints[fromCheckpoint - 2].endArchiveRoot;
-    ourPublicInputs.endArchiveRoot = checkpoints[toCheckpoint - 1].endArchiveRoot;
-
-    const ourBatchedBlob = new BatchedBlob(
-      ourPublicInputs.blobPublicInputs.blobCommitmentsHash,
-      ourPublicInputs.blobPublicInputs.z,
-      ourPublicInputs.blobPublicInputs.y,
-      ourPublicInputs.blobPublicInputs.c,
-      ourPublicInputs.blobPublicInputs.c.negate(),
-    );
-
-    rollup.getEpochProofPublicInputs.mockResolvedValue(ourPublicInputs.toFields());
-
-    await publisher.submitEpochProof({
-      epochNumber: EpochNumber(2),
-      fromCheckpoint,
-      toCheckpoint,
-      publicInputs: ourPublicInputs,
-      proof: Proof.empty(),
-      batchedBlobInputs: ourBatchedBlob,
-      attestations: [],
-    });
-
-    expect(rollup.getRollupConstants).toHaveBeenCalled();
-    expect(rollup.getTips).toHaveBeenCalledTimes(3);
-    expect(l1Utils.sendAndMonitorTransaction).toHaveBeenCalled();
-  });
-
   it('analyzeEpochProofSubmission validates, estimates, and does not send tx', async () => {
     const fromCheckpoint = 33;
     const toCheckpoint = 64;

diff --git a/yarn-project/prover-node/src/prover-node-publisher.ts b/yarn-project/prover-node/src/prover-node-publisher.ts
@@ -8,13 +8,11 @@ import { areArraysEqual } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
-import { retryUntil } from '@aztec/foundation/retry';
 import type { Tuple } from '@aztec/foundation/serialize';
 import { Timer } from '@aztec/foundation/timer';
 import { RollupAbi } from '@aztec/l1-artifacts';
 import type { PublisherConfig, TxSenderConfig } from '@aztec/sequencer-client';
 import { CommitteeAttestation, CommitteeAttestationsAndSigners } from '@aztec/stdlib/block';
-import { getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers';
 import type { Proof } from '@aztec/stdlib/proofs';
 import type { FeeRecipient, RootRollupPublicInputs } from '@aztec/stdlib/rollup';
 import type { L1PublishProofStats } from '@aztec/stdlib/stats';
@@ -80,15 +78,12 @@ export class ProverNodePublisher {
     proof: Proof;
     batchedBlobInputs: BatchedBlob;
     attestations: ViemCommitteeAttestation[];
+    /** Wall-clock deadline (proof-submission window end) past which the L1 tx should stop retrying. */
+    deadline?: Date;
   }): Promise<boolean> {
     const { epochNumber, fromCheckpoint, toCheckpoint } = args;
     const ctx = { epochNumber, fromCheckpoint, toCheckpoint };
 
-    if (!(await this.waitUntilStartBuildsOnProven(args))) {
-      this.log.verbose('Timed out waiting for proven checkpoint to reach proof start', ctx);
-      return false;
-    }
-
     const timer = new Timer();
     // Validate epoch proof range and hashes are correct before submitting
     await this.validateEpochProofSubmission(args);
@@ -132,49 +127,6 @@ export class ProverNodePublisher {
     return false;
   }
 
-  private async waitUntilStartBuildsOnProven(args: { epochNumber: EpochNumber; fromCheckpoint: CheckpointNumber }) {
-    const { epochNumber, fromCheckpoint } = args;
-    const provenCheckpoint = await this.getProvenCheckpoint();
-    if (this.isStartBuildingOnProven(fromCheckpoint, provenCheckpoint)) {
-      return true;
-    }
-
-    const timeout = await this.getSecondsUntilProofSubmissionWindowEnd(epochNumber);
-    this.log.info(`Waiting for proven checkpoint to reach proof start`, {
-      epochNumber,
-      fromCheckpoint,
-      provenCheckpoint,
-      timeout,
-    });
-
-    await retryUntil(
-      async () => {
-        const proven = await this.getProvenCheckpoint();
-        this.log.verbose(`Proven checkpoint is at ${proven} (waiting for ${fromCheckpoint - 1})`, { epochNumber });
-        return this.isStartBuildingOnProven(fromCheckpoint, proven) ? true : undefined;
-      },
-      `proven checkpoint to reach ${fromCheckpoint - 1}`,
-      timeout,
-      4,
-    );
-
-    return true;
-  }
-
-  private async getProvenCheckpoint() {
-    return (await this.rollupContract.getTips()).proven;
-  }
-
-  private isStartBuildingOnProven(fromCheckpoint: CheckpointNumber, provenCheckpoint: CheckpointNumber) {
-    return fromCheckpoint - 1 <= provenCheckpoint;
-  }
-
-  private async getSecondsUntilProofSubmissionWindowEnd(epochNumber: EpochNumber) {
-    const deadline = getProofSubmissionDeadlineTimestamp(epochNumber, await this.rollupContract.getRollupConstants());
-    const now = BigInt(Math.floor(Date.now() / 1000));
-    return Math.max(Number(deadline - now), 0.001);
-  }
-
   private async validateEpochProofSubmission(args: {
     fromCheckpoint: CheckpointNumber;
     toCheckpoint: CheckpointNumber;
@@ -313,6 +265,7 @@ export class ProverNodePublisher {
   private async sendSubmitEpochProofTx(args: {
     fromCheckpoint: CheckpointNumber;
     toCheckpoint: CheckpointNumber;
+    deadline?: Date;
     publicInputs: RootRollupPublicInputs;
     proof: Proof;
     batchedBlobInputs: BatchedBlob;
@@ -331,7 +284,10 @@ export class ProverNodePublisher {
       args: txArgs,
     });
     try {
-      const { receipt } = await this.l1TxUtils.sendAndMonitorTransaction({ to: this.rollupContract.address, data });
+      const { receipt } = await this.l1TxUtils.sendAndMonitorTransaction(
+        { to: this.rollupContract.address, data },
+        { txTimeoutAt: args.deadline },
+      );
       if (receipt.status !== 'success') {
         const errorMsg = await this.l1TxUtils.tryGetErrorFromRevertedTx(
           data,