feat: merge-train/spartan-v5

ci3/run_compose_test

@@ -45,4 +45,8 @@ docker logs -f "$cid" &
 # Block until the target container exits; exit status is the container's exit code.
 rc=$(docker wait "$cid")
 
+# Give the backgrounded `docker logs -f` a moment to flush the container's final output
+# (e.g. the trailing local-network logs) before the EXIT trap tears the container down.
+sleep 1
+
 exit "$rc"

yarn-project/end-to-end/bootstrap.sh

@@ -34,12 +34,12 @@ function test_cmds {
   fi
   echo "$prefix:TIMEOUT=25m:NAME=e2e_block_building $(set_dump_avm e2e_block_building) $run_test_script simple e2e_block_building"
   echo "$prefix:TIMEOUT=30m:NAME=e2e_avm_simulator $(set_dump_avm e2e_avm_simulator) $run_test_script simple src/e2e_avm_simulator.test.ts"
-
-
+  echo "$prefix:TIMEOUT=15m:NAME=e2e_epochs/epochs_long_proving_time $run_test_script simple src/e2e_epochs/epochs_long_proving_time.test.ts"
 
   local tests=(
     # List all standalone and nested tests, except for the ones listed above.
-    src/e2e_!(prover)/*.test.ts
+    src/e2e_!(prover|epochs)/*.test.ts
+    src/e2e_epochs/!(epochs_long_proving_time).test.ts
     src/e2e_p2p/reqresp/*.test.ts
     src/e2e_!(block_building|avm_simulator).test.ts
   )
@@ -106,12 +106,8 @@ function test_cmds {
   # compose-based tests with custom scripts
   for flow in ../cli-wallet/test/flows/*.sh; do
     # Note these scripts are ran directly by docker-compose.yml because it ends in '.sh'.
-    # Set LOG_LEVEL=info for a better output experience. Deeper debugging should happen with other e2e tests.
-    if [[ "$flow" == *private_transfer.sh ]]; then
-      echo "$hash:ONLY_TERM_PARENT=1 LOG_LEVEL=info LOCAL_NETWORK_LOG_LEVEL='info; debug:p2p,sequencer,archiver,world-state,aztec-node' $run_test_script compose $flow"
-    else
-      echo "$hash:ONLY_TERM_PARENT=1 LOG_LEVEL=info $run_test_script compose $flow"
-    fi
+    # Run at LOG_LEVEL=verbose so the captured local-network logs are detailed enough for diagnostics.
+    echo "$hash:ONLY_TERM_PARENT=1 LOG_LEVEL=verbose $run_test_script compose $flow"
   done
 }
 

yarn-project/end-to-end/scripts/docker-compose.yml

@@ -19,13 +19,11 @@ services:
     working_dir: /root/aztec-packages/yarn-project/aztec
     entrypoint: >
       bash -c '
-        export LOG_LEVEL="$${LOCAL_NETWORK_LOG_LEVEL:-$${LOG_LEVEL:-verbose}}"
         set -o pipefail
         node ./dest/bin start --local-network 2>&1 | tee /logs/local-network.log
       '
     environment:
       LOG_LEVEL: ${LOG_LEVEL:-verbose}
-      LOCAL_NETWORK_LOG_LEVEL: ${LOCAL_NETWORK_LOG_LEVEL:-}
       ETHEREUM_HOSTS: http://fork:8545
       L1_CHAIN_ID: 31337
       FORCE_COLOR: ${FORCE_COLOR:-1}
@@ -87,11 +85,9 @@ services:
         while kill -0 -$$pgid 2>/dev/null; do sleep 0.1; done
         wait $$pid
         rc=$$?
-        if [ $$rc -ne 0 ]; then
-          echo "===== local-network logs ====="
-          cat /logs/local-network.log || true
-          echo "===== end local-network logs ====="
-        fi
+        echo "===== local-network logs ====="
+        cat /logs/local-network.log || true
+        echo "===== end local-network logs ====="
         exit $$rc
       '
     depends_on:

yarn-project/end-to-end/scripts/run_test.sh

@@ -18,7 +18,8 @@ case "$type" in
   ;;
   "compose")
     # TODO: Replace this file with test_simple.sh, and just emit the below as part of test_cmds.
-    TEST=$test exec run_compose_test $test end-to-end $PWD
+    # Remove volumes on cleanup so the local-network logs volume doesn't persist across runs.
+    TEST=$test REMOVE_COMPOSE_VOLUMES=1 exec run_compose_test $test end-to-end $PWD
   ;;
   "web3signer")
     TEST=$test exec run_compose_test $test end-to-end $PWD/web3signer

yarn-project/end-to-end/src/e2e_cross_chain_messaging/cross_chain_messaging_test.ts

@@ -15,11 +15,13 @@ import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract';
 import { pickL1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
 import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
 import { EpochNumber } from '@aztec/foundation/branded-types';
+import { retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import { TestERC20Abi, TestERC20Bytecode } from '@aztec/l1-artifacts';
 import { TokenContract } from '@aztec/noir-contracts.js/Token';
 import { TokenBridgeContract } from '@aztec/noir-contracts.js/TokenBridge';
 import type { PXEConfig } from '@aztec/pxe/server';
+import { getEpochAtSlot } from '@aztec/stdlib/epoch-helpers';
 import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client';
 
 import { MNEMONIC } from '../fixtures/fixtures.js';
@@ -102,6 +104,15 @@ export class CrossChainMessagingTest {
         fundSponsoredFPC: true,
         skipAccountDeployment: true,
         l1ContractsArgs: { ...this.deployL1ContractsArgs, ...opts.l1ContractsArgs },
+        // `advanceToEpochProven` warps anvil's L1 clock forward by up to a full epoch in one
+        // step. The prover-node tracks L1 time via `dateProvider.setTime(...)`, so any
+        // in-flight tx-gather sees its deadline jump into the past and short-circuits. Use
+        // a generous gather window so the deadline survives the warp.
+        proverNodeConfig: {
+          ...this.setupOptions.proverNodeConfig,
+          ...opts.proverNodeConfig,
+          txGatheringTimeoutMs: opts.proverNodeConfig?.txGatheringTimeoutMs ?? 10 * 60 * 1000,
+        },
       },
       { ...this.pxeOpts, ...pxeOpts },
     );
@@ -110,7 +121,14 @@ export class CrossChainMessagingTest {
 
   async advanceToEpochProven(l2TxReceipt: TxReceipt): Promise<EpochNumber> {
     const block = await this.aztecNode.getBlock(l2TxReceipt.blockNumber!);
-    const epoch = await this.rollup.getEpochNumberForCheckpoint(block!.checkpointNumber);
+    const cp = await retryUntil(
+      async () => (await this.aztecNode.getCheckpoints(block!.checkpointNumber, 1))[0],
+      `archiver indexes checkpoint ${block!.checkpointNumber}`,
+      120,
+      0.5,
+    );
+    const epochDuration = await this.rollup.getEpochDuration();
+    const epoch = getEpochAtSlot(cp.header.slotNumber, { epochDuration });
     // Warp to the next epoch.
     await this.cheatCodes.rollup.advanceToEpoch(EpochNumber(epoch + 1));
     // Wait for the tx to be proven.

yarn-project/end-to-end/src/e2e_cross_chain_messaging/l2_to_l1.test.ts

@@ -10,7 +10,7 @@ import { type Sequencer, type SequencerEvents, SequencerState } from '@aztec/seq
 import { computeL2ToL1MessageHash } from '@aztec/stdlib/hash';
 import type { AztecNode, AztecNodeAdmin } from '@aztec/stdlib/interfaces/client';
 import { type L2ToL1MembershipWitness, getL2ToL1MessageLeafId } from '@aztec/stdlib/messaging';
-import { type TxHash, TxStatus } from '@aztec/stdlib/tx';
+import { TxExecutionResult, type TxHash, TxStatus } from '@aztec/stdlib/tx';
 
 import { jest } from '@jest/globals';
 import { type Hex, decodeEventLog } from 'viem';
@@ -114,6 +114,73 @@ describe('e2e_cross_chain_messaging l2_to_l1', () => {
     await expectConsumeMessageToSucceed(messages[1], txReceipt.txHash);
   });
 
+  // A message-bearing tx that gets reorged out of its checkpoint and remined into a fresh
+  // one must still prove correctly — the message has to follow the tx into its new home and
+  // end up in the epoch out-hash. A successful outbox consume after `advanceToEpochProven`
+  // proves the message survived the reorg+remine all the way through to a valid epoch proof.
+  it('proves an L2-to-L1 message whose tx is reorged out and remined', async () => {
+    const recipient = msgSender;
+    const content = Fr.random();
+    const message = makeL2ToL1Message(recipient, content);
+
+    // One tx per block so the message-bearing tx owns its checkpoint.
+    await aztecNodeAdmin.setConfig({ minTxsPerBlock: 1 });
+    await waitForSequencerIdle(t.context.sequencer!.getSequencer());
+
+    // Send the message-bearing tx and note where it first landed.
+    const { receipt: txReceipt } = await contract.methods
+      .create_l2_to_l1_message_arbitrary_recipient_private(content, recipient)
+      .send({ from: user1Address });
+    const originalBlock = (await aztecNode.getBlock(txReceipt.blockNumber!))!;
+    const originalCheckpoint = originalBlock.checkpointNumber;
+    t.logger.info(`Message tx landed in checkpoint ${originalCheckpoint} (block ${txReceipt.blockNumber})`);
+
+    // Reorg L1 deeply enough to drop the L1 block that published this checkpoint.
+    const [cp] = await aztecNode.getCheckpoints(originalCheckpoint, 1, { includeL1PublishInfo: true });
+    if (!cp.l1.published) {
+      throw new Error(`Expected checkpoint ${originalCheckpoint} to have L1 publish info`);
+    }
+    const checkpointL1Block = Number(cp.l1.blockNumber);
+    const currentL1Block = await t.context.cheatCodes.eth.blockNumber();
+    const reorgDepth = currentL1Block - checkpointL1Block + 1;
+    t.logger.info(`Reorging ${reorgDepth} L1 blocks to remove checkpoint ${originalCheckpoint}`);
+    await t.context.cheatCodes.eth.reorgWithReplacement(reorgDepth);
+
+    // The node detects the prune and drops back below the reorged-out checkpoint.
+    await retryUntil(
+      () => aztecNode.getCheckpointNumber('checkpointed').then(cpNum => cpNum < originalCheckpoint),
+      'node detects reorg',
+      60,
+      0.5,
+    );
+    t.logger.info(`Node observed the reorg removing checkpoint ${originalCheckpoint}`);
+
+    // The tx returns to the mempool and is remined. Poll for a successful receipt whose
+    // checkpoint is at or beyond the reorged-out one (i.e. the freshly-mined instance,
+    // not a stale read of the removed block).
+    const reminedReceipt = await retryUntil(
+      async () => {
+        const r = await aztecNode.getTxReceipt(txReceipt.txHash);
+        if (r.executionResult !== TxExecutionResult.SUCCESS || !r.blockNumber) {
+          return undefined;
+        }
+        const block = await aztecNode.getBlock(r.blockNumber);
+        return block && block.checkpointNumber >= originalCheckpoint ? r : undefined;
+      },
+      'tx remined after reorg',
+      120,
+      0.5,
+    );
+    const reminedBlock = (await aztecNode.getBlock(reminedReceipt.blockNumber!))!;
+    t.logger.info(
+      `Message tx remined into checkpoint ${reminedBlock.checkpointNumber} (block ${reminedReceipt.blockNumber})`,
+    );
+
+    // Prove the epoch containing the remined tx, then consume its message from the outbox.
+    await t.advanceToEpochProven(reminedReceipt);
+    await expectConsumeMessageToSucceed(message, txReceipt.txHash);
+  });
+
   // When the block contains a tx with no messages, the zero txOutHash is skipped and won't be included in the top tree.
   // In this test, we test that the correct tree class is used, and the final out hash equals the only message leaf.
   it('2 txs in the same block, one with no messages, one with a message', async () => {

yarn-project/end-to-end/src/e2e_epochs/epochs_long_proving_time.test.ts

@@ -6,7 +6,9 @@ import { jest } from '@jest/globals';
 
 import { EpochsTestContext } from './epochs_test.js';
 
-jest.setTimeout(1000 * 60 * 10);
+jest.setTimeout(1000 * 60 * 15);
+
+const MAX_JOB_COUNT = 20;
 
 describe('e2e_epochs/epochs_long_proving_time', () => {
   let logger: Logger;
@@ -24,11 +26,14 @@ describe('e2e_epochs/epochs_long_proving_time', () => {
     const { aztecSlotDuration } = EpochsTestContext.getSlotDurations({ aztecEpochDuration });
     const epochDurationInSeconds = aztecSlotDuration * aztecEpochDuration;
     const proverTestDelayMs = (epochDurationInSeconds * 1000 * 3) / 4;
+    // Each epoch takes ~3 epochs to prove, so the broker needs to keep results for
+    // at least that many epochs to avoid rejecting jobs as stale.
     test = await EpochsTestContext.setup({
       aztecEpochDuration,
       aztecProofSubmissionEpochs: 1000, // Effectively don't re-org
       proverTestDelayMs,
-      proverNodeMaxPendingJobs: 1, // We test for only a single job at once
+      proverNodeMaxPendingJobs: MAX_JOB_COUNT, // Prove multiple epochs concurrently
+      proverBrokerMaxEpochsToKeepResultsFor: 10,
     });
     ({ logger, monitor, L1_BLOCK_TIME_IN_S } = test);
     logger.warn(`Initialized with prover delay set to ${proverTestDelayMs}ms (epoch is ${epochDurationInSeconds}s)`);
@@ -58,10 +63,7 @@ describe('e2e_epochs/epochs_long_proving_time', () => {
     // At least 3 epochs should have passed after the proven one (though we add a -1 just in case)
     expect(monitor.checkpointNumber).toBeGreaterThanOrEqual(targetProvenEpochs * test.epochDuration * 3 - 1);
 
-    // We expect maxJobCount to equal 1, since the prover node epoch monitor defines an epoch as ready to be proven
-    // only if the previous one has already been proven. We can relax this check if we want to support multiple epochs
-    // to be proven in parallel, in which case we should update the assertion below.
-    expect(maxJobCount).toEqual(1);
-    logger.info(`Test succeeded`);
+    expect(maxJobCount).toBeLessThanOrEqual(MAX_JOB_COUNT);
+    logger.info(`Test succeeded, max prover jobs ${maxJobCount}`);
   });
 });

yarn-project/end-to-end/src/e2e_epochs/epochs_multi_proof.test.ts

@@ -46,18 +46,18 @@ describe('e2e_epochs/epochs_multi_proof', () => {
     // This prevents the race condition where multiple provers submit to L1 at the same time
     test.proverNodes.forEach((proverAztecNode, index) => {
       const proverManager = proverAztecNode.getProverNode()!.getProver();
-      const origCreateEpochProver = proverManager.createEpochProver.bind(proverManager);
-      proverManager.createEpochProver = () => {
-        const epochProver = origCreateEpochProver();
-        const origFinalizeEpoch = epochProver.finalizeEpoch.bind(epochProver);
-        epochProver.finalizeEpoch = async () => {
-          const result = await origFinalizeEpoch();
+      const origCreateTopTree = proverManager.createTopTreeOrchestrator.bind(proverManager);
+      proverManager.createTopTreeOrchestrator = () => {
+        const topTree = origCreateTopTree();
+        const origProve = topTree.prove.bind(topTree);
+        topTree.prove = async (...args: Parameters<typeof origProve>) => {
+          const result = await origProve(...args);
           const sleepTime = index * 1000 * test.constants.ethereumSlotDuration;
-          logger.warn(`Delaying finalizeEpoch for prover node ${index} by ${sleepTime}ms`);
+          logger.warn(`Delaying top-tree prove for prover node ${index} by ${sleepTime}ms`);
           await sleep(sleepTime);
           return result;
         };
-        return epochProver;
+        return topTree;
       };
     });