From 8212373ab0e6f95a57383d6b453afb25e149a582 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 17:45:25 -0300 Subject: [PATCH] test(e2e): fix 'proposer invalidates multiple checkpoints' timeout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Test Summary** `proposer invalidates multiple checkpoints` verifies that two intended bad checkpoints land with insufficient attestations, a later good proposer invalidates the first bad checkpoint, and the chain then progresses. **Failed Run Error** CI run `8b1c0f4ec6031f2b` timed out at Jest’s 600s limit. The failure was not the shutdown L1 send error; that happened after the timeout while teardown was interrupting pending work. **Failed vs Successful Divergence** First meaningful divergence: checkpoint 4 at slot 23. Failed log: slot 23 published checkpoint 4 with only 1 attestation, then archivers reported `Insufficient attestations ... actualAttestations:1`. Successful log: slot 23 collected all 5 attestations before publishing checkpoint 4, so the first intentionally bad checkpoints were later. **Timeline** Failed: - `15:59:11` selected intended bad slots 25/26, applied bad config to proposer `0x15...` - `15:59:35` slot 23 job prepared by that same proposer - `16:00:15` checkpoint 4 at slot 23 landed with 1 attestation - repeated rollback/retry consumed enough time to hit Jest timeout Successful: - slot 23 checkpoint landed cleanly with 5 attestations - intended bad checkpoints at slots 24/25 landed with 1 attestation - checkpoint 5 was invalidated - test completed successfully **Hypothesis** High confidence: the test’s bad-slot selection only excluded `candidateSlot1 - 1` as a pre-bad pipelined target. In the failed run, `candidateSlot1 - 2` was still unsnapshotted and owned by a bad proposer, so applying malicious config leaked into slot 23. **Evidence** - Logs: failed run selected slots 25/26 but slot 23 later published with 1 attestation from the newly bad proposer. - Source: pipelined checkpoint jobs snapshot sequencer config when the target-slot job is created, so applying config while sequencers are running can affect any not-yet-created pre-bad job. - Skeptic check: no contradiction found; it also caught a broken local timeout race. **Proposed Fix** Implemented in [epochs_invalidate_block.parallel.test.ts](/home/santiago/Projects/aztec-1/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts:393): the selector now excludes bad proposers from every pre-bad target slot from `currentSlot + 2` through `candidateSlot1 - 1`, not just the immediately prior slot. Also fixed the broken timeout race at [line 475](/home/santiago/Projects/aztec-1/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts:475) by removing the accidental inner `await`. --- .../epochs_invalidate_block.parallel.test.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts index d3ecedaac027..2d8323251021 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts @@ -390,31 +390,40 @@ describe('e2e_epochs/epochs_invalidate_block', () => { const { l2SlotNumber: currentSlot } = await test.monitor.run(); logger.warn(`First checkpoint mined, current slot is ${currentSlot}`); - // The bad config is applied while sequencers are already running; skip pairs where the prior pipelined - // target slot could snapshot that config before the intended bad slots. + // The bad config is applied while sequencers are already running; skip pairs where a pipelined + // pre-bad target slot could snapshot that config before the intended bad slots. let badSlot1: SlotNumber | undefined; let badSlot2: SlotNumber | undefined; let badProposers: EthAddress[] = []; const firstCandidateSlot = Number(currentSlot) + 3; + const firstUnsnapshottedTargetSlot = SlotNumber.add(currentSlot, 2); const maxBadSlotSearchAttempts = 20; for (let attempt = 0; attempt < maxBadSlotSearchAttempts && badSlot1 === undefined; attempt++) { const candidateSlot1 = SlotNumber(firstCandidateSlot + attempt); const candidateSlot2 = SlotNumber.add(candidateSlot1, 1); - const priorPipelinedTargetSlot = SlotNumber.add(candidateSlot1, -1); - const [priorProposer, p1, p2] = await Promise.all([ - test.epochCache.getProposerAttesterAddressInSlot(priorPipelinedTargetSlot), + const preBadTargetSlots = range( + Math.max(0, Number(candidateSlot1) - Number(firstUnsnapshottedTargetSlot)), + Number(firstUnsnapshottedTargetSlot), + ).map(SlotNumber); + const [preBadProposers, p1, p2] = await Promise.all([ + Promise.all(preBadTargetSlots.map(slot => test.epochCache.getProposerAttesterAddressInSlot(slot))), test.epochCache.getProposerAttesterAddressInSlot(candidateSlot1), test.epochCache.getProposerAttesterAddressInSlot(candidateSlot2), ]); logger.warn(`Checking bad checkpoint slots ${candidateSlot1} and ${candidateSlot2}`, { - priorPipelinedTargetSlot, - priorProposer: priorProposer?.toString(), + preBadTargetSlots, + preBadProposers: preBadProposers.map(proposer => proposer?.toString()), p1: p1?.toString(), p2: p2?.toString(), }); - if (p1 && p2 && !priorProposer?.equals(p1) && !priorProposer?.equals(p2)) { + const badProposerHasUnsnapshottedPreBadSlot = + p1 !== undefined && + p2 !== undefined && + preBadProposers.some(proposer => proposer !== undefined && (proposer.equals(p1) || proposer.equals(p2))); + + if (p1 && p2 && !badProposerHasUnsnapshottedPreBadSlot) { badSlot1 = candidateSlot1; badSlot2 = candidateSlot2; badProposers = [p1, p2]; @@ -464,7 +473,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { // Wait for both checkpoints to be mined logger.warn(`Waiting for two checkpoints to be mined on slots ${expectedFirstSlot} and ${expectedSecondSlot}`); const [firstCheckpoint, secondCheckpoint] = await Promise.race([ - await Promise.all([firstCheckpointPromise.promise, secondCheckpointPromise.promise]), + Promise.all([firstCheckpointPromise.promise, secondCheckpointPromise.promise]), timeoutPromise(test.L2_SLOT_DURATION_IN_S * 8 * 1000).then(() => [CheckpointNumber(0), CheckpointNumber(0)]), ]);