From 9be46bc9d07c96b223300a5137e3987504ae2162 Mon Sep 17 00:00:00 2001 From: cheethas Date: Mon, 3 Apr 2023 13:48:26 +0000 Subject: [PATCH 01/27] fix: synched -> synced more common spelling --- yarn-project/sequencer-client/src/sequencer/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn-project/sequencer-client/src/sequencer/index.ts b/yarn-project/sequencer-client/src/sequencer/index.ts index 69a46c77a3a8..a5d65cf4a5e4 100644 --- a/yarn-project/sequencer-client/src/sequencer/index.ts +++ b/yarn-project/sequencer-client/src/sequencer/index.ts @@ -95,7 +95,7 @@ export class Sequencer { protected async work() { try { // Update state when the previous block has been synched - const prevBlockSynched = await this.isBlockSynched(); + const prevBlockSynched = await this.isBlockSynced(); if (prevBlockSynched && this.state === SequencerState.PUBLISHING_BLOCK) { this.log(`Block has been synched`); this.state = SequencerState.IDLE; @@ -157,9 +157,9 @@ export class Sequencer { /** * Returns whether the previous block sent has been mined, and all dependencies have caught up with it. - * @returns Boolean indicating if our dependencies are synched to the latest block. + * @returns Boolean indicating if our dependencies are synced to the latest block. */ - protected async isBlockSynched() { + protected async isBlockSynced() { return ( (await this.worldState.status().then((s: WorldStateStatus) => s.syncedToL2Block)) >= this.lastBlockNumber && (await this.p2pClient.getStatus().then(s => s.syncedToL2Block)) >= this.lastBlockNumber From a6f6a0dadbdd70b5180474d0d47ab62604bcfdea Mon Sep 17 00:00:00 2001 From: cheethas Date: Mon, 3 Apr 2023 15:24:28 +0000 Subject: [PATCH 02/27] feat(merkle_tree): initial nullifier tree changes --- .../src/indexed_tree/indexed_tree.ts | 150 ++++++++++++++++++ .../circuit_powered_block_builder.ts | 14 +- .../sequencer-client/src/sequencer/index.ts | 10 +- 3 files changed, 163 insertions(+), 11 deletions(-) diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index 71443dc580be..d21a88416539 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -27,6 +27,23 @@ export interface LeafData { nextValue: bigint; } +export interface BaseRollupBatchInsertionProofData { + /** + * Preimage of the low nullifier that proves non membership + */ + lowNullifierLeafPreimages: LeafData[]; + /** + * Sibling path to prove membership of low nullifier + */ + lowNullifierMembershipWitnesses: SiblingPath[]; + // /** + // * Sibling path of the batch insertion subtree root + // * For efficiently proving insertion of all nullifiers in the batch + // */ + // newNullifiersSubtreeInsertionPath: SiblingPath; +} + + // eslint-disable-next-line @typescript-eslint/no-unused-vars const encodeTreeValue = (leafData: LeafData) => { const valueAsBuffer = toBufferBE(leafData.value, 32); @@ -124,6 +141,137 @@ export class IndexedTree implements MerkleTree { } } + + // TODO: this will have alot more information returned from it to craft the correct inputs + // TODO: should this function similate on a copy of the tree rather than making actual changes? + /** + * Each base rollup needs to provide non membership / inclusion proofs for each of the nullifiers + * generated in the kernel circuit that it is rolling up. + * + * As leaves are batch inserted at the end, batch updates are a special case. + * + * WARNING: This function has side effects, it will insert values into the tree. + * + * TODO: include indepth insertion writeup in this comment + * @param leaves Values to insert into the tree + * @returns + */ + // TODO: assumptions + // 1. There are 8 nullifiers provided and they are all unique + // 2. If kc 0 has 1 nullifier, and kc 1 has 3 nullifiers the layout will assume to be the sparse + // nullifier layout: [kc0N, 0, 0, 0, kc1N, kc1N, kc1N, 0] + public async getAndPerformBaseRollupBatchInsertionProofs(leaves: Buffer[]): Promise { + // Keep track of the touched during batch insertion + const touchedNodes: Set = new Set(); + const lowNullifiers: LeafData[] = []; + const lowNullifierIndexes: bigint[] = []; + const lowNullifierSiblingPaths: SiblingPath[] = []; + const startInsertionIndex: bigint = this.getNumLeaves(); + let currInsertionIndex: bigint = startInsertionIndex; + + // Leaf data of hte leaves to be inserted + const insertionSubtree: LeafData[] = []; + + // Low nullifier membership proof sibling paths + for (const leaf of leaves) { + const newValue = toBigIntBE(leaf); + const indexOfPrevious = this.findIndexOfPreviousValue(newValue); + + // NOTE: null values for nullfier leaves are being changed to 0n current impl is a hack + // Default value + let nullifierLeaf: LeafData = { + value: 0n, + nextIndex: 0n, + nextValue: 0n + }; + + if (touchedNodes.has(indexOfPrevious.index)) { + // If the node has already been touched, then we return an empty leaf and sibling path + const emptySP = new SiblingPath(); + emptySP.data = Array(this.underlying.getDepth()).fill(Buffer.from("0000000000000000000000000000000000000000000000000000000000000000", "hex")); + lowNullifierSiblingPaths.push(emptySP); + lowNullifiers.push(initialLeaf); + lowNullifierIndexes.push(0n); + } else { + // If the node has not been touched, we update its low nullifier pointer, but we do NOT insert it yet, inserting it now + // will alter non membership paths of the not yet inserted members + // Insertion is done at the end once updates have already occured. + touchedNodes.add(indexOfPrevious.index); + + const newValue = toBigIntBE(leaf); + const lowNullifier = this.getLatestLeafDataCopy(indexOfPrevious.index); + + if (!lowNullifier) { + // TODO: work out what to throw if the index is not found + return null; + } + + // Get sibling path for existence of the old leaf + const siblingPath = await this.underlying.getSiblingPath(BigInt(indexOfPrevious.index)); + + // Update the running paths + lowNullifierSiblingPaths.push(siblingPath); + lowNullifierIndexes.push(BigInt(indexOfPrevious.index)); + lowNullifiers.push(lowNullifier); + + + // Update subtree insertion leaf from null data + nullifierLeaf = { + value: newValue, + nextIndex: lowNullifier.nextIndex, + nextValue: lowNullifier.nextValue, + }; + + // Update the current low nullifier + lowNullifier.nextIndex = currInsertionIndex; + lowNullifier.nextValue = BigInt(newValue); + + // Update the old leaf in the tree + this.cachedLeaves[Number(indexOfPrevious.index)] = lowNullifier; + await this.underlying.updateLeaf(hashEncodedTreeValue(lowNullifier, this.hasher), BigInt(indexOfPrevious.index)); + } + + // increment insertion index + currInsertionIndex++; + insertionSubtree.push(nullifierLeaf); + } + + // Create insertion subtree and forcefully insert in series + // Here we calculate the pointers for the inserted values, if they have not already been updated + for (let i = 0 ; i < leaves.length; i++) { + const newValue = toBigIntBE(leaves[i]); + + // We have already fetched the new low nullifier for this leaf, so we can set its low nullifier + const lowNullifier = lowNullifiers[i]; + // If the lowNullifier is 0, then we check the previous leaves for the low nullifier leaf + if (lowNullifier.value === 0n && lowNullifier.nextIndex === 0n && lowNullifier.nextValue === 0n) { + for (let j = 0; j < i; j++) { + if ((insertionSubtree[j].nextValue >newValue&& + insertionSubtree[j].value < newValue) || + (insertionSubtree[j].nextValue == 0n && insertionSubtree[j].nextIndex == 0n)) { + + insertionSubtree[j].nextIndex = startInsertionIndex + BigInt(i); + insertionSubtree[j].nextValue = newValue; + } + } + } + } + + // For each calculated new leaf, we insert it into the tree at the next position + for (let i = 0 ; i < leaves.length; i++) { + // TODO: can we skip inserting the empty values + this.cachedLeaves[Number(startInsertionIndex)+i] = insertionSubtree[i]; + await this.underlying.appendLeaves([hashEncodedTreeValue(insertionSubtree[i], this.hasher)]); + } + + return { + lowNullifierLeafPreimages: lowNullifiers, + lowNullifierMembershipWitnesses: lowNullifierSiblingPaths, + } + } + + + /** * Commits the changes to the database. * @returns Empty promise. @@ -229,6 +377,8 @@ export class IndexedTree implements MerkleTree { * Saves the initial leaf to this object and saves it to a database. */ private async init() { + // TODO: increase the initial size of the tree to the size of a full rollup insertion - change reflected in c++ to allow subtree insertion + this.leaves.push(initialLeaf); await this.underlying.appendLeaves([hashEncodedTreeValue(initialLeaf, this.hasher)]); await this.commit(); diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index c84bdd21b5db..e12b3fd000a5 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -415,10 +415,12 @@ export class CircuitPoweredBlockBuilder { // await this.getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE).then(t => '0x' + t.root.toBuffer().toString('hex')), // ); // console.log(`Inserting new data`, newNullifiers.join(', ')); - for (const nullifier of newNullifiers) { - lowNullifierInfos.push(await this.getLowNullifierInfo(nullifier)); - await this.db.appendLeaves(MerkleTreeId.NULLIFIER_TREE, [nullifier.toBuffer()]); - } + + const nullifierWitnesses = await this.db.getAndPerformBaseRollupBatchInsertionProofs( + MerkleTreeId.NULLIFIER_TREE, + newNullifiers + ); + // console.log( // `Nullifier root after insertion: `, // await this.getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE).then(t => '0x' + t.root.toBuffer().toString('hex')), @@ -446,8 +448,8 @@ export class CircuitPoweredBlockBuilder { newCommitmentsSubtreeSiblingPath, newContractsSubtreeSiblingPath, newNullifiersSubtreeSiblingPath, - lowNullifierLeafPreimages: lowNullifierInfos.map(i => i.leafPreimage), - lowNullifierMembershipWitness: lowNullifierInfos.map(i => i.witness), + lowNullifierLeafPreimages: nullifierWitnesses.lowNullifierLeafPreimages, + lowNullifierMembershipWitness: nullifierWitnesses.lowNullifierMembershipWitness, kernelData: [this.getKernelDataFor(tx1), this.getKernelDataFor(tx2)], historicContractsTreeRootMembershipWitnesses: [ await this.getContractMembershipWitnessFor(tx1), diff --git a/yarn-project/sequencer-client/src/sequencer/index.ts b/yarn-project/sequencer-client/src/sequencer/index.ts index a5d65cf4a5e4..7eb31272ab39 100644 --- a/yarn-project/sequencer-client/src/sequencer/index.ts +++ b/yarn-project/sequencer-client/src/sequencer/index.ts @@ -94,15 +94,15 @@ export class Sequencer { */ protected async work() { try { - // Update state when the previous block has been synched - const prevBlockSynched = await this.isBlockSynced(); - if (prevBlockSynched && this.state === SequencerState.PUBLISHING_BLOCK) { - this.log(`Block has been synched`); + // Update state when the previous block has been synced + const prevBlockSynced = await this.isBlockSynced(); + if (prevBlockSynced && this.state === SequencerState.PUBLISHING_BLOCK) { + this.log(`Block has been synced`); this.state = SequencerState.IDLE; } // Do not go forward with new block if the previous one has not been mined and processed - if (!prevBlockSynched) { + if (!prevBlockSynced) { return; } From 2abdb32406479eda470a6214d1e33f3bba212da2 Mon Sep 17 00:00:00 2001 From: cheethas Date: Mon, 3 Apr 2023 15:38:19 +0000 Subject: [PATCH 03/27] fix: minor naming and types refactor --- .../src/indexed_tree/indexed_tree.ts | 48 +++++++++---------- .../circuit_powered_block_builder.ts | 9 +++- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index d21a88416539..0f3cb5b5c0cf 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -27,20 +27,19 @@ export interface LeafData { nextValue: bigint; } -export interface BaseRollupBatchInsertionProofData { +export interface LowNullifierWitnessData { /** * Preimage of the low nullifier that proves non membership */ - lowNullifierLeafPreimages: LeafData[]; + preimage: LeafData; /** * Sibling path to prove membership of low nullifier */ - lowNullifierMembershipWitnesses: SiblingPath[]; - // /** - // * Sibling path of the batch insertion subtree root - // * For efficiently proving insertion of all nullifiers in the batch - // */ - // newNullifiersSubtreeInsertionPath: SiblingPath; + siblingPath: SiblingPath; + /** + * The index of low nullifier + */ + index: bigint; } @@ -160,12 +159,11 @@ export class IndexedTree implements MerkleTree { // 1. There are 8 nullifiers provided and they are all unique // 2. If kc 0 has 1 nullifier, and kc 1 has 3 nullifiers the layout will assume to be the sparse // nullifier layout: [kc0N, 0, 0, 0, kc1N, kc1N, kc1N, 0] - public async getAndPerformBaseRollupBatchInsertionProofs(leaves: Buffer[]): Promise { + public async getAndPerformBaseRollupBatchInsertionProofs(leaves: Buffer[]): Promise { // Keep track of the touched during batch insertion const touchedNodes: Set = new Set(); - const lowNullifiers: LeafData[] = []; - const lowNullifierIndexes: bigint[] = []; - const lowNullifierSiblingPaths: SiblingPath[] = []; + + const lowNullifierWitnesses: LowNullifierWitnessData[] = []; const startInsertionIndex: bigint = this.getNumLeaves(); let currInsertionIndex: bigint = startInsertionIndex; @@ -189,9 +187,12 @@ export class IndexedTree implements MerkleTree { // If the node has already been touched, then we return an empty leaf and sibling path const emptySP = new SiblingPath(); emptySP.data = Array(this.underlying.getDepth()).fill(Buffer.from("0000000000000000000000000000000000000000000000000000000000000000", "hex")); - lowNullifierSiblingPaths.push(emptySP); - lowNullifiers.push(initialLeaf); - lowNullifierIndexes.push(0n); + const witness: LowNullifierWitnessData = { + preimage: initialLeaf, + index: 0n, + siblingPath: emptySP + }; + lowNullifierWitnesses.push(witness); } else { // If the node has not been touched, we update its low nullifier pointer, but we do NOT insert it yet, inserting it now // will alter non membership paths of the not yet inserted members @@ -210,10 +211,12 @@ export class IndexedTree implements MerkleTree { const siblingPath = await this.underlying.getSiblingPath(BigInt(indexOfPrevious.index)); // Update the running paths - lowNullifierSiblingPaths.push(siblingPath); - lowNullifierIndexes.push(BigInt(indexOfPrevious.index)); - lowNullifiers.push(lowNullifier); - + const witness = { + preimage: lowNullifier, + index: BigInt(indexOfPrevious.index), + siblingPath: siblingPath + } + lowNullifierWitnesses.push(witness); // Update subtree insertion leaf from null data nullifierLeaf = { @@ -242,7 +245,7 @@ export class IndexedTree implements MerkleTree { const newValue = toBigIntBE(leaves[i]); // We have already fetched the new low nullifier for this leaf, so we can set its low nullifier - const lowNullifier = lowNullifiers[i]; + const lowNullifier = lowNullifierWitnesses[i].preimage; // If the lowNullifier is 0, then we check the previous leaves for the low nullifier leaf if (lowNullifier.value === 0n && lowNullifier.nextIndex === 0n && lowNullifier.nextValue === 0n) { for (let j = 0; j < i; j++) { @@ -264,10 +267,7 @@ export class IndexedTree implements MerkleTree { await this.underlying.appendLeaves([hashEncodedTreeValue(insertionSubtree[i], this.hasher)]); } - return { - lowNullifierLeafPreimages: lowNullifiers, - lowNullifierMembershipWitnesses: lowNullifierSiblingPaths, - } + return lowNullifierWitnesses; } diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index e12b3fd000a5..29c48ef67a36 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -420,6 +420,10 @@ export class CircuitPoweredBlockBuilder { MerkleTreeId.NULLIFIER_TREE, newNullifiers ); + const lowNullifierMembershipWitnesses: MembershipWitness = nullifierWitnesses.map(w => ({ + leafIndex: w.index, + siblingPath: w.witness.siblingPath.map((b: Buffer) => Fr.fromBuffer(b)), + })) // console.log( // `Nullifier root after insertion: `, @@ -448,8 +452,9 @@ export class CircuitPoweredBlockBuilder { newCommitmentsSubtreeSiblingPath, newContractsSubtreeSiblingPath, newNullifiersSubtreeSiblingPath, - lowNullifierLeafPreimages: nullifierWitnesses.lowNullifierLeafPreimages, - lowNullifierMembershipWitness: nullifierWitnesses.lowNullifierMembershipWitness, + // TODO: typing for `LowNullifierWitnessData` + lowNullifierLeafPreimages: nullifierWitnesses.map((w: any) => w.preimage), + lowNullifierMembershipWitness: lowNullifierMembershipWitnesses, kernelData: [this.getKernelDataFor(tx1), this.getKernelDataFor(tx2)], historicContractsTreeRootMembershipWitnesses: [ await this.getContractMembershipWitnessFor(tx1), From 12fd380e1b983179b1a968ebaec6ee3a89d1bf87 Mon Sep 17 00:00:00 2001 From: cheethas Date: Mon, 3 Apr 2023 16:18:37 +0000 Subject: [PATCH 04/27] fix: type coersion --- yarn-project/circuits.js/src/structs/shared.ts | 9 +++++++++ .../merkle-tree/src/indexed_tree/indexed_tree.ts | 7 +++---- .../block_builder/circuit_powered_block_builder.ts | 11 ++++++----- .../synchroniser/server_world_state_synchroniser.ts | 3 +++ yarn-project/world-state/src/world-state-db/index.ts | 2 ++ 5 files changed, 23 insertions(+), 9 deletions(-) diff --git a/yarn-project/circuits.js/src/structs/shared.ts b/yarn-project/circuits.js/src/structs/shared.ts index 029290f056ab..1770849e0529 100644 --- a/yarn-project/circuits.js/src/structs/shared.ts +++ b/yarn-project/circuits.js/src/structs/shared.ts @@ -1,5 +1,6 @@ import { BufferReader, randomBytes } from '@aztec/foundation'; import { Fq, Fr } from '@aztec/foundation/fields'; +import { SiblingPath } from '@aztec/merkle-tree'; import { assertLength, range } from '../utils/jsUtils.js'; import { Bufferable, serializeToBuffer } from '../utils/serialize.js'; import times from 'lodash.times'; @@ -27,6 +28,14 @@ export class MembershipWitness { .map(() => Fr.ZERO); return new MembershipWitness(pathSize, leafIndex, arr); } + + static fromSiblingPath(leafIndex: number, siblingPath: SiblingPath) { + return new MembershipWitness( + siblingPath.data.length, + leafIndex, + siblingPath.data.map(x => Fr.fromBuffer(x)), + ); + } } export class AggregationObject { diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index 0f3cb5b5c0cf..666f501c051b 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -159,7 +159,7 @@ export class IndexedTree implements MerkleTree { // 1. There are 8 nullifiers provided and they are all unique // 2. If kc 0 has 1 nullifier, and kc 1 has 3 nullifiers the layout will assume to be the sparse // nullifier layout: [kc0N, 0, 0, 0, kc1N, kc1N, kc1N, 0] - public async getAndPerformBaseRollupBatchInsertionProofs(leaves: Buffer[]): Promise { + public async getAndPerformBaseRollupBatchInsertionProofs(leaves: Buffer[]): Promise { // Keep track of the touched during batch insertion const touchedNodes: Set = new Set(); @@ -202,9 +202,8 @@ export class IndexedTree implements MerkleTree { const newValue = toBigIntBE(leaf); const lowNullifier = this.getLatestLeafDataCopy(indexOfPrevious.index); - if (!lowNullifier) { - // TODO: work out what to throw if the index is not found - return null; + if (lowNullifier === undefined) { + throw new Error(`Previous leaf not found!`); } // Get sibling path for existence of the old leaf diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 29c48ef67a36..4a055ac44e0b 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -11,6 +11,7 @@ import { PreviousKernelData, PreviousRollupData, PRIVATE_DATA_TREE_ROOTS_TREE_HEIGHT, + PRIVATE_DATA_TREE_HEIGHT, ROLLUP_VK_TREE_HEIGHT, RootRollupInputs, RootRollupPublicInputs, @@ -416,14 +417,14 @@ export class CircuitPoweredBlockBuilder { // ); // console.log(`Inserting new data`, newNullifiers.join(', ')); + // TODO: handle exception const nullifierWitnesses = await this.db.getAndPerformBaseRollupBatchInsertionProofs( MerkleTreeId.NULLIFIER_TREE, - newNullifiers + newNullifiers.map(fr => fr.toBuffer()) ); - const lowNullifierMembershipWitnesses: MembershipWitness = nullifierWitnesses.map(w => ({ - leafIndex: w.index, - siblingPath: w.witness.siblingPath.map((b: Buffer) => Fr.fromBuffer(b)), - })) + // Extract witness objects from returned data + const lowNullifierMembershipWitnesses = nullifierWitnesses.map(w => MembershipWitness.fromSiblingPath(Number(w.index), w.siblingPath), + ) // console.log( // `Nullifier root after insertion: `, diff --git a/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts b/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts index 8bc7a12fdb61..0cf82d630466 100644 --- a/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts +++ b/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts @@ -3,6 +3,9 @@ import { L2Block, L2BlockDownloader, L2BlockSource } from '@aztec/l2-block'; import { MerkleTreeDb, MerkleTreeId, MerkleTreeOperations } from '../index.js'; import { MerkleTreeOperationsFacade } from '../merkle-tree/merkle_tree_operations_facade.js'; import { WorldStateRunningState, WorldStateStatus, WorldStateSynchroniser } from './world_state_synchroniser.js'; +import { MerkleTreeDb, MerkleTreeId, TreeInfo } from '../index.js'; +import { LeafData, SiblingPath } from '@aztec/merkle-tree'; +import { LowNullifierWitnessData } from '@aztec/merkle-tree'; /** * Synchronises the world state with the L2 blocks from a L2BlockSource. diff --git a/yarn-project/world-state/src/world-state-db/index.ts b/yarn-project/world-state/src/world-state-db/index.ts index d13ea4db461e..61277eb5ffc7 100644 --- a/yarn-project/world-state/src/world-state-db/index.ts +++ b/yarn-project/world-state/src/world-state-db/index.ts @@ -1,3 +1,4 @@ +import { LowNullifierWitnessData } from '@aztec/merkle-tree'; import { LeafData, SiblingPath } from '@aztec/merkle-tree'; export * from './merkle_trees.js'; @@ -100,6 +101,7 @@ export interface MerkleTreeOperations { * @param index - The index of the leaf */ getLeafValue(treeId: MerkleTreeId, index: bigint): Promise; + getAndPerformBaseRollupBatchInsertionProofs(treeId: MerkleTreeId, leaves: Buffer[]): Promise; } /** From 65bd308339aa27413d14a9c35d968f73a3071180 Mon Sep 17 00:00:00 2001 From: cheethas Date: Tue, 4 Apr 2023 14:35:23 +0000 Subject: [PATCH 05/27] fix: use `compressInputs` rather than `hashToField` --- .../src/indexed_tree/indexed_tree.ts | 95 +++++++++++-------- 1 file changed, 53 insertions(+), 42 deletions(-) diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index 666f501c051b..3cf5efc45bbc 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -30,11 +30,11 @@ export interface LeafData { export interface LowNullifierWitnessData { /** * Preimage of the low nullifier that proves non membership - */ + */ preimage: LeafData; /** * Sibling path to prove membership of low nullifier - */ + */ siblingPath: SiblingPath; /** * The index of low nullifier @@ -42,7 +42,6 @@ export interface LowNullifierWitnessData { index: bigint; } - // eslint-disable-next-line @typescript-eslint/no-unused-vars const encodeTreeValue = (leafData: LeafData) => { const valueAsBuffer = toBufferBE(leafData.value, 32); @@ -53,9 +52,6 @@ const encodeTreeValue = (leafData: LeafData) => { // TODO: Check which version of hash we need to match the cpp implementation const hashEncodedTreeValue = (leaf: LeafData, hasher: Hasher) => { - return hasher.hashToField( - Buffer.concat([leaf.value, leaf.nextIndex, leaf.nextValue].map(val => toBufferBE(val, 32))), - ); return hasher.compressInputs([leaf.value, leaf.nextIndex, leaf.nextValue].map(val => toBufferBE(val, 32))); }; @@ -90,12 +86,19 @@ export class IndexedTree implements MerkleTree { * @param hasher - A hasher used to compute hash paths. * @param name - A name of the tree. * @param depth - A depth of the tree. + * @param prefilledSize - {optional} A number of leaves that are prefilled with values. * @returns A promise with the new Merkle tree. */ - public static async new(db: LevelUp, hasher: Hasher, name: string, depth: number): Promise { + public static async new( + db: LevelUp, + hasher: Hasher, + name: string, + depth: number, + prefilledSize = 0, + ): Promise { const underlying = await StandardMerkleTree.new(db, hasher, name, depth, hashEncodedTreeValue(initialLeaf, hasher)); const tree = new IndexedTree(underlying, hasher, db); - await tree.init(); + await tree.init(prefilledSize); return tree; } @@ -140,20 +143,19 @@ export class IndexedTree implements MerkleTree { } } - // TODO: this will have alot more information returned from it to craft the correct inputs // TODO: should this function similate on a copy of the tree rather than making actual changes? /** - * Each base rollup needs to provide non membership / inclusion proofs for each of the nullifiers - * generated in the kernel circuit that it is rolling up. - * - * As leaves are batch inserted at the end, batch updates are a special case. - * + * Each base rollup needs to provide non membership / inclusion proofs for each of the nullifiers + * generated in the kernel circuit that it is rolling up. + * + * As leaves are batch inserted at the end, batch updates are a special case. + * * WARNING: This function has side effects, it will insert values into the tree. * * TODO: include indepth insertion writeup in this comment * @param leaves Values to insert into the tree - * @returns + * @returns */ // TODO: assumptions // 1. There are 8 nullifiers provided and they are all unique @@ -162,11 +164,11 @@ export class IndexedTree implements MerkleTree { public async getAndPerformBaseRollupBatchInsertionProofs(leaves: Buffer[]): Promise { // Keep track of the touched during batch insertion const touchedNodes: Set = new Set(); - + const lowNullifierWitnesses: LowNullifierWitnessData[] = []; const startInsertionIndex: bigint = this.getNumLeaves(); let currInsertionIndex: bigint = startInsertionIndex; - + // Leaf data of hte leaves to be inserted const insertionSubtree: LeafData[] = []; @@ -174,27 +176,29 @@ export class IndexedTree implements MerkleTree { for (const leaf of leaves) { const newValue = toBigIntBE(leaf); const indexOfPrevious = this.findIndexOfPreviousValue(newValue); - + // NOTE: null values for nullfier leaves are being changed to 0n current impl is a hack // Default value let nullifierLeaf: LeafData = { value: 0n, nextIndex: 0n, - nextValue: 0n + nextValue: 0n, }; if (touchedNodes.has(indexOfPrevious.index)) { // If the node has already been touched, then we return an empty leaf and sibling path const emptySP = new SiblingPath(); - emptySP.data = Array(this.underlying.getDepth()).fill(Buffer.from("0000000000000000000000000000000000000000000000000000000000000000", "hex")); + emptySP.data = Array(this.underlying.getDepth()).fill( + Buffer.from('0000000000000000000000000000000000000000000000000000000000000000', 'hex'), + ); const witness: LowNullifierWitnessData = { - preimage: initialLeaf, - index: 0n, - siblingPath: emptySP + preimage: initialLeaf, + index: 0n, + siblingPath: emptySP, }; lowNullifierWitnesses.push(witness); } else { - // If the node has not been touched, we update its low nullifier pointer, but we do NOT insert it yet, inserting it now + // If the node has not been touched, we update its low nullifier pointer, but we do NOT insert it yet, inserting it now // will alter non membership paths of the not yet inserted members // Insertion is done at the end once updates have already occured. touchedNodes.add(indexOfPrevious.index); @@ -205,7 +209,7 @@ export class IndexedTree implements MerkleTree { if (lowNullifier === undefined) { throw new Error(`Previous leaf not found!`); } - + // Get sibling path for existence of the old leaf const siblingPath = await this.underlying.getSiblingPath(BigInt(indexOfPrevious.index)); @@ -213,12 +217,12 @@ export class IndexedTree implements MerkleTree { const witness = { preimage: lowNullifier, index: BigInt(indexOfPrevious.index), - siblingPath: siblingPath - } + siblingPath: siblingPath, + }; lowNullifierWitnesses.push(witness); // Update subtree insertion leaf from null data - nullifierLeaf = { + nullifierLeaf = { value: newValue, nextIndex: lowNullifier.nextIndex, nextValue: lowNullifier.nextValue, @@ -230,7 +234,10 @@ export class IndexedTree implements MerkleTree { // Update the old leaf in the tree this.cachedLeaves[Number(indexOfPrevious.index)] = lowNullifier; - await this.underlying.updateLeaf(hashEncodedTreeValue(lowNullifier, this.hasher), BigInt(indexOfPrevious.index)); + await this.underlying.updateLeaf( + hashEncodedTreeValue(lowNullifier, this.hasher), + BigInt(indexOfPrevious.index), + ); } // increment insertion index @@ -240,7 +247,7 @@ export class IndexedTree implements MerkleTree { // Create insertion subtree and forcefully insert in series // Here we calculate the pointers for the inserted values, if they have not already been updated - for (let i = 0 ; i < leaves.length; i++) { + for (let i = 0; i < leaves.length; i++) { const newValue = toBigIntBE(leaves[i]); // We have already fetched the new low nullifier for this leaf, so we can set its low nullifier @@ -248,29 +255,27 @@ export class IndexedTree implements MerkleTree { // If the lowNullifier is 0, then we check the previous leaves for the low nullifier leaf if (lowNullifier.value === 0n && lowNullifier.nextIndex === 0n && lowNullifier.nextValue === 0n) { for (let j = 0; j < i; j++) { - if ((insertionSubtree[j].nextValue >newValue&& - insertionSubtree[j].value < newValue) || - (insertionSubtree[j].nextValue == 0n && insertionSubtree[j].nextIndex == 0n)) { - - insertionSubtree[j].nextIndex = startInsertionIndex + BigInt(i); - insertionSubtree[j].nextValue = newValue; + if ( + (insertionSubtree[j].nextValue > newValue && insertionSubtree[j].value < newValue) || + (insertionSubtree[j].nextValue == 0n && insertionSubtree[j].nextIndex == 0n) + ) { + insertionSubtree[j].nextIndex = startInsertionIndex + BigInt(i); + insertionSubtree[j].nextValue = newValue; } - } + } } } // For each calculated new leaf, we insert it into the tree at the next position - for (let i = 0 ; i < leaves.length; i++) { + for (let i = 0; i < leaves.length; i++) { // TODO: can we skip inserting the empty values - this.cachedLeaves[Number(startInsertionIndex)+i] = insertionSubtree[i]; + this.cachedLeaves[Number(startInsertionIndex) + i] = insertionSubtree[i]; await this.underlying.appendLeaves([hashEncodedTreeValue(insertionSubtree[i], this.hasher)]); } return lowNullifierWitnesses; } - - /** * Commits the changes to the database. * @returns Empty promise. @@ -374,12 +379,18 @@ export class IndexedTree implements MerkleTree { /** * Saves the initial leaf to this object and saves it to a database. + * TODO: what will the size be */ - private async init() { + private async init(initialSize: number = 0) { // TODO: increase the initial size of the tree to the size of a full rollup insertion - change reflected in c++ to allow subtree insertion this.leaves.push(initialLeaf); await this.underlying.appendLeaves([hashEncodedTreeValue(initialLeaf, this.hasher)]); + + // TODO: optimise + for (let i = 1; i < initialSize; i++) { + await this.appendLeaf(Buffer.from([i])); + } await this.commit(); } From 426ce2ff96611b7ee6424143133440677a0b80a8 Mon Sep 17 00:00:00 2001 From: cheethas Date: Tue, 4 Apr 2023 15:47:33 +0000 Subject: [PATCH 06/27] fix: nullifier tree parity (with cpp) --- .../src/indexed_tree/indexed_tree.ts | 50 +++++++++++-------- .../circuit_powered_block_builder.ts | 16 +++--- .../world-state/src/world-state-db/index.ts | 5 +- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index 3cf5efc45bbc..4559ef61e159 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -143,8 +143,6 @@ export class IndexedTree implements MerkleTree { } } - // TODO: this will have alot more information returned from it to craft the correct inputs - // TODO: should this function similate on a copy of the tree rather than making actual changes? /** * Each base rollup needs to provide non membership / inclusion proofs for each of the nullifiers * generated in the kernel circuit that it is rolling up. @@ -153,18 +151,22 @@ export class IndexedTree implements MerkleTree { * * WARNING: This function has side effects, it will insert values into the tree. * + * Assumptions: + * 1. There are 8 nullifiers provided and they are all unique + * 2. If kc 0 has 1 nullifier, and kc 1 has 3 nullifiers the layout will assume to be the sparse + * nullifier layout: [kc0N, 0, 0, 0, kc1N, kc1N, kc1N, 0] + * * TODO: include indepth insertion writeup in this comment * @param leaves Values to insert into the tree * @returns */ - // TODO: assumptions - // 1. There are 8 nullifiers provided and they are all unique - // 2. If kc 0 has 1 nullifier, and kc 1 has 3 nullifiers the layout will assume to be the sparse - // nullifier layout: [kc0N, 0, 0, 0, kc1N, kc1N, kc1N, 0] - public async getAndPerformBaseRollupBatchInsertionProofs(leaves: Buffer[]): Promise { + public async getAndPerformBaseRollupBatchInsertionProofs( + leaves: Buffer[], + ): Promise { // Keep track of the touched during batch insertion const touchedNodes: Set = new Set(); + // Return data const lowNullifierWitnesses: LowNullifierWitnessData[] = []; const startInsertionIndex: bigint = this.getNumLeaves(); let currInsertionIndex: bigint = startInsertionIndex; @@ -179,8 +181,8 @@ export class IndexedTree implements MerkleTree { // NOTE: null values for nullfier leaves are being changed to 0n current impl is a hack // Default value - let nullifierLeaf: LeafData = { - value: 0n, + const nullifierLeaf: LeafData = { + value: newValue, nextIndex: 0n, nextValue: 0n, }; @@ -200,14 +202,15 @@ export class IndexedTree implements MerkleTree { } else { // If the node has not been touched, we update its low nullifier pointer, but we do NOT insert it yet, inserting it now // will alter non membership paths of the not yet inserted members - // Insertion is done at the end once updates have already occured. + // Insertion is done at the end once updates have already occurred. touchedNodes.add(indexOfPrevious.index); - const newValue = toBigIntBE(leaf); const lowNullifier = this.getLatestLeafDataCopy(indexOfPrevious.index); + // If no low nullifier can be found, abort - this means the nullifier is invalid + // in some way (it should not happen) if (lowNullifier === undefined) { - throw new Error(`Previous leaf not found!`); + return undefined; } // Get sibling path for existence of the old leaf @@ -222,11 +225,8 @@ export class IndexedTree implements MerkleTree { lowNullifierWitnesses.push(witness); // Update subtree insertion leaf from null data - nullifierLeaf = { - value: newValue, - nextIndex: lowNullifier.nextIndex, - nextValue: lowNullifier.nextValue, - }; + nullifierLeaf.nextIndex = lowNullifier.nextValue; + nullifierLeaf.nextValue = lowNullifier.nextValue; // Update the current low nullifier lowNullifier.nextIndex = currInsertionIndex; @@ -268,9 +268,17 @@ export class IndexedTree implements MerkleTree { // For each calculated new leaf, we insert it into the tree at the next position for (let i = 0; i < leaves.length; i++) { - // TODO: can we skip inserting the empty values - this.cachedLeaves[Number(startInsertionIndex) + i] = insertionSubtree[i]; - await this.underlying.appendLeaves([hashEncodedTreeValue(insertionSubtree[i], this.hasher)]); + // We can skip inserting empty leaves + if ( + !( + insertionSubtree[i].value === 0n && + insertionSubtree[i].nextIndex === 0n && + insertionSubtree[i].nextValue === 0n + ) + ) { + this.cachedLeaves[Number(startInsertionIndex) + i] = insertionSubtree[i]; + await this.underlying.appendLeaves([hashEncodedTreeValue(insertionSubtree[i], this.hasher)]); + } } return lowNullifierWitnesses; @@ -381,7 +389,7 @@ export class IndexedTree implements MerkleTree { * Saves the initial leaf to this object and saves it to a database. * TODO: what will the size be */ - private async init(initialSize: number = 0) { + private async init(initialSize = 0) { // TODO: increase the initial size of the tree to the size of a full rollup insertion - change reflected in c++ to allow subtree insertion this.leaves.push(initialLeaf); diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 4a055ac44e0b..550cb6964621 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -410,21 +410,23 @@ export class CircuitPoweredBlockBuilder { // Update the nullifier tree, capturing the low nullifier info for each individual operation const newNullifiers = [...tx1.data.end.newNullifiers, ...tx2.data.end.newNullifiers]; - const lowNullifierInfos = []; // console.log( // `Nullifier root before insertion: `, // await this.getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE).then(t => '0x' + t.root.toBuffer().toString('hex')), // ); // console.log(`Inserting new data`, newNullifiers.join(', ')); - - // TODO: handle exception + const nullifierWitnesses = await this.db.getAndPerformBaseRollupBatchInsertionProofs( MerkleTreeId.NULLIFIER_TREE, - newNullifiers.map(fr => fr.toBuffer()) + newNullifiers.map(fr => fr.toBuffer()), ); + if (nullifierWitnesses === undefined) { + throw new Error(`Could not craft nullifier batch insertion proofs`); + } // Extract witness objects from returned data - const lowNullifierMembershipWitnesses = nullifierWitnesses.map(w => MembershipWitness.fromSiblingPath(Number(w.index), w.siblingPath), - ) + const lowNullifierMembershipWitnesses = nullifierWitnesses.map(w => + MembershipWitness.fromSiblingPath(Number(w.index), w.siblingPath), + ); // console.log( // `Nullifier root after insertion: `, @@ -454,7 +456,7 @@ export class CircuitPoweredBlockBuilder { newContractsSubtreeSiblingPath, newNullifiersSubtreeSiblingPath, // TODO: typing for `LowNullifierWitnessData` - lowNullifierLeafPreimages: nullifierWitnesses.map((w: any) => w.preimage), + lowNullifierLeafPreimages: nullifierWitnesses.map((w: any) => w.preimage), lowNullifierMembershipWitness: lowNullifierMembershipWitnesses, kernelData: [this.getKernelDataFor(tx1), this.getKernelDataFor(tx2)], historicContractsTreeRootMembershipWitnesses: [ diff --git a/yarn-project/world-state/src/world-state-db/index.ts b/yarn-project/world-state/src/world-state-db/index.ts index 61277eb5ffc7..3baae284bd0f 100644 --- a/yarn-project/world-state/src/world-state-db/index.ts +++ b/yarn-project/world-state/src/world-state-db/index.ts @@ -101,7 +101,10 @@ export interface MerkleTreeOperations { * @param index - The index of the leaf */ getLeafValue(treeId: MerkleTreeId, index: bigint): Promise; - getAndPerformBaseRollupBatchInsertionProofs(treeId: MerkleTreeId, leaves: Buffer[]): Promise; + getAndPerformBaseRollupBatchInsertionProofs( + treeId: MerkleTreeId, + leaves: Buffer[], + ): Promise; } /** From 8388dc1376c62f3803778863cbddb66028716155 Mon Sep 17 00:00:00 2001 From: cheethas Date: Tue, 4 Apr 2023 21:55:59 +0000 Subject: [PATCH 07/27] refactor: move base rollup logic outside of the IndexedTree impl --- .../src/indexed_tree/indexed_tree.ts | 184 +++--------------- yarn-project/merkle-tree/src/merkle_tree.ts | 7 + .../circuit_powered_block_builder.ts | 156 ++++++++++++++- .../server_world_state_synchroniser.ts | 20 +- .../world-state/src/world-state-db/index.ts | 17 +- .../src/world-state-db/merkle_trees.ts | 11 ++ 6 files changed, 229 insertions(+), 166 deletions(-) diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index 4559ef61e159..5c5c5f792cb2 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -27,21 +27,6 @@ export interface LeafData { nextValue: bigint; } -export interface LowNullifierWitnessData { - /** - * Preimage of the low nullifier that proves non membership - */ - preimage: LeafData; - /** - * Sibling path to prove membership of low nullifier - */ - siblingPath: SiblingPath; - /** - * The index of low nullifier - */ - index: bigint; -} - // eslint-disable-next-line @typescript-eslint/no-unused-vars const encodeTreeValue = (leafData: LeafData) => { const valueAsBuffer = toBufferBE(leafData.value, 32); @@ -116,6 +101,15 @@ export class IndexedTree implements MerkleTree { return tree; } + /** + * TODO: Just export this or keep it static? + * Returns an empty leaf of the tree. + * @returns An empty leaf. + */ + static initialLeaf(): LeafData { + return initialLeaf; + } + /** * Returns the root of the tree. * @returns The root of the tree. @@ -124,6 +118,14 @@ export class IndexedTree implements MerkleTree { return this.underlying.getRoot(includeUncommitted); } + /** + * Returns the depth of the tree. + * @returns The depth of the tree. + */ + public getDepth(): number { + return this.underlying.getDepth(); + } + /** * Returns the number of leaves in the tree. * @returns The number of leaves in the tree. @@ -143,147 +145,6 @@ export class IndexedTree implements MerkleTree { } } - /** - * Each base rollup needs to provide non membership / inclusion proofs for each of the nullifiers - * generated in the kernel circuit that it is rolling up. - * - * As leaves are batch inserted at the end, batch updates are a special case. - * - * WARNING: This function has side effects, it will insert values into the tree. - * - * Assumptions: - * 1. There are 8 nullifiers provided and they are all unique - * 2. If kc 0 has 1 nullifier, and kc 1 has 3 nullifiers the layout will assume to be the sparse - * nullifier layout: [kc0N, 0, 0, 0, kc1N, kc1N, kc1N, 0] - * - * TODO: include indepth insertion writeup in this comment - * @param leaves Values to insert into the tree - * @returns - */ - public async getAndPerformBaseRollupBatchInsertionProofs( - leaves: Buffer[], - ): Promise { - // Keep track of the touched during batch insertion - const touchedNodes: Set = new Set(); - - // Return data - const lowNullifierWitnesses: LowNullifierWitnessData[] = []; - const startInsertionIndex: bigint = this.getNumLeaves(); - let currInsertionIndex: bigint = startInsertionIndex; - - // Leaf data of hte leaves to be inserted - const insertionSubtree: LeafData[] = []; - - // Low nullifier membership proof sibling paths - for (const leaf of leaves) { - const newValue = toBigIntBE(leaf); - const indexOfPrevious = this.findIndexOfPreviousValue(newValue); - - // NOTE: null values for nullfier leaves are being changed to 0n current impl is a hack - // Default value - const nullifierLeaf: LeafData = { - value: newValue, - nextIndex: 0n, - nextValue: 0n, - }; - - if (touchedNodes.has(indexOfPrevious.index)) { - // If the node has already been touched, then we return an empty leaf and sibling path - const emptySP = new SiblingPath(); - emptySP.data = Array(this.underlying.getDepth()).fill( - Buffer.from('0000000000000000000000000000000000000000000000000000000000000000', 'hex'), - ); - const witness: LowNullifierWitnessData = { - preimage: initialLeaf, - index: 0n, - siblingPath: emptySP, - }; - lowNullifierWitnesses.push(witness); - } else { - // If the node has not been touched, we update its low nullifier pointer, but we do NOT insert it yet, inserting it now - // will alter non membership paths of the not yet inserted members - // Insertion is done at the end once updates have already occurred. - touchedNodes.add(indexOfPrevious.index); - - const lowNullifier = this.getLatestLeafDataCopy(indexOfPrevious.index); - - // If no low nullifier can be found, abort - this means the nullifier is invalid - // in some way (it should not happen) - if (lowNullifier === undefined) { - return undefined; - } - - // Get sibling path for existence of the old leaf - const siblingPath = await this.underlying.getSiblingPath(BigInt(indexOfPrevious.index)); - - // Update the running paths - const witness = { - preimage: lowNullifier, - index: BigInt(indexOfPrevious.index), - siblingPath: siblingPath, - }; - lowNullifierWitnesses.push(witness); - - // Update subtree insertion leaf from null data - nullifierLeaf.nextIndex = lowNullifier.nextValue; - nullifierLeaf.nextValue = lowNullifier.nextValue; - - // Update the current low nullifier - lowNullifier.nextIndex = currInsertionIndex; - lowNullifier.nextValue = BigInt(newValue); - - // Update the old leaf in the tree - this.cachedLeaves[Number(indexOfPrevious.index)] = lowNullifier; - await this.underlying.updateLeaf( - hashEncodedTreeValue(lowNullifier, this.hasher), - BigInt(indexOfPrevious.index), - ); - } - - // increment insertion index - currInsertionIndex++; - insertionSubtree.push(nullifierLeaf); - } - - // Create insertion subtree and forcefully insert in series - // Here we calculate the pointers for the inserted values, if they have not already been updated - for (let i = 0; i < leaves.length; i++) { - const newValue = toBigIntBE(leaves[i]); - - // We have already fetched the new low nullifier for this leaf, so we can set its low nullifier - const lowNullifier = lowNullifierWitnesses[i].preimage; - // If the lowNullifier is 0, then we check the previous leaves for the low nullifier leaf - if (lowNullifier.value === 0n && lowNullifier.nextIndex === 0n && lowNullifier.nextValue === 0n) { - for (let j = 0; j < i; j++) { - if ( - (insertionSubtree[j].nextValue > newValue && insertionSubtree[j].value < newValue) || - (insertionSubtree[j].nextValue == 0n && insertionSubtree[j].nextIndex == 0n) - ) { - insertionSubtree[j].nextIndex = startInsertionIndex + BigInt(i); - insertionSubtree[j].nextValue = newValue; - } - } - } - } - - // For each calculated new leaf, we insert it into the tree at the next position - for (let i = 0; i < leaves.length; i++) { - // We can skip inserting empty leaves - if ( - !( - insertionSubtree[i].value === 0n && - insertionSubtree[i].nextIndex === 0n && - insertionSubtree[i].nextValue === 0n - ) - ) { - this.cachedLeaves[Number(startInsertionIndex) + i] = insertionSubtree[i]; - await this.underlying.appendLeaves([hashEncodedTreeValue(insertionSubtree[i], this.hasher)]); - } - } - - return lowNullifierWitnesses; - } - /** * Commits the changes to the database. * @returns Empty promise. @@ -312,6 +173,17 @@ export class IndexedTree implements MerkleTree { return await this.underlying.getSiblingPath(index, includeUncommitted); } + /** + * Exposes the underlying tree's update leaf method + * @param leaf - The hash to set at the leaf + * @param index - The index of the element + */ + public async updateLeaf(leaf: LeafData, index: bigint): Promise { + this.cachedLeaves[Number(index)] = leaf; + const encodedLeaf = encodeTreeValue(leaf); + await this.underlying.updateLeaf(encodedLeaf, index); + } + /** * Appends the given leaf to the tree. * @param leaf - The leaf to append. diff --git a/yarn-project/merkle-tree/src/merkle_tree.ts b/yarn-project/merkle-tree/src/merkle_tree.ts index acdc8d96aae5..005e16d52ec3 100644 --- a/yarn-project/merkle-tree/src/merkle_tree.ts +++ b/yarn-project/merkle-tree/src/merkle_tree.ts @@ -1,3 +1,4 @@ +import { LeafData } from './index.js'; import { SiblingPath } from './sibling_path/sibling_path.js'; /** @@ -35,6 +36,12 @@ export interface MerkleTree extends SiblingPathSource { * Commit pending updates to the tree */ commit(): Promise; + /** + * Updates a leaf at a given index in the tree + * @param leaf The leaf value to be updated + * @param index The leaf to be updated + */ + updateLeaf(leaf: Buffer | LeafData, index: bigint): Promise; /** * Rollback pending update to the tree */ diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 550cb6964621..2dcfab6d630f 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -19,6 +19,7 @@ import { VK_TREE_HEIGHT, } from '@aztec/circuits.js'; import { Fr, createDebugLogger, toBigIntBE } from '@aztec/foundation'; +import { IndexedTree, LeafData, SiblingPath } from '@aztec/merkle-tree'; import { Tx } from '@aztec/tx'; import { MerkleTreeId, MerkleTreeOperations } from '@aztec/world-state'; import flatMap from 'lodash.flatmap'; @@ -41,6 +42,21 @@ const FUTURE_NUM = 0; const DELETE_FR = new Fr(0n); const DELETE_NUM = 0; +// TODO: doc +export interface LowNullifierWitnessData { + /** + * Preimage of the low nullifier that proves non membership + */ + preimage: LeafData; + /** + * Sibling path to prove membership of low nullifier + */ + siblingPath: SiblingPath; + /** + * The index of low nullifier + */ + index: bigint; +} export class CircuitPoweredBlockBuilder { constructor( protected db: MerkleTreeOperations, @@ -382,6 +398,141 @@ export class CircuitPoweredBlockBuilder { return fullSiblingPath.data.slice(subtreeHeight).map(b => Fr.fromBuffer(b)); } + /** + * Each base rollup needs to provide non membership / inclusion proofs for each of the nullifiers + * generated in the kernel circuit that it is rolling up. + * + * As leaves are batch inserted at the end, batch updates are a special case. + * + * WARNING: This function has side effects, it will insert values into the tree. + * + * Assumptions: + * 1. There are 8 nullifiers provided and they are all unique + * 2. If kc 0 has 1 nullifier, and kc 1 has 3 nullifiers the layout will assume to be the sparse + * nullifier layout: [kc0N, 0, 0, 0, kc1N, kc1N, kc1N, 0] + * + * TODO: include indepth insertion writeup in this comment + * @param leaves Values to insert into the tree + * @returns + */ + public async performBaseRollupBatchInsertionProofs(leaves: Buffer[]): Promise { + // Keep track of the touched during batch insertion + const touchedNodes: Set = new Set(); + + // Return data + const lowNullifierWitnesses: LowNullifierWitnessData[] = []; + const dbInfo = await this.db.getTreeInfo(MerkleTreeId.NULLIFIER_TREE); + const startInsertionIndex: bigint = dbInfo.size; + let currInsertionIndex: bigint = startInsertionIndex; + + // Leaf data of hte leaves to be inserted + const insertionSubtree: LeafData[] = []; + + // Low nullifier membership proof sibling paths + for (const leaf of leaves) { + const newValue = toBigIntBE(leaf); + const indexOfPrevious = await this.db.getPreviousValueIndex(MerkleTreeId.NULLIFIER_TREE, newValue); + + // NOTE: null values for nullfier leaves are being changed to 0n current impl is a hack + // Default value + const nullifierLeaf: LeafData = { + value: newValue, + nextIndex: 0n, + nextValue: 0n, + }; + + if (touchedNodes.has(indexOfPrevious.index)) { + // If the node has already been touched, then we return an empty leaf and sibling path + const emptySP = new SiblingPath(); + emptySP.data = Array(dbInfo.depth).fill( + Buffer.from('0000000000000000000000000000000000000000000000000000000000000000', 'hex'), + ); + const witness: LowNullifierWitnessData = { + preimage: IndexedTree.initialLeaf(), + index: 0n, + siblingPath: emptySP, + }; + lowNullifierWitnesses.push(witness); + } else { + // If the node has not been touched, we update its low nullifier pointer, but we do NOT insert it yet, inserting it now + // will alter non membership paths of the not yet inserted members + // Insertion is done at the end once updates have already occurred. + touchedNodes.add(indexOfPrevious.index); + + const lowNullifier = this.db.getLeafData(MerkleTreeId.NULLIFIER_TREE, indexOfPrevious.index); + + // If no low nullifier can be found, abort - this means the nullifier is invalid + // in some way (it should not happen) + if (lowNullifier === undefined) { + return undefined; + } + + // Get sibling path for existence of the old leaf + const siblingPath = await this.db.getSiblingPath(MerkleTreeId.NULLIFIER_TREE, BigInt(indexOfPrevious.index)); + + // Update the running paths + const witness = { + preimage: lowNullifier, + index: BigInt(indexOfPrevious.index), + siblingPath: siblingPath, + }; + lowNullifierWitnesses.push(witness); + + // Update subtree insertion leaf from null data + nullifierLeaf.nextIndex = lowNullifier.nextValue; + nullifierLeaf.nextValue = lowNullifier.nextValue; + + // Update the current low nullifier + lowNullifier.nextIndex = currInsertionIndex; + lowNullifier.nextValue = BigInt(newValue); + + // Update the old leaf in the tree + await this.db.updateLeaf(MerkleTreeId.NULLIFIER_TREE, lowNullifier, BigInt(indexOfPrevious.index)); + } + + // increment insertion index + currInsertionIndex++; + insertionSubtree.push(nullifierLeaf); + } + + // Create insertion subtree and forcefully insert in series + // Here we calculate the pointers for the inserted values, if they have not already been updated + for (let i = 0; i < leaves.length; i++) { + const newValue = toBigIntBE(leaves[i]); + + // We have already fetched the new low nullifier for this leaf, so we can set its low nullifier + const lowNullifier = lowNullifierWitnesses[i].preimage; + // If the lowNullifier is 0, then we check the previous leaves for the low nullifier leaf + if (lowNullifier.value === 0n && lowNullifier.nextIndex === 0n && lowNullifier.nextValue === 0n) { + for (let j = 0; j < i; j++) { + if ( + (insertionSubtree[j].nextValue > newValue && insertionSubtree[j].value < newValue) || + (insertionSubtree[j].nextValue == 0n && insertionSubtree[j].nextIndex == 0n) + ) { + insertionSubtree[j].nextIndex = startInsertionIndex + BigInt(i); + insertionSubtree[j].nextValue = newValue; + } + } + } + } + + // For each calculated new leaf, we insert it into the tree at the next position + for (let i = 0; i < insertionSubtree.length; i++) { + // We can skip inserting empty leaves + if ( + !( + insertionSubtree[i].value === 0n && + insertionSubtree[i].nextIndex === 0n && + insertionSubtree[i].nextValue === 0n + ) + ) { + await this.db.updateLeaf(MerkleTreeId.NULLIFIER_TREE, insertionSubtree[i], startInsertionIndex + BigInt(i)); + } + } + + return lowNullifierWitnesses; + } + // Builds the base rollup inputs, updating the contract, nullifier, and data trees in the process protected async buildBaseRollupInput(tx1: Tx, tx2: Tx) { // Get trees info before any changes hit @@ -416,10 +567,7 @@ export class CircuitPoweredBlockBuilder { // ); // console.log(`Inserting new data`, newNullifiers.join(', ')); - const nullifierWitnesses = await this.db.getAndPerformBaseRollupBatchInsertionProofs( - MerkleTreeId.NULLIFIER_TREE, - newNullifiers.map(fr => fr.toBuffer()), - ); + const nullifierWitnesses = await this.performBaseRollupBatchInsertionProofs(newNullifiers.map(fr => fr.toBuffer())); if (nullifierWitnesses === undefined) { throw new Error(`Could not craft nullifier batch insertion proofs`); } diff --git a/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts b/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts index 0cf82d630466..8a941a239ca5 100644 --- a/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts +++ b/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts @@ -5,7 +5,6 @@ import { MerkleTreeOperationsFacade } from '../merkle-tree/merkle_tree_operation import { WorldStateRunningState, WorldStateStatus, WorldStateSynchroniser } from './world_state_synchroniser.js'; import { MerkleTreeDb, MerkleTreeId, TreeInfo } from '../index.js'; import { LeafData, SiblingPath } from '@aztec/merkle-tree'; -import { LowNullifierWitnessData } from '@aztec/merkle-tree'; /** * Synchronises the world state with the L2 blocks from a L2BlockSource. @@ -38,6 +37,25 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser { return new MerkleTreeOperationsFacade(this.merkleTreeDb, false); } + public getPreviousValueIndex( + treeId: MerkleTreeId.NULLIFIER_TREE, + value: bigint, + ): Promise<{ index: number; alreadyPresent: boolean }> { + return this.merkleTreeDb.getPreviousValueIndex(treeId, value); + } + + public getLeafData(treeId: MerkleTreeId.NULLIFIER_TREE, index: number): LeafData | undefined { + return this.merkleTreeDb.getLeafData(treeId, index); + } + + public updateLeaf(treeId: MerkleTreeId.NULLIFIER_TREE, leaf: LeafData, index: bigint): Promise { + return this.merkleTreeDb.updateLeaf(treeId, leaf, index); + } + + public findLeafIndex(treeId: MerkleTreeId, value: Buffer): Promise { + return this.merkleTreeDb.findLeafIndex(treeId, value); + } + /** * Starts the synchroniser. * @returns A promise that resolves once the initial sync is completed. diff --git a/yarn-project/world-state/src/world-state-db/index.ts b/yarn-project/world-state/src/world-state-db/index.ts index 3baae284bd0f..64e83de2ef14 100644 --- a/yarn-project/world-state/src/world-state-db/index.ts +++ b/yarn-project/world-state/src/world-state-db/index.ts @@ -1,4 +1,3 @@ -import { LowNullifierWitnessData } from '@aztec/merkle-tree'; import { LeafData, SiblingPath } from '@aztec/merkle-tree'; export * from './merkle_trees.js'; @@ -33,6 +32,11 @@ export interface TreeInfo { * The number of leaves in the tree. */ size: bigint; + + /** + * The depth of the tree. + */ + depth: number; } /** @@ -89,6 +93,13 @@ export interface MerkleTreeOperations { * @param index - The index of the leaf required */ getLeafData(treeId: IndexedMerkleTreeId, index: number): Promise; + /** + * Update the leaf data at the given index + * @param treeId - The tree for which leaf data should be edited + * @param leaf - The updated leaf value + * @param index - The index of the leaf to be updated + */ + updateLeaf(treeId: IndexedMerkleTreeId, leaf: LeafData, index: bigint): Promise; /** * Returns the index containing a leaf value * @param treeId - The tree for which the index should be returned @@ -101,10 +112,6 @@ export interface MerkleTreeOperations { * @param index - The index of the leaf */ getLeafValue(treeId: MerkleTreeId, index: bigint): Promise; - getAndPerformBaseRollupBatchInsertionProofs( - treeId: MerkleTreeId, - leaves: Buffer[], - ): Promise; } /** diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 32efefa31fe1..8e5cb868072f 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -191,6 +191,16 @@ export class MerkleTrees implements MerkleTreeDb { }); } + /** + * Updates a leaf in a tree at a given index. + * @param treeId - The ID of the tree + * @param leaf - The new leaf value + * @param index - The index to insert into + */ + public async updateLeaf(treeId: IndexedMerkleTreeId, leaf: Buffer | LeafData, index: bigint): Promise { + return await this.synchronise(() => this.trees[treeId].updateLeaf(leaf, index)); + } + /** * Waits for all jobs to finish before executing the given function. * @param fn - The function to execute. @@ -210,6 +220,7 @@ export class MerkleTrees implements MerkleTreeDb { treeId, root: this.trees[treeId].getRoot(includeUncommitted), size: this.trees[treeId].getNumLeaves(includeUncommitted), + depth: this.trees[treeId].getDepth(), } as TreeInfo; return Promise.resolve(treeInfo); } From 847c972130dcfef5363d6373141f87018c4989ba Mon Sep 17 00:00:00 2001 From: cheethas Date: Tue, 4 Apr 2023 21:56:28 +0000 Subject: [PATCH 08/27] feat(world_state): give the nullifier tree an initial size --- yarn-project/world-state/src/world-state-db/index.ts | 7 +++++++ .../world-state/src/world-state-db/merkle_trees.ts | 10 +++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/yarn-project/world-state/src/world-state-db/index.ts b/yarn-project/world-state/src/world-state-db/index.ts index 64e83de2ef14..fc5aa0a83b45 100644 --- a/yarn-project/world-state/src/world-state-db/index.ts +++ b/yarn-project/world-state/src/world-state-db/index.ts @@ -16,6 +16,13 @@ export enum MerkleTreeId { export type IndexedMerkleTreeId = MerkleTreeId.NULLIFIER_TREE; +/** + * The nullifier tree must be pre filled with the number of leaves that are added by one rollup. + * The tree must be initially padded as the pre-populated 0 index prevents efficient subtree insertion. + * Padding with some values solves this issue. + */ +export const INITIAL_NULLIFIER_TREE_SIZE = 8; + /** * Defines tree information. */ diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 8e5cb868072f..8b3ab458c44d 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -9,8 +9,15 @@ import { BarretenbergWasm } from '@aztec/barretenberg.js/wasm'; import { SerialQueue } from '@aztec/foundation'; import { IndexedTree, LeafData, MerkleTree, Pedersen, SiblingPath, StandardMerkleTree } from '@aztec/merkle-tree'; import { default as levelup } from 'levelup'; -import { IndexedMerkleTreeId, MerkleTreeDb, MerkleTreeId, MerkleTreeOperations, TreeInfo } from './index.js'; import { MerkleTreeOperationsFacade } from '../merkle-tree/merkle_tree_operations_facade.js'; +import { + INITIAL_NULLIFIER_TREE_SIZE, + IndexedMerkleTreeId, + MerkleTreeDb, + MerkleTreeId, + MerkleTreeOperations, + TreeInfo, +} from './index.js'; /** * A convenience class for managing multiple merkle trees. @@ -44,6 +51,7 @@ export class MerkleTrees implements MerkleTreeDb { hasher, `${MerkleTreeId[MerkleTreeId.NULLIFIER_TREE]}`, NULLIFIER_TREE_HEIGHT, + INITIAL_NULLIFIER_TREE_SIZE, ); const dataTree = await StandardMerkleTree.new( this.db, From 80e55f23fa12f9202e39448b3b92ab8e5f433e88 Mon Sep 17 00:00:00 2001 From: cheethas Date: Tue, 4 Apr 2023 22:30:31 +0000 Subject: [PATCH 09/27] merge fixes --- yarn-project/merkle-tree/src/merkle_tree.ts | 4 ++++ .../circuit_powered_block_builder.ts | 2 +- .../merkle_tree_operations_facade.ts | 3 +++ .../server_world_state_synchroniser.ts | 22 +------------------ 4 files changed, 9 insertions(+), 22 deletions(-) diff --git a/yarn-project/merkle-tree/src/merkle_tree.ts b/yarn-project/merkle-tree/src/merkle_tree.ts index 005e16d52ec3..6c7158ac010d 100644 --- a/yarn-project/merkle-tree/src/merkle_tree.ts +++ b/yarn-project/merkle-tree/src/merkle_tree.ts @@ -42,6 +42,10 @@ export interface MerkleTree extends SiblingPathSource { * @param index The leaf to be updated */ updateLeaf(leaf: Buffer | LeafData, index: bigint): Promise; + /** + * Returns the depth of the tree + */ + getDepth(): number; /** * Rollback pending update to the tree */ diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 2dcfab6d630f..59eac6b4c019 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -459,7 +459,7 @@ export class CircuitPoweredBlockBuilder { // Insertion is done at the end once updates have already occurred. touchedNodes.add(indexOfPrevious.index); - const lowNullifier = this.db.getLeafData(MerkleTreeId.NULLIFIER_TREE, indexOfPrevious.index); + const lowNullifier = await this.db.getLeafData(MerkleTreeId.NULLIFIER_TREE, indexOfPrevious.index); // If no low nullifier can be found, abort - this means the nullifier is invalid // in some way (it should not happen) diff --git a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts index 7b25ebee0662..09e2289e2380 100644 --- a/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts +++ b/yarn-project/world-state/src/merkle-tree/merkle_tree_operations_facade.ts @@ -21,6 +21,9 @@ export class MerkleTreeOperationsFacade implements MerkleTreeOperations { ): Promise<{ index: number; alreadyPresent: boolean }> { return this.trees.getPreviousValueIndex(treeId, value, this.includeUncommitted); } + updateLeaf(treeId: MerkleTreeId.NULLIFIER_TREE, leaf: LeafData, index: bigint): Promise { + return this.trees.updateLeaf(treeId, leaf, index, this.includeUncommitted); + } getLeafData(treeId: MerkleTreeId.NULLIFIER_TREE, index: number): Promise { return this.trees.getLeafData(treeId, index, this.includeUncommitted); } diff --git a/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts b/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts index 8a941a239ca5..beb493ad569d 100644 --- a/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts +++ b/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts @@ -3,8 +3,7 @@ import { L2Block, L2BlockDownloader, L2BlockSource } from '@aztec/l2-block'; import { MerkleTreeDb, MerkleTreeId, MerkleTreeOperations } from '../index.js'; import { MerkleTreeOperationsFacade } from '../merkle-tree/merkle_tree_operations_facade.js'; import { WorldStateRunningState, WorldStateStatus, WorldStateSynchroniser } from './world_state_synchroniser.js'; -import { MerkleTreeDb, MerkleTreeId, TreeInfo } from '../index.js'; -import { LeafData, SiblingPath } from '@aztec/merkle-tree'; +import { LeafData } from '@aztec/merkle-tree'; /** * Synchronises the world state with the L2 blocks from a L2BlockSource. @@ -37,25 +36,6 @@ export class ServerWorldStateSynchroniser implements WorldStateSynchroniser { return new MerkleTreeOperationsFacade(this.merkleTreeDb, false); } - public getPreviousValueIndex( - treeId: MerkleTreeId.NULLIFIER_TREE, - value: bigint, - ): Promise<{ index: number; alreadyPresent: boolean }> { - return this.merkleTreeDb.getPreviousValueIndex(treeId, value); - } - - public getLeafData(treeId: MerkleTreeId.NULLIFIER_TREE, index: number): LeafData | undefined { - return this.merkleTreeDb.getLeafData(treeId, index); - } - - public updateLeaf(treeId: MerkleTreeId.NULLIFIER_TREE, leaf: LeafData, index: bigint): Promise { - return this.merkleTreeDb.updateLeaf(treeId, leaf, index); - } - - public findLeafIndex(treeId: MerkleTreeId, value: Buffer): Promise { - return this.merkleTreeDb.findLeafIndex(treeId, value); - } - /** * Starts the synchroniser. * @returns A promise that resolves once the initial sync is completed. From 9c704bfdad149df84a0d73227bc72b536f390739 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 09:53:59 +0000 Subject: [PATCH 10/27] temp --- .../src/block_builder/circuit_powered_block_builder.test.ts | 2 +- .../src/block_builder/circuit_powered_block_builder.ts | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts index fcf0c985098a..923ce417b054 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts @@ -161,7 +161,7 @@ describe('sequencer/circuit_block_builder', () => { expect(proof).toEqual(emptyProof); }); - it('builds an L2 block with empty txs using wasm circuits', async () => { + it.only('builds an L2 block with empty txs using wasm circuits', async () => { const simulator = new WasmCircuitSimulator(wasm); const prover = new EmptyProver(); builder = new TestSubject(builderDb, vks, simulator, prover, wasm); diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 59eac6b4c019..6a64b9ffdbd6 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -18,6 +18,7 @@ import { UInt8Vector, VK_TREE_HEIGHT, } from '@aztec/circuits.js'; +import { toFriendlyJSON } from '@aztec/circuits.js/utils'; import { Fr, createDebugLogger, toBigIntBE } from '@aztec/foundation'; import { IndexedTree, LeafData, SiblingPath } from '@aztec/merkle-tree'; import { Tx } from '@aztec/tx'; @@ -174,6 +175,7 @@ export class CircuitPoweredBlockBuilder { protected async baseRollupCircuit(tx1: Tx, tx2: Tx) { const rollupInput = await this.buildBaseRollupInput(tx1, tx2); + console.log(`Da rollup input`, toFriendlyJSON(rollupInput)); const rollupOutput = await this.simulator.baseRollupCircuit(rollupInput); await this.validateTrees(rollupOutput); return [rollupInput, rollupOutput] as const; From a0e67dabff42dc279aebfa5d1bd669e82d701c75 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 09:56:02 +0000 Subject: [PATCH 11/27] temp --- yarn-project/barretenberg.js/src/crypto/pedersen/pedersen.ts | 2 +- yarn-project/circuits.js/src/utils/serialize.ts | 2 ++ yarn-project/foundation/src/bigint-buffer/index.ts | 4 +++- yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts | 3 +-- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/yarn-project/barretenberg.js/src/crypto/pedersen/pedersen.ts b/yarn-project/barretenberg.js/src/crypto/pedersen/pedersen.ts index 67c29e8a84c1..930aeb82aba6 100644 --- a/yarn-project/barretenberg.js/src/crypto/pedersen/pedersen.ts +++ b/yarn-project/barretenberg.js/src/crypto/pedersen/pedersen.ts @@ -13,7 +13,7 @@ export function pedersenCompress(wasm: BarretenbergWasm, lhs: Uint8Array, rhs: U // If not done already, precompute constants. wasm.call('pedersen__init'); if (lhs.length !== 32 || rhs.length !== 32) { - throw new Error('lhs and rhs must be equal to 32 bytes'); + throw new Error(`Pedersen lhs and rhs inputs must be 32 bytes (got ${lhs.length} and ${rhs.length} respectively)`); } wasm.writeMemory(0, lhs); wasm.writeMemory(32, rhs); diff --git a/yarn-project/circuits.js/src/utils/serialize.ts b/yarn-project/circuits.js/src/utils/serialize.ts index 95fb9773e7e8..3190b61f0f70 100644 --- a/yarn-project/circuits.js/src/utils/serialize.ts +++ b/yarn-project/circuits.js/src/utils/serialize.ts @@ -175,6 +175,8 @@ export function toFriendlyJSON(obj: object): string { return '0x' + Buffer.from(value.data).toString('hex'); } else if (typeof value === 'bigint') { return value.toString(); + } else if ((value as { toFriendlyJSON: () => string }).toFriendlyJSON) { + return value.toFriendlyJSON(); } else { return value; } diff --git a/yarn-project/foundation/src/bigint-buffer/index.ts b/yarn-project/foundation/src/bigint-buffer/index.ts index b04faaaf8644..d4de49c9c928 100644 --- a/yarn-project/foundation/src/bigint-buffer/index.ts +++ b/yarn-project/foundation/src/bigint-buffer/index.ts @@ -47,5 +47,7 @@ export function toBufferLE(num: bigint, width: number): Buffer { */ export function toBufferBE(num: bigint, width: number): Buffer { const hex = num.toString(16); - return Buffer.from(hex.padStart(width * 2, '0').slice(0, width * 2), 'hex'); + const buffer = Buffer.from(hex.padStart(width * 2, '0').slice(0, width * 2), 'hex'); + if (buffer.length > width) throw new Error(`Number ${num.toString(16)} does not fit in ${width}`); + return buffer; } diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index 5c5c5f792cb2..31fddba11b4e 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -35,7 +35,6 @@ const encodeTreeValue = (leafData: LeafData) => { return Buffer.concat([valueAsBuffer, indexAsBuffer, nextValueAsBuffer]); }; -// TODO: Check which version of hash we need to match the cpp implementation const hashEncodedTreeValue = (leaf: LeafData, hasher: Hasher) => { return hasher.compressInputs([leaf.value, leaf.nextIndex, leaf.nextValue].map(val => toBufferBE(val, 32))); }; @@ -180,7 +179,7 @@ export class IndexedTree implements MerkleTree { */ public async updateLeaf(leaf: LeafData, index: bigint): Promise { this.cachedLeaves[Number(index)] = leaf; - const encodedLeaf = encodeTreeValue(leaf); + const encodedLeaf = hashEncodedTreeValue(leaf, this.hasher); await this.underlying.updateLeaf(encodedLeaf, index); } From c5e5e058969076b5e9e1ffeb5e157a040c7148c3 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 11:22:37 +0000 Subject: [PATCH 12/27] fix: types --- .../circuits.js/src/structs/base_rollup.ts | 4 ++ .../circuit_powered_block_builder.ts | 63 +++++++++++-------- 2 files changed, 40 insertions(+), 27 deletions(-) diff --git a/yarn-project/circuits.js/src/structs/base_rollup.ts b/yarn-project/circuits.js/src/structs/base_rollup.ts index 91c67d6f9e43..fbd2a508f58b 100644 --- a/yarn-project/circuits.js/src/structs/base_rollup.ts +++ b/yarn-project/circuits.js/src/structs/base_rollup.ts @@ -20,6 +20,10 @@ export class NullifierLeafPreimage { toBuffer() { return serializeToBuffer(this.leafValue, this.nextValue, this.nextIndex); } + + static empty() { + return new NullifierLeafPreimage(Fr.ZERO, Fr.ZERO, 0); + } } export class AppendOnlyTreeSnapshot { diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 6a64b9ffdbd6..2682b0e533d5 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -18,9 +18,8 @@ import { UInt8Vector, VK_TREE_HEIGHT, } from '@aztec/circuits.js'; -import { toFriendlyJSON } from '@aztec/circuits.js/utils'; import { Fr, createDebugLogger, toBigIntBE } from '@aztec/foundation'; -import { IndexedTree, LeafData, SiblingPath } from '@aztec/merkle-tree'; +import { LeafData, SiblingPath } from '@aztec/merkle-tree'; import { Tx } from '@aztec/tx'; import { MerkleTreeId, MerkleTreeOperations } from '@aztec/world-state'; import flatMap from 'lodash.flatmap'; @@ -48,7 +47,7 @@ export interface LowNullifierWitnessData { /** * Preimage of the low nullifier that proves non membership */ - preimage: LeafData; + preimage: NullifierLeafPreimage; /** * Sibling path to prove membership of low nullifier */ @@ -139,6 +138,10 @@ export class CircuitPoweredBlockBuilder { // Simulate both base rollup circuits, updating the data, contract, and nullifier trees in the process this.debug(`Running left base rollup simulator`); const [baseRollupInputLeft, baseRollupOutputLeft] = await this.baseRollupCircuit(tx1, tx2); + console.log('BASE ROLLUP OUTPUT'); + console.log(baseRollupOutputLeft); + console.log('END NULL'); + console.log(baseRollupOutputLeft.endNullifierTreeSnapshot.root.toBuffer().toString('hex')); this.debug(`Running right base rollup simulator`); const [baseRollupInputRight, baseRollupOutputRight] = await this.baseRollupCircuit(tx3, tx4); @@ -175,7 +178,7 @@ export class CircuitPoweredBlockBuilder { protected async baseRollupCircuit(tx1: Tx, tx2: Tx) { const rollupInput = await this.buildBaseRollupInput(tx1, tx2); - console.log(`Da rollup input`, toFriendlyJSON(rollupInput)); + // console.log(`Da rollup input`, toFriendlyJSON(rollupInput)); const rollupOutput = await this.simulator.baseRollupCircuit(rollupInput); await this.validateTrees(rollupOutput); return [rollupInput, rollupOutput] as const; @@ -198,7 +201,7 @@ export class CircuitPoweredBlockBuilder { this.validateTree(rollupOutput, MerkleTreeId.CONTRACT_TREE, 'Contract'), this.validateTree(rollupOutput, MerkleTreeId.DATA_TREE, 'PrivateData'), // TODO: Wait for new implementation of nullifier tree to avoid mismatches here - // this.validateTree(rollupOutput, MerkleTreeId.NULLIFIER_TREE, 'Nullifier'), + this.validateTree(rollupOutput, MerkleTreeId.NULLIFIER_TREE, 'Nullifier'), ]); } @@ -428,7 +431,7 @@ export class CircuitPoweredBlockBuilder { let currInsertionIndex: bigint = startInsertionIndex; // Leaf data of hte leaves to be inserted - const insertionSubtree: LeafData[] = []; + const insertionSubtree: NullifierLeafPreimage[] = []; // Low nullifier membership proof sibling paths for (const leaf of leaves) { @@ -437,11 +440,7 @@ export class CircuitPoweredBlockBuilder { // NOTE: null values for nullfier leaves are being changed to 0n current impl is a hack // Default value - const nullifierLeaf: LeafData = { - value: newValue, - nextIndex: 0n, - nextValue: 0n, - }; + const nullifierLeaf: NullifierLeafPreimage = new NullifierLeafPreimage(new Fr(newValue), new Fr(0n), 0); if (touchedNodes.has(indexOfPrevious.index)) { // If the node has already been touched, then we return an empty leaf and sibling path @@ -450,7 +449,7 @@ export class CircuitPoweredBlockBuilder { Buffer.from('0000000000000000000000000000000000000000000000000000000000000000', 'hex'), ); const witness: LowNullifierWitnessData = { - preimage: IndexedTree.initialLeaf(), + preimage: NullifierLeafPreimage.empty(), index: 0n, siblingPath: emptySP, }; @@ -462,27 +461,32 @@ export class CircuitPoweredBlockBuilder { touchedNodes.add(indexOfPrevious.index); const lowNullifier = await this.db.getLeafData(MerkleTreeId.NULLIFIER_TREE, indexOfPrevious.index); - // If no low nullifier can be found, abort - this means the nullifier is invalid // in some way (it should not happen) if (lowNullifier === undefined) { return undefined; } + const lowNullifierPreimage = new NullifierLeafPreimage( + new Fr(lowNullifier.value), + new Fr(lowNullifier.nextValue), + Number(lowNullifier.nextIndex), + ); + // Get sibling path for existence of the old leaf const siblingPath = await this.db.getSiblingPath(MerkleTreeId.NULLIFIER_TREE, BigInt(indexOfPrevious.index)); // Update the running paths const witness = { - preimage: lowNullifier, + preimage: lowNullifierPreimage, index: BigInt(indexOfPrevious.index), siblingPath: siblingPath, }; lowNullifierWitnesses.push(witness); // Update subtree insertion leaf from null data - nullifierLeaf.nextIndex = lowNullifier.nextValue; - nullifierLeaf.nextValue = lowNullifier.nextValue; + nullifierLeaf.nextIndex = lowNullifierPreimage.nextIndex; + nullifierLeaf.nextValue = lowNullifierPreimage.nextValue; // Update the current low nullifier lowNullifier.nextIndex = currInsertionIndex; @@ -500,18 +504,18 @@ export class CircuitPoweredBlockBuilder { // Create insertion subtree and forcefully insert in series // Here we calculate the pointers for the inserted values, if they have not already been updated for (let i = 0; i < leaves.length; i++) { - const newValue = toBigIntBE(leaves[i]); + const newValue = new Fr(toBigIntBE(leaves[i])); // We have already fetched the new low nullifier for this leaf, so we can set its low nullifier const lowNullifier = lowNullifierWitnesses[i].preimage; // If the lowNullifier is 0, then we check the previous leaves for the low nullifier leaf - if (lowNullifier.value === 0n && lowNullifier.nextIndex === 0n && lowNullifier.nextValue === 0n) { + if (lowNullifier.leafValue.isZero() && lowNullifier.nextValue.isZero() && lowNullifier.nextIndex === 0) { for (let j = 0; j < i; j++) { if ( - (insertionSubtree[j].nextValue > newValue && insertionSubtree[j].value < newValue) || - (insertionSubtree[j].nextValue == 0n && insertionSubtree[j].nextIndex == 0n) + (insertionSubtree[j].nextValue > newValue && insertionSubtree[j].leafValue < newValue) || + (insertionSubtree[j].nextValue.isZero() && insertionSubtree[j].nextIndex === 0) ) { - insertionSubtree[j].nextIndex = startInsertionIndex + BigInt(i); + insertionSubtree[j].nextIndex = Number(startInsertionIndex) + i; insertionSubtree[j].nextValue = newValue; } } @@ -523,12 +527,18 @@ export class CircuitPoweredBlockBuilder { // We can skip inserting empty leaves if ( !( - insertionSubtree[i].value === 0n && - insertionSubtree[i].nextIndex === 0n && - insertionSubtree[i].nextValue === 0n + insertionSubtree[i].leafValue.isZero() && + insertionSubtree[i].nextValue.isZero() && + insertionSubtree[i].nextIndex === 0 ) ) { - await this.db.updateLeaf(MerkleTreeId.NULLIFIER_TREE, insertionSubtree[i], startInsertionIndex + BigInt(i)); + // TODO: build first class conversion methods + const asLeafData: LeafData = { + value: insertionSubtree[i].leafValue.value, + nextValue: insertionSubtree[i].nextValue.value, + nextIndex: BigInt(insertionSubtree[i].nextIndex), + }; + await this.db.updateLeaf(MerkleTreeId.NULLIFIER_TREE, asLeafData, startInsertionIndex + BigInt(i)); } } @@ -605,8 +615,7 @@ export class CircuitPoweredBlockBuilder { newCommitmentsSubtreeSiblingPath, newContractsSubtreeSiblingPath, newNullifiersSubtreeSiblingPath, - // TODO: typing for `LowNullifierWitnessData` - lowNullifierLeafPreimages: nullifierWitnesses.map((w: any) => w.preimage), + lowNullifierLeafPreimages: nullifierWitnesses.map((w: LowNullifierWitnessData) => w.preimage), lowNullifierMembershipWitness: lowNullifierMembershipWitnesses, kernelData: [this.getKernelDataFor(tx1), this.getKernelDataFor(tx2)], historicContractsTreeRootMembershipWitnesses: [ From ed9b31c0b3c5fc9fc763ce73f2fb3be584ccfce6 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 11:40:15 +0000 Subject: [PATCH 13/27] fixes --- yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts | 2 +- .../src/block_builder/circuit_powered_block_builder.ts | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index 31fddba11b4e..8ee2b43f2e65 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -260,7 +260,7 @@ export class IndexedTree implements MerkleTree { * Saves the initial leaf to this object and saves it to a database. * TODO: what will the size be */ - private async init(initialSize = 0) { + private async init(initialSize = 1) { // TODO: increase the initial size of the tree to the size of a full rollup insertion - change reflected in c++ to allow subtree insertion this.leaves.push(initialLeaf); diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 2682b0e533d5..8e75cb4ca687 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -178,7 +178,6 @@ export class CircuitPoweredBlockBuilder { protected async baseRollupCircuit(tx1: Tx, tx2: Tx) { const rollupInput = await this.buildBaseRollupInput(tx1, tx2); - // console.log(`Da rollup input`, toFriendlyJSON(rollupInput)); const rollupOutput = await this.simulator.baseRollupCircuit(rollupInput); await this.validateTrees(rollupOutput); return [rollupInput, rollupOutput] as const; From 31ac240c039d2b5974a7b7abe7d0376566c0d4d0 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 11:48:04 +0000 Subject: [PATCH 14/27] exp: trigger ypb rebuild --- yarn-project/yarn-project-base/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index e58bb13b59c7..d4fff549f35a 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -8,6 +8,7 @@ RUN apk update && apk add --no-cache build-base git python3 curl bash jq # COPY --from=1 /usr/src/circuits/build/bin /usr/src/circuits/build-wasm/bin # COPY --from=2 /usr/src/l1-contracts /usr/src/l1-contracts + # TODO: submodule circuits as aztec3-circuits so we don't have two names for the same thing. COPY --from=0 /usr/src/aztec3-circuits/cpp/build-wasm/bin/aztec3-circuits.wasm /usr/src/circuits/cpp/build-wasm/bin/aztec3-circuits.wasm COPY --from=0 /usr/src/aztec3-circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh /usr/src/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh From 4247525a1584ac01f6085844eb04ebdf21002fda Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 12:13:45 +0000 Subject: [PATCH 15/27] chore(circuits): update --- circuits | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuits b/circuits index 501e6bbaff7a..f1599edb4665 160000 --- a/circuits +++ b/circuits @@ -1 +1 @@ -Subproject commit 501e6bbaff7a9e4bd29cff7dd2ba65ae77648b4d +Subproject commit f1599edb46652f404b3b0802157080a81bc92784 From 0d75493f5e31b1bd28bbbe38c02730c1f57da2e5 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 12:29:28 +0000 Subject: [PATCH 16/27] fix: update lockfile --- yarn-project/circuits.js/package.json | 1 + yarn-project/yarn.lock | 1 + 2 files changed, 2 insertions(+) diff --git a/yarn-project/circuits.js/package.json b/yarn-project/circuits.js/package.json index b2f0b0533154..2e7a26f40ff5 100644 --- a/yarn-project/circuits.js/package.json +++ b/yarn-project/circuits.js/package.json @@ -40,6 +40,7 @@ }, "dependencies": { "@aztec/foundation": "workspace:^", + "@aztec/merkle-tree": "workspace:^", "@types/lodash.times": "^4.3.7", "cross-fetch": "^3.1.5", "detect-node": "^2.1.0", diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 40e990a3917d..8b7ec6f06c53 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -206,6 +206,7 @@ __metadata: resolution: "@aztec/circuits.js@workspace:circuits.js" dependencies: "@aztec/foundation": "workspace:^" + "@aztec/merkle-tree": "workspace:^" "@jest/globals": ^29.4.3 "@types/detect-node": ^2.0.0 "@types/jest": ^29.4.0 From 00421ed6939cbb6b7ea95b480c08d136cb4cd653 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 13:03:48 +0000 Subject: [PATCH 17/27] fix: rm only and comments --- .../src/block_builder/circuit_powered_block_builder.test.ts | 2 +- .../src/block_builder/circuit_powered_block_builder.ts | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts index 923ce417b054..fcf0c985098a 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts @@ -161,7 +161,7 @@ describe('sequencer/circuit_block_builder', () => { expect(proof).toEqual(emptyProof); }); - it.only('builds an L2 block with empty txs using wasm circuits', async () => { + it('builds an L2 block with empty txs using wasm circuits', async () => { const simulator = new WasmCircuitSimulator(wasm); const prover = new EmptyProver(); builder = new TestSubject(builderDb, vks, simulator, prover, wasm); diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 8e75cb4ca687..f9929af3c565 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -138,10 +138,6 @@ export class CircuitPoweredBlockBuilder { // Simulate both base rollup circuits, updating the data, contract, and nullifier trees in the process this.debug(`Running left base rollup simulator`); const [baseRollupInputLeft, baseRollupOutputLeft] = await this.baseRollupCircuit(tx1, tx2); - console.log('BASE ROLLUP OUTPUT'); - console.log(baseRollupOutputLeft); - console.log('END NULL'); - console.log(baseRollupOutputLeft.endNullifierTreeSnapshot.root.toBuffer().toString('hex')); this.debug(`Running right base rollup simulator`); const [baseRollupInputRight, baseRollupOutputRight] = await this.baseRollupCircuit(tx3, tx4); @@ -200,7 +196,7 @@ export class CircuitPoweredBlockBuilder { this.validateTree(rollupOutput, MerkleTreeId.CONTRACT_TREE, 'Contract'), this.validateTree(rollupOutput, MerkleTreeId.DATA_TREE, 'PrivateData'), // TODO: Wait for new implementation of nullifier tree to avoid mismatches here - this.validateTree(rollupOutput, MerkleTreeId.NULLIFIER_TREE, 'Nullifier'), + // this.validateTree(rollupOutput, MerkleTreeId.NULLIFIER_TREE, 'Nullifier'), ]); } From 629d89bdcf20bd5168711dd0fea59debe2084856 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 13:17:19 +0000 Subject: [PATCH 18/27] chore(bb): update bb --- circuits | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circuits b/circuits index f1599edb4665..b7ec0b28356a 160000 --- a/circuits +++ b/circuits @@ -1 +1 @@ -Subproject commit f1599edb46652f404b3b0802157080a81bc92784 +Subproject commit b7ec0b28356a1f1f96951dfc605b925adf33c2ff From 0b5b318a7c4371f0b6229fe7740d0a63bea0a43a Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 13:25:41 +0000 Subject: [PATCH 19/27] fix: build issue --- yarn-project/circuits.js/tsconfig.dest.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/yarn-project/circuits.js/tsconfig.dest.json b/yarn-project/circuits.js/tsconfig.dest.json index fea2a64dc145..7e7ab28ea006 100644 --- a/yarn-project/circuits.js/tsconfig.dest.json +++ b/yarn-project/circuits.js/tsconfig.dest.json @@ -8,6 +8,9 @@ "references": [ { "path": "../foundation/tsconfig.dest.json" + }, + { + "path": "../merkle-tree/tsconfig.dest.json" } ], "include": ["src"], From 565853f48d2b4d5825904df6d98c98e8235dfbac Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 18:08:46 +0000 Subject: [PATCH 20/27] fix: nullifier tree fixes --- .../src/indexed_tree/indexed_tree.ts | 14 ++++++ .../src/standard_tree/standard_tree.ts | 7 +++ .../circuit_powered_block_builder.test.ts | 4 +- .../circuit_powered_block_builder.ts | 49 +++++-------------- .../src/world-state-db/merkle_trees.ts | 2 +- 5 files changed, 37 insertions(+), 39 deletions(-) diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index 8ee2b43f2e65..42cc9ce64460 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -183,6 +183,13 @@ export class IndexedTree implements MerkleTree { await this.underlying.updateLeaf(encodedLeaf, index); } + /** + * Special case which will force append zero into the tree by increasing its size + */ + private async appendZero(): Promise { + this.underlying.forceAppendEmptyLeaf(); + } + /** * Appends the given leaf to the tree. * @param leaf - The leaf to append. @@ -190,6 +197,13 @@ export class IndexedTree implements MerkleTree { */ private async appendLeaf(leaf: Buffer): Promise { const newValue = toBigIntBE(leaf); + + // Special case when appending zero + if (newValue === 0n) { + this.appendZero(); + return; + } + const indexOfPrevious = this.findIndexOfPreviousValue(newValue, true); const previousLeafCopy = this.getLatestLeafDataCopy(indexOfPrevious.index, true); if (previousLeafCopy === undefined) { diff --git a/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts b/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts index 92ebae9d9842..eb89aa3a31c2 100644 --- a/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts +++ b/yarn-project/merkle-tree/src/standard_tree/standard_tree.ts @@ -188,6 +188,13 @@ export class StandardMerkleTree implements MerkleTree { } } + /** + * Force increase the size of the tree + */ + public forceAppendEmptyLeaf() { + this.cachedSize = (this.cachedSize ?? this.size) + 1n; + } + /** * Commits the changes to the database. * @returns Empty promise. diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts index fcf0c985098a..cba3a13aa7c2 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.test.ts @@ -159,7 +159,7 @@ describe('sequencer/circuit_block_builder', () => { expect(l2Block.number).toEqual(blockNumber); expect(proof).toEqual(emptyProof); - }); + }, 20000); it('builds an L2 block with empty txs using wasm circuits', async () => { const simulator = new WasmCircuitSimulator(wasm); @@ -197,7 +197,7 @@ describe('sequencer/circuit_block_builder', () => { expect(contractTreeAfter.root).not.toEqual(contractTreeBefore.root); expect(contractTreeAfter.root).toEqual(await expectsDb.getTreeInfo(MerkleTreeId.CONTRACT_TREE).then(t => t.root)); expect(contractTreeAfter.size).toEqual(4n); - }); + }, 10000); }); // Test subject class that exposes internal functions for testing diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index f9929af3c565..0cd9d280d8b8 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -195,8 +195,7 @@ export class CircuitPoweredBlockBuilder { await Promise.all([ this.validateTree(rollupOutput, MerkleTreeId.CONTRACT_TREE, 'Contract'), this.validateTree(rollupOutput, MerkleTreeId.DATA_TREE, 'PrivateData'), - // TODO: Wait for new implementation of nullifier tree to avoid mismatches here - // this.validateTree(rollupOutput, MerkleTreeId.NULLIFIER_TREE, 'Nullifier'), + this.validateTree(rollupOutput, MerkleTreeId.NULLIFIER_TREE, 'Nullifier'), ]); } @@ -436,8 +435,7 @@ export class CircuitPoweredBlockBuilder { // NOTE: null values for nullfier leaves are being changed to 0n current impl is a hack // Default value const nullifierLeaf: NullifierLeafPreimage = new NullifierLeafPreimage(new Fr(newValue), new Fr(0n), 0); - - if (touchedNodes.has(indexOfPrevious.index)) { + if (touchedNodes.has(indexOfPrevious.index) || newValue === 0n) { // If the node has already been touched, then we return an empty leaf and sibling path const emptySP = new SiblingPath(); emptySP.data = Array(dbInfo.depth).fill( @@ -456,6 +454,7 @@ export class CircuitPoweredBlockBuilder { touchedNodes.add(indexOfPrevious.index); const lowNullifier = await this.db.getLeafData(MerkleTreeId.NULLIFIER_TREE, indexOfPrevious.index); + // If no low nullifier can be found, abort - this means the nullifier is invalid // in some way (it should not happen) if (lowNullifier === undefined) { @@ -501,6 +500,8 @@ export class CircuitPoweredBlockBuilder { for (let i = 0; i < leaves.length; i++) { const newValue = new Fr(toBigIntBE(leaves[i])); + if (newValue.isZero()) continue; + // We have already fetched the new low nullifier for this leaf, so we can set its low nullifier const lowNullifier = lowNullifierWitnesses[i].preimage; // If the lowNullifier is 0, then we check the previous leaves for the low nullifier leaf @@ -520,21 +521,14 @@ export class CircuitPoweredBlockBuilder { // For each calculated new leaf, we insert it into the tree at the next position for (let i = 0; i < insertionSubtree.length; i++) { // We can skip inserting empty leaves - if ( - !( - insertionSubtree[i].leafValue.isZero() && - insertionSubtree[i].nextValue.isZero() && - insertionSubtree[i].nextIndex === 0 - ) - ) { - // TODO: build first class conversion methods - const asLeafData: LeafData = { - value: insertionSubtree[i].leafValue.value, - nextValue: insertionSubtree[i].nextValue.value, - nextIndex: BigInt(insertionSubtree[i].nextIndex), - }; - await this.db.updateLeaf(MerkleTreeId.NULLIFIER_TREE, asLeafData, startInsertionIndex + BigInt(i)); - } + + const asLeafData: LeafData = { + value: insertionSubtree[i].leafValue.value, + nextValue: insertionSubtree[i].nextValue.value, + nextIndex: BigInt(insertionSubtree[i].nextIndex), + }; + + await this.db.updateLeaf(MerkleTreeId.NULLIFIER_TREE, asLeafData, startInsertionIndex + BigInt(i)); } return lowNullifierWitnesses; @@ -555,24 +549,12 @@ export class CircuitPoweredBlockBuilder { ); const newCommitments = flatMap([tx1, tx2], tx => tx.data.end.newCommitments.map(x => x.toBuffer())); - // console.log(`Contract root before insertion: `, await this.getTreeSnapshot(MerkleTreeId.CONTRACT_TREE).then(t => t.root.toBuffer().toString('hex'))) - // console.log(`New contracts to insert`, flatMap([tx1, tx2], tx => tx.data.end.newContracts.map(nc => [nc.contractAddress, nc.functionTreeRoot, nc.portalContractAddress].join('/'))).join(', ')) - // console.log(`Inserting new contracts hashes`, newContracts.map(c => c.toString('hex')).join(', ')) await this.db.appendLeaves(MerkleTreeId.CONTRACT_TREE, newContracts); - // console.log(`Contract root after insertion: `, await this.getTreeSnapshot(MerkleTreeId.CONTRACT_TREE).then(t => t.root)) - // console.log(`Data root before insertion: `, await this.getTreeSnapshot(MerkleTreeId.DATA_TREE).then(t => t.root)) - // console.log(`Inserting new data`, newCommitments.map(c => c.toString('hex')).join(', ')) await this.db.appendLeaves(MerkleTreeId.DATA_TREE, newCommitments); - // console.log(`Data root after insertion: `, await this.getTreeSnapshot(MerkleTreeId.DATA_TREE).then(t => t.root)) // Update the nullifier tree, capturing the low nullifier info for each individual operation const newNullifiers = [...tx1.data.end.newNullifiers, ...tx2.data.end.newNullifiers]; - // console.log( - // `Nullifier root before insertion: `, - // await this.getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE).then(t => '0x' + t.root.toBuffer().toString('hex')), - // ); - // console.log(`Inserting new data`, newNullifiers.join(', ')); const nullifierWitnesses = await this.performBaseRollupBatchInsertionProofs(newNullifiers.map(fr => fr.toBuffer())); if (nullifierWitnesses === undefined) { @@ -583,11 +565,6 @@ export class CircuitPoweredBlockBuilder { MembershipWitness.fromSiblingPath(Number(w.index), w.siblingPath), ); - // console.log( - // `Nullifier root after insertion: `, - // await this.getTreeSnapshot(MerkleTreeId.NULLIFIER_TREE).then(t => '0x' + t.root.toBuffer().toString('hex')), - // ); - // Get the subtree sibling paths for the circuit const newCommitmentsSubtreeSiblingPath = await this.getSubtreeSiblingPath( MerkleTreeId.DATA_TREE, diff --git a/yarn-project/world-state/src/world-state-db/merkle_trees.ts b/yarn-project/world-state/src/world-state-db/merkle_trees.ts index 8b3ab458c44d..fc191150bbd5 100644 --- a/yarn-project/world-state/src/world-state-db/merkle_trees.ts +++ b/yarn-project/world-state/src/world-state-db/merkle_trees.ts @@ -205,7 +205,7 @@ export class MerkleTrees implements MerkleTreeDb { * @param leaf - The new leaf value * @param index - The index to insert into */ - public async updateLeaf(treeId: IndexedMerkleTreeId, leaf: Buffer | LeafData, index: bigint): Promise { + public async updateLeaf(treeId: IndexedMerkleTreeId, leaf: LeafData, index: bigint): Promise { return await this.synchronise(() => this.trees[treeId].updateLeaf(leaf, index)); } From 6f8abd4b566ac7a5a0031c9daaf2f7f5f2668bf0 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 18:20:59 +0000 Subject: [PATCH 21/27] fix: indexed tree tests - hashing change --- .../src/indexed_tree/indexed_tree.test.ts | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.test.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.test.ts index 09cbb393793f..4556b7ab3c9e 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.test.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.test.ts @@ -14,11 +14,7 @@ const createFromName = async (levelUp: levelup.LevelUp, hasher: Hasher, name: st }; const createIndexedTreeLeaf = (value: number, nextIndex: number, nextValue: number) => { - return Buffer.concat([ - toBufferBE(BigInt(value), 32), - toBufferBE(BigInt(nextIndex), 32), - toBufferBE(BigInt(nextValue), 32), - ]); + return [toBufferBE(BigInt(value), 32), toBufferBE(BigInt(nextIndex), 32), toBufferBE(BigInt(nextValue), 32)]; }; const verifyCommittedState = async ( @@ -58,7 +54,7 @@ describe('IndexedMerkleTreeSpecific', () => { * nextVal 0 0 0 0 0 0 0 0. */ - const zeroTreeLeafHash = pedersen.hashToField(createIndexedTreeLeaf(0, 0, 0)); + const zeroTreeLeafHash = pedersen.compressInputs(createIndexedTreeLeaf(0, 0, 0)); const level1ZeroHash = pedersen.compress(zeroTreeLeafHash, zeroTreeLeafHash); const level2ZeroHash = pedersen.compress(level1ZeroHash, level1ZeroHash); let root = pedersen.compress(level2ZeroHash, level2ZeroHash); @@ -82,8 +78,8 @@ describe('IndexedMerkleTreeSpecific', () => { * nextIdx 1 0 0 0 0 0 0 0 * nextVal 30 0 0 0 0 0 0 0. */ - let index0Hash = pedersen.hashToField(createIndexedTreeLeaf(0, 1, 30)); - let index1Hash = pedersen.hashToField(createIndexedTreeLeaf(30, 0, 0)); + let index0Hash = pedersen.compressInputs(createIndexedTreeLeaf(0, 1, 30)); + let index1Hash = pedersen.compressInputs(createIndexedTreeLeaf(30, 0, 0)); let e10 = pedersen.compress(index0Hash, index1Hash); let e20 = pedersen.compress(e10, level1ZeroHash); root = pedersen.compress(e20, level2ZeroHash); @@ -106,8 +102,8 @@ describe('IndexedMerkleTreeSpecific', () => { * nextIdx 2 0 1 0 0 0 0 0 * nextVal 10 0 30 0 0 0 0 0. */ - index0Hash = pedersen.hashToField(createIndexedTreeLeaf(0, 2, 10)); - let index2Hash = pedersen.hashToField(createIndexedTreeLeaf(10, 1, 30)); + index0Hash = pedersen.compressInputs(createIndexedTreeLeaf(0, 2, 10)); + let index2Hash = pedersen.compressInputs(createIndexedTreeLeaf(10, 1, 30)); e10 = pedersen.compress(index0Hash, index1Hash); let e11 = pedersen.compress(index2Hash, zeroTreeLeafHash); e20 = pedersen.compress(e10, e11); @@ -132,8 +128,8 @@ describe('IndexedMerkleTreeSpecific', () => { * nextVal 10 0 20 30 0 0 0 0. */ e10 = pedersen.compress(index0Hash, index1Hash); - index2Hash = pedersen.hashToField(createIndexedTreeLeaf(10, 3, 20)); - const index3Hash = pedersen.hashToField(createIndexedTreeLeaf(20, 1, 30)); + index2Hash = pedersen.compressInputs(createIndexedTreeLeaf(10, 3, 20)); + const index3Hash = pedersen.compressInputs(createIndexedTreeLeaf(20, 1, 30)); e11 = pedersen.compress(index2Hash, index3Hash); e20 = pedersen.compress(e10, e11); root = pedersen.compress(e20, level2ZeroHash); @@ -156,8 +152,8 @@ describe('IndexedMerkleTreeSpecific', () => { * nextIdx 2 4 3 1 0 0 0 0 * nextVal 10 50 20 30 0 0 0 0. */ - index1Hash = pedersen.hashToField(createIndexedTreeLeaf(30, 4, 50)); - const index4Hash = pedersen.hashToField(createIndexedTreeLeaf(50, 0, 0)); + index1Hash = pedersen.compressInputs(createIndexedTreeLeaf(30, 4, 50)); + const index4Hash = pedersen.compressInputs(createIndexedTreeLeaf(50, 0, 0)); e10 = pedersen.compress(index0Hash, index1Hash); e20 = pedersen.compress(e10, e11); const e12 = pedersen.compress(index4Hash, zeroTreeLeafHash); From 1e9adb08b3c788cadacc2f85993b0ede420f6884 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 18:32:12 +0000 Subject: [PATCH 22/27] exp --- yarn-project/archiver/tsconfig.dest.json | 3 ++- yarn-project/p2p/tsconfig.dest.json | 3 ++- .../src/synchroniser/server_world_state_synchroniser.ts | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/yarn-project/archiver/tsconfig.dest.json b/yarn-project/archiver/tsconfig.dest.json index b7586cab1eec..fa9d0bec6b09 100644 --- a/yarn-project/archiver/tsconfig.dest.json +++ b/yarn-project/archiver/tsconfig.dest.json @@ -14,7 +14,8 @@ }, { "path": "../l2-block/tsconfig.dest.json" - } + }, + { "path": "../merkle-tree/tsconfig.dest.json" } ], "exclude": ["**/*.test.*", "**/fixtures/*"], "include": ["src"] diff --git a/yarn-project/p2p/tsconfig.dest.json b/yarn-project/p2p/tsconfig.dest.json index 924501cbdd91..b37dbbf386f1 100644 --- a/yarn-project/p2p/tsconfig.dest.json +++ b/yarn-project/p2p/tsconfig.dest.json @@ -8,7 +8,8 @@ "references": [ { "path": "../foundation/tsconfig.dest.json" }, { "path": "../l2-block/tsconfig.dest.json" }, - { "path": "../tx/tsconfig.dest.json" } + { "path": "../tx/tsconfig.dest.json" }, + { "path": "../merkle-tree/tsconfig.dest.json" } ], "include": ["src"], "exclude": ["**/*.test.*", "**/fixtures/*"] diff --git a/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts b/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts index beb493ad569d..8bc7a12fdb61 100644 --- a/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts +++ b/yarn-project/world-state/src/synchroniser/server_world_state_synchroniser.ts @@ -3,7 +3,6 @@ import { L2Block, L2BlockDownloader, L2BlockSource } from '@aztec/l2-block'; import { MerkleTreeDb, MerkleTreeId, MerkleTreeOperations } from '../index.js'; import { MerkleTreeOperationsFacade } from '../merkle-tree/merkle_tree_operations_facade.js'; import { WorldStateRunningState, WorldStateStatus, WorldStateSynchroniser } from './world_state_synchroniser.js'; -import { LeafData } from '@aztec/merkle-tree'; /** * Synchronises the world state with the L2 blocks from a L2BlockSource. From 315d4ee4449413622efb4d658910338f2dd6bfe3 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 19:08:32 +0000 Subject: [PATCH 23/27] remove sibling path merkle tree spread --- yarn-project/circuits.js/src/structs/shared.ts | 7 +++---- yarn-project/circuits.js/tsconfig.dest.json | 3 --- yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts | 2 +- .../src/block_builder/circuit_powered_block_builder.ts | 4 ++-- yarn-project/yarn-project-base/Dockerfile | 1 - 5 files changed, 6 insertions(+), 11 deletions(-) diff --git a/yarn-project/circuits.js/src/structs/shared.ts b/yarn-project/circuits.js/src/structs/shared.ts index 1770849e0529..1e5ba604de58 100644 --- a/yarn-project/circuits.js/src/structs/shared.ts +++ b/yarn-project/circuits.js/src/structs/shared.ts @@ -1,6 +1,5 @@ import { BufferReader, randomBytes } from '@aztec/foundation'; import { Fq, Fr } from '@aztec/foundation/fields'; -import { SiblingPath } from '@aztec/merkle-tree'; import { assertLength, range } from '../utils/jsUtils.js'; import { Bufferable, serializeToBuffer } from '../utils/serialize.js'; import times from 'lodash.times'; @@ -29,11 +28,11 @@ export class MembershipWitness { return new MembershipWitness(pathSize, leafIndex, arr); } - static fromSiblingPath(leafIndex: number, siblingPath: SiblingPath) { + static fromBufferArray(leafIndex: number, siblingPath: Buffer[]) { return new MembershipWitness( - siblingPath.data.length, + siblingPath.length, leafIndex, - siblingPath.data.map(x => Fr.fromBuffer(x)), + siblingPath.map(x => Fr.fromBuffer(x)), ); } } diff --git a/yarn-project/circuits.js/tsconfig.dest.json b/yarn-project/circuits.js/tsconfig.dest.json index 7e7ab28ea006..fea2a64dc145 100644 --- a/yarn-project/circuits.js/tsconfig.dest.json +++ b/yarn-project/circuits.js/tsconfig.dest.json @@ -8,9 +8,6 @@ "references": [ { "path": "../foundation/tsconfig.dest.json" - }, - { - "path": "../merkle-tree/tsconfig.dest.json" } ], "include": ["src"], diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index 42cc9ce64460..6d889469d4b9 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -186,7 +186,7 @@ export class IndexedTree implements MerkleTree { /** * Special case which will force append zero into the tree by increasing its size */ - private async appendZero(): Promise { + private appendZero(): void { this.underlying.forceAppendEmptyLeaf(); } diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 0cd9d280d8b8..3430599cd54c 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -19,7 +19,7 @@ import { VK_TREE_HEIGHT, } from '@aztec/circuits.js'; import { Fr, createDebugLogger, toBigIntBE } from '@aztec/foundation'; -import { LeafData, SiblingPath } from '@aztec/merkle-tree'; +import { LeafData } from '@aztec/merkle-tree'; import { Tx } from '@aztec/tx'; import { MerkleTreeId, MerkleTreeOperations } from '@aztec/world-state'; import flatMap from 'lodash.flatmap'; @@ -562,7 +562,7 @@ export class CircuitPoweredBlockBuilder { } // Extract witness objects from returned data const lowNullifierMembershipWitnesses = nullifierWitnesses.map(w => - MembershipWitness.fromSiblingPath(Number(w.index), w.siblingPath), + MembershipWitness.fromBufferArray(Number(w.index), w.siblingPath.data), ); // Get the subtree sibling paths for the circuit diff --git a/yarn-project/yarn-project-base/Dockerfile b/yarn-project/yarn-project-base/Dockerfile index d4fff549f35a..e58bb13b59c7 100644 --- a/yarn-project/yarn-project-base/Dockerfile +++ b/yarn-project/yarn-project-base/Dockerfile @@ -8,7 +8,6 @@ RUN apk update && apk add --no-cache build-base git python3 curl bash jq # COPY --from=1 /usr/src/circuits/build/bin /usr/src/circuits/build-wasm/bin # COPY --from=2 /usr/src/l1-contracts /usr/src/l1-contracts - # TODO: submodule circuits as aztec3-circuits so we don't have two names for the same thing. COPY --from=0 /usr/src/aztec3-circuits/cpp/build-wasm/bin/aztec3-circuits.wasm /usr/src/circuits/cpp/build-wasm/bin/aztec3-circuits.wasm COPY --from=0 /usr/src/aztec3-circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh /usr/src/circuits/cpp/barretenberg/cpp/srs_db/download_ignition.sh From 99f0b542c0928248b5a289cc3fe4ceb2e47e7255 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 19:21:01 +0000 Subject: [PATCH 24/27] wip --- yarn-project/archiver/tsconfig.dest.json | 3 +-- yarn-project/p2p/tsconfig.dest.json | 3 +-- .../src/block_builder/circuit_powered_block_builder.ts | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/yarn-project/archiver/tsconfig.dest.json b/yarn-project/archiver/tsconfig.dest.json index fa9d0bec6b09..b7586cab1eec 100644 --- a/yarn-project/archiver/tsconfig.dest.json +++ b/yarn-project/archiver/tsconfig.dest.json @@ -14,8 +14,7 @@ }, { "path": "../l2-block/tsconfig.dest.json" - }, - { "path": "../merkle-tree/tsconfig.dest.json" } + } ], "exclude": ["**/*.test.*", "**/fixtures/*"], "include": ["src"] diff --git a/yarn-project/p2p/tsconfig.dest.json b/yarn-project/p2p/tsconfig.dest.json index b37dbbf386f1..924501cbdd91 100644 --- a/yarn-project/p2p/tsconfig.dest.json +++ b/yarn-project/p2p/tsconfig.dest.json @@ -8,8 +8,7 @@ "references": [ { "path": "../foundation/tsconfig.dest.json" }, { "path": "../l2-block/tsconfig.dest.json" }, - { "path": "../tx/tsconfig.dest.json" }, - { "path": "../merkle-tree/tsconfig.dest.json" } + { "path": "../tx/tsconfig.dest.json" } ], "include": ["src"], "exclude": ["**/*.test.*", "**/fixtures/*"] diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 3430599cd54c..fac8c32d68d1 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -19,7 +19,7 @@ import { VK_TREE_HEIGHT, } from '@aztec/circuits.js'; import { Fr, createDebugLogger, toBigIntBE } from '@aztec/foundation'; -import { LeafData } from '@aztec/merkle-tree'; +import { LeafData, SiblingPath } from '@aztec/merkle-tree'; import { Tx } from '@aztec/tx'; import { MerkleTreeId, MerkleTreeOperations } from '@aztec/world-state'; import flatMap from 'lodash.flatmap'; From 75d4b2c120108268f931748f16bd6469dfe957f3 Mon Sep 17 00:00:00 2001 From: cheethas Date: Wed, 5 Apr 2023 21:43:54 +0000 Subject: [PATCH 25/27] clean --- .../src/indexed_tree/indexed_tree.ts | 5 ----- .../circuit_powered_block_builder.ts | 18 ++++++++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts index 6d889469d4b9..bfcbafe032c1 100644 --- a/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts +++ b/yarn-project/merkle-tree/src/indexed_tree/indexed_tree.ts @@ -101,7 +101,6 @@ export class IndexedTree implements MerkleTree { } /** - * TODO: Just export this or keep it static? * Returns an empty leaf of the tree. * @returns An empty leaf. */ @@ -272,15 +271,11 @@ export class IndexedTree implements MerkleTree { /** * Saves the initial leaf to this object and saves it to a database. - * TODO: what will the size be */ private async init(initialSize = 1) { - // TODO: increase the initial size of the tree to the size of a full rollup insertion - change reflected in c++ to allow subtree insertion - this.leaves.push(initialLeaf); await this.underlying.appendLeaves([hashEncodedTreeValue(initialLeaf, this.hasher)]); - // TODO: optimise for (let i = 1; i < initialSize; i++) { await this.appendLeaf(Buffer.from([i])); } diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index fac8c32d68d1..35754e4f25cf 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -42,7 +42,9 @@ const FUTURE_NUM = 0; const DELETE_FR = new Fr(0n); const DELETE_NUM = 0; -// TODO: doc +/** + * All of the data required for the circuit compute and verify nullifiers + */ export interface LowNullifierWitnessData { /** * Preimage of the low nullifier that proves non membership @@ -57,6 +59,7 @@ export interface LowNullifierWitnessData { */ index: bigint; } + export class CircuitPoweredBlockBuilder { constructor( protected db: MerkleTreeOperations, @@ -398,19 +401,22 @@ export class CircuitPoweredBlockBuilder { } /** - * Each base rollup needs to provide non membership / inclusion proofs for each of the nullifiers - * generated in the kernel circuit that it is rolling up. + * Each base rollup needs to provide non membership / inclusion proofs for each of the nullifier. + * This method will return membership proofs and perform partial node updates that will + * allow the circuit to incrementally update the tree and perform a batch insertion. + * + * This offers massive circuit performance savings over doing incremental insertions. * - * As leaves are batch inserted at the end, batch updates are a special case. + * A description of the algorithm can be found here: https://colab.research.google.com/drive/1A0gizduSi4FIiIJZ8OylwIpO9-OTqV-R * * WARNING: This function has side effects, it will insert values into the tree. * * Assumptions: * 1. There are 8 nullifiers provided and they are all unique * 2. If kc 0 has 1 nullifier, and kc 1 has 3 nullifiers the layout will assume to be the sparse - * nullifier layout: [kc0N, 0, 0, 0, kc1N, kc1N, kc1N, 0] + * nullifier layout: [kc0-0, 0, 0, 0, kc1-0, kc1-1, kc1-2, 0] * - * TODO: include indepth insertion writeup in this comment + * TODO: this implementation will change once the zero value is changed from h(0,0,0). Changes incoming over the next sprint * @param leaves Values to insert into the tree * @returns */ From 3c49d688ca84bf672adcfc34d33f2806632d2c36 Mon Sep 17 00:00:00 2001 From: cheethas Date: Thu, 6 Apr 2023 08:38:52 +0000 Subject: [PATCH 26/27] fix: remove unused dep, update comment --- yarn-project/circuits.js/package.json | 1 - .../src/block_builder/circuit_powered_block_builder.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/yarn-project/circuits.js/package.json b/yarn-project/circuits.js/package.json index 2e7a26f40ff5..b2f0b0533154 100644 --- a/yarn-project/circuits.js/package.json +++ b/yarn-project/circuits.js/package.json @@ -40,7 +40,6 @@ }, "dependencies": { "@aztec/foundation": "workspace:^", - "@aztec/merkle-tree": "workspace:^", "@types/lodash.times": "^4.3.7", "cross-fetch": "^3.1.5", "detect-node": "^2.1.0", diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 35754e4f25cf..78f7a6807443 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -412,7 +412,7 @@ export class CircuitPoweredBlockBuilder { * WARNING: This function has side effects, it will insert values into the tree. * * Assumptions: - * 1. There are 8 nullifiers provided and they are all unique + * 1. There are 8 nullifiers provided and they are either unique or empty. (denoted as 0) * 2. If kc 0 has 1 nullifier, and kc 1 has 3 nullifiers the layout will assume to be the sparse * nullifier layout: [kc0-0, 0, 0, 0, kc1-0, kc1-1, kc1-2, 0] * From f571f33b7857203cc8c3176a5adfcadbf8f996df Mon Sep 17 00:00:00 2001 From: cheethas Date: Thu, 6 Apr 2023 08:49:28 +0000 Subject: [PATCH 27/27] fix: stale comment --- .../src/block_builder/circuit_powered_block_builder.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts index 78f7a6807443..4eabd13bc1bb 100644 --- a/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts +++ b/yarn-project/sequencer-client/src/block_builder/circuit_powered_block_builder.ts @@ -526,8 +526,6 @@ export class CircuitPoweredBlockBuilder { // For each calculated new leaf, we insert it into the tree at the next position for (let i = 0; i < insertionSubtree.length; i++) { - // We can skip inserting empty leaves - const asLeafData: LeafData = { value: insertionSubtree[i].leafValue.value, nextValue: insertionSubtree[i].nextValue.value,