From edf7a3b3539258cac59a7aa742fdc35cb66c877b Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 13 Dec 2024 17:17:33 +0000 Subject: [PATCH 01/10] refactor: getLogsByTags request batching --- yarn-project/circuit-types/src/interfaces/aztec-node.ts | 3 ++- yarn-project/pxe/src/simulator_oracle/index.ts | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 427ff8d88ace..8d9812bac3b0 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -315,7 +315,8 @@ export interface AztecNode * Gets all logs that match any of the received tags (i.e. logs with their first field equal to a tag). * @param tags - The tags to filter the logs by. * @returns For each received tag, an array of matching logs and metadata (e.g. tx hash) is returned. An empty - array implies no logs match that tag. + * array implies no logs match that tag. There can be multiple logs for 1 tag because it could happen that a tag + * would be reused --> e.g. when sending a note from multiple unsynched devices. */ getLogsByTags(tags: Fr[]): Promise; diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 160947e25c17..e5aecf4258b2 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -492,6 +492,8 @@ export class SimulatorOracle implements DBOracle { contractAddress, }, ); + // There can be multiple logs for 1 tag because it could happen that a tag would be reused + // --> e.g. when sending a note from multiple unsynched devices. logs.push(...logsByTag); if (currentIndex >= initialIndexesMap[currentSecretAsStr]) { From e35accb409e3fde2e2fc0437c526625e3075e123 Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 13 Dec 2024 18:46:50 +0000 Subject: [PATCH 02/10] WIP --- .../src/interfaces/aztec-node.ts | 4 +- .../pxe/src/simulator_oracle/index.ts | 139 +++++++----------- .../pxe/src/simulator_oracle/tagging_utils.ts | 19 ++- 3 files changed, 72 insertions(+), 90 deletions(-) diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 8d9812bac3b0..e19794888137 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -315,8 +315,8 @@ export interface AztecNode * Gets all logs that match any of the received tags (i.e. logs with their first field equal to a tag). * @param tags - The tags to filter the logs by. * @returns For each received tag, an array of matching logs and metadata (e.g. tx hash) is returned. An empty - * array implies no logs match that tag. There can be multiple logs for 1 tag because it could happen that a tag - * would be reused --> e.g. when sending a note from multiple unsynched devices. + * array implies no logs match that tag. There can be multiple logs for 1 tag because tag reuse can happen + * --> e.g. when sending a note from multiple unsynched devices. */ getLogsByTags(tags: Fr[]): Promise; diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index e5aecf4258b2..7431edaa5d1a 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -38,7 +38,12 @@ import { type IncomingNoteDao } from '../database/incoming_note_dao.js'; import { type PxeDatabase } from '../database/index.js'; import { produceNoteDaos } from '../note_decryption_utils/produce_note_daos.js'; import { getAcirSimulator } from '../simulator/index.js'; -import { getInitialIndexes, getLeftMostIndexedTaggingSecrets, getRightMostIndexes } from './tagging_utils.js'; +import { + getIndexedTaggingSecretsForTheWholeWindow, + getInitialIndexesMap, + getLeftMostIndexedTaggingSecrets, + getRightMostIndexes, +} from './tagging_utils.js'; /** * A data oracle that provides information needed for simulating a transaction. @@ -437,7 +442,7 @@ export class SimulatorOracle implements DBOracle { // length, since we don't really know the note they correspond to until we decrypt them. // 1. Get all the secrets for the recipient and sender pairs (#9365) - const indexedTaggingSecrets = await this.#getIndexedTaggingSecretsForContacts(contractAddress, recipient); + const secrets = await this.#getIndexedTaggingSecretsForContacts(contractAddress, recipient); // 1.1 Set up a sliding window with an offset. Chances are the sender might have messed up // and inadvertently incremented their index without us getting any logs (for example, in case @@ -447,93 +452,55 @@ export class SimulatorOracle implements DBOracle { // so we might have missed some logs. For these reasons, we have to look both back and ahead of // the stored index. - // App tagging secrets along with an index in a window to check in the current iteration. Called current because - // this value will be updated as we iterate through the window. - let currentSecrets = getLeftMostIndexedTaggingSecrets(indexedTaggingSecrets, WINDOW_HALF_SIZE); - // Right-most indexes in a window to check stored in a key-value map where key is the app tagging secret - // and value is the index to check (the right-most index in the window). - const rightMostIndexesMap = getRightMostIndexes(indexedTaggingSecrets, WINDOW_HALF_SIZE); // The initial/unmodified indexes of the secrets stored in a key-value map where key is the app tagging secret. - const initialIndexesMap = getInitialIndexes(indexedTaggingSecrets); - // A map of indexes to increment for secrets for which we have found logs with an index higher than the one - // stored. - const indexesToIncrementMap: { [k: string]: number } = {}; - - while (currentSecrets.length > 0) { - // 2. Compute tags using the secrets, recipient and index. Obtain logs for each tag (#9380) - const currentTags = currentSecrets.map(secret => - // We compute the siloed tags since we need the tags as they appear in the log. - secret.computeSiloedTag(recipient, contractAddress), - ); + const initialIndexesMap = getInitialIndexesMap(secrets); + + // TODO(benesjan): merge with #getIndexedTaggingSecretsForContacts? + const secretsForTheWholeWindow = getIndexedTaggingSecretsForTheWholeWindow( + contractAddress, + recipient, + secrets, + WINDOW_HALF_SIZE, + ); + const tagsForTheWholeWindow = secretsForTheWholeWindow.map(secret => + secret.computeSiloedTag(recipient, contractAddress), + ); - // Fetch the logs for the tags and iterate over them - const logsByTags = await this.aztecNode.getLogsByTags(currentTags); - const secretsWithNewIndex: IndexedTaggingSecret[] = []; - logsByTags.forEach((logsByTag, logIndex) => { - const { appTaggingSecret: currentSecret, index: currentIndex } = currentSecrets[logIndex]; - const currentSecretAsStr = currentSecret.toString(); - this.log.debug(`Syncing logs for recipient ${recipient} at contract ${contractName}(${contractAddress})`, { - recipient, - secret: currentSecret, - index: currentIndex, - contractName, - contractAddress, - }); - // 3.1. Append logs to the list and increment the index for the tags that have logs (#9380) - if (logsByTag.length > 0) { - const newIndex = currentIndex + 1; - this.log.debug( - `Found ${logsByTag.length} logs as recipient ${recipient}. Incrementing index to ${newIndex} at contract ${contractName}(${contractAddress})`, - { - recipient, - secret: currentSecret, - newIndex, - contractName, - contractAddress, - }, - ); - // There can be multiple logs for 1 tag because it could happen that a tag would be reused - // --> e.g. when sending a note from multiple unsynched devices. - logs.push(...logsByTag); - - if (currentIndex >= initialIndexesMap[currentSecretAsStr]) { - // 3.2. We found an index higher than the stored/initial one so we update it in the db later on (#9380) - indexesToIncrementMap[currentSecretAsStr] = newIndex; - // 3.3. We found an index higher than the initial one so we slide the window. - rightMostIndexesMap[currentSecretAsStr] = currentIndex + WINDOW_HALF_SIZE; - } + // If we find a log with an index greater than the one stored in the db, we want to fetch logs for tags that + // correspond to the difference of the window sets and then we want to store the new largest index in the db. + const newLargestIndexMap: { [k: string]: number } = {}; + + // Fetch the logs for the tags and iterate over them + const logsByTags = await this.aztecNode.getLogsByTags(tagsForTheWholeWindow); + + logsByTags.forEach((logsByTag, logIndex) => { + // What do I even want to do here? + // I want to check if the log for the given tag exists + // --> if it does, I want to: + // - add it to the logs to process + // - if it corresponds to an index > current one in the db + // - I want update it in the db + // - I want to fetch the difference of the window sets + if (logsByTag.length > 0) { + // The logs for the given tag exist so I add them for later processing + logs.push(...logsByTag); + + // I fetch the indexed tagging secret corresponding to the log as I need that to evaluate whether + // a new largest index have been found. + const secretCorrespondingToLog = secretsForTheWholeWindow[logIndex]; + const initialIndex = initialIndexesMap[secretCorrespondingToLog.appTaggingSecret.toString()]; + + if ( + secretCorrespondingToLog.index > initialIndex && + (!newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()] || + secretCorrespondingToLog.index > newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()]) + ) { + // We have found a new largest index so we store it for later processing (storing it in db + fetching + // the difference of the window sets of current and the next iteration) + newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()] = secretCorrespondingToLog.index; } - // 3.4 Keep increasing the index (inside the window) temporarily for the tags that have no logs - // There's a chance the sender missed some and we want to catch up - if (currentIndex < rightMostIndexesMap[currentSecretAsStr]) { - const newTaggingSecret = new IndexedTaggingSecret(currentSecret, currentIndex + 1); - secretsWithNewIndex.push(newTaggingSecret); - } - }); - - // We store the new indexes for the secrets that have logs with an index higher than the one stored. - await this.db.setTaggingSecretsIndexesAsRecipient( - Object.keys(indexesToIncrementMap).map( - secret => new IndexedTaggingSecret(Fr.fromHexString(secret), indexesToIncrementMap[secret]), - ), - ); - - // We've processed all the current secret-index pairs so we proceed to the next iteration. - currentSecrets = secretsWithNewIndex; - } - - newLogsMap.set( - recipient.toString(), - // Remove logs with a block number higher than the max block number - // Duplicates are likely to happen due to the sliding window, so we also filter them out - logs.filter( - (log, index, self) => - // The following condition is true if the log has small enough block number and is unique - // --> the right side of the && is true if the index of the current log is the first occurrence - // of the log in the array --> that way we ensure uniqueness. - log.blockNumber <= maxBlockNumber && index === self.findIndex(otherLog => otherLog.equals(log)), - ), - ); + } + }); } return newLogsMap; } diff --git a/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts b/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts index 52ed10a1cfc3..5e7d22e37a53 100644 --- a/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts +++ b/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts @@ -1,4 +1,4 @@ -import { IndexedTaggingSecret } from '@aztec/circuits.js'; +import { AztecAddress, Fr, IndexedTaggingSecret } from '@aztec/circuits.js'; /** * Gets indexed tagging secrets with leftmost indexes. @@ -19,6 +19,21 @@ export function getLeftMostIndexedTaggingSecrets( ); } +export function getIndexedTaggingSecretsForTheWholeWindow( + app: AztecAddress, + recipient: AztecAddress, + storedIndexedTaggingSecrets: IndexedTaggingSecret[], + windowHalfSize: number, +): IndexedTaggingSecret[] { + const secrets: IndexedTaggingSecret[] = []; + for (const storedIndexedTaggingSecret of storedIndexedTaggingSecrets) { + for (let i = storedIndexedTaggingSecret.index - windowHalfSize; i <= storedIndexedTaggingSecret.index + windowHalfSize; i++) { + secrets.push(new IndexedTaggingSecret(storedIndexedTaggingSecret.appTaggingSecret, i)); + } + } + return secrets; +} + /** * Creates a map from app tagging secret to rightmost index. * @param indexedTaggingSecrets - The indexed tagging secrets to get the rightmost indexes from. @@ -43,7 +58,7 @@ export function getRightMostIndexes( * @param indexedTaggingSecrets - The indexed tagging secrets to get the initial indexes from. * @returns The map from app tagging secret to initial index. */ -export function getInitialIndexes(indexedTaggingSecrets: IndexedTaggingSecret[]): { [k: string]: number } { +export function getInitialIndexesMap(indexedTaggingSecrets: IndexedTaggingSecret[]): { [k: string]: number } { const initialIndexes: { [k: string]: number } = {}; for (const indexedTaggingSecret of indexedTaggingSecrets) { From aa432f273d3ae14007a063d5653d1bfe4b912d86 Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 13 Dec 2024 19:08:57 +0000 Subject: [PATCH 03/10] WIP --- .../pxe/src/simulator_oracle/index.ts | 97 ++++++++++--------- .../pxe/src/simulator_oracle/tagging_utils.ts | 13 +-- 2 files changed, 57 insertions(+), 53 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 7431edaa5d1a..e2dc803aee79 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -39,7 +39,7 @@ import { type PxeDatabase } from '../database/index.js'; import { produceNoteDaos } from '../note_decryption_utils/produce_note_daos.js'; import { getAcirSimulator } from '../simulator/index.js'; import { - getIndexedTaggingSecretsForTheWholeWindow, + getIndexedTaggingSecretsForTheWindow, getInitialIndexesMap, getLeftMostIndexedTaggingSecrets, getRightMostIndexes, @@ -455,52 +455,59 @@ export class SimulatorOracle implements DBOracle { // The initial/unmodified indexes of the secrets stored in a key-value map where key is the app tagging secret. const initialIndexesMap = getInitialIndexesMap(secrets); - // TODO(benesjan): merge with #getIndexedTaggingSecretsForContacts? - const secretsForTheWholeWindow = getIndexedTaggingSecretsForTheWholeWindow( - contractAddress, - recipient, - secrets, - WINDOW_HALF_SIZE, - ); - const tagsForTheWholeWindow = secretsForTheWholeWindow.map(secret => - secret.computeSiloedTag(recipient, contractAddress), - ); + const secretsAndWindows = secrets.map(secret => { + return { + appTaggingSecret: secret.appTaggingSecret, + leftMostIndex: secret.index - WINDOW_HALF_SIZE, + rightMostIndex: secret.index + WINDOW_HALF_SIZE, + }; + }); + + while (secretsAndWindows.length > 0) { + const secretsForTheWholeWindow = getIndexedTaggingSecretsForTheWindow(secretsAndWindows); + const tagsForTheWholeWindow = secretsForTheWholeWindow.map(secret => + secret.computeSiloedTag(recipient, contractAddress), + ); - // If we find a log with an index greater than the one stored in the db, we want to fetch logs for tags that - // correspond to the difference of the window sets and then we want to store the new largest index in the db. - const newLargestIndexMap: { [k: string]: number } = {}; - - // Fetch the logs for the tags and iterate over them - const logsByTags = await this.aztecNode.getLogsByTags(tagsForTheWholeWindow); - - logsByTags.forEach((logsByTag, logIndex) => { - // What do I even want to do here? - // I want to check if the log for the given tag exists - // --> if it does, I want to: - // - add it to the logs to process - // - if it corresponds to an index > current one in the db - // - I want update it in the db - // - I want to fetch the difference of the window sets - if (logsByTag.length > 0) { - // The logs for the given tag exist so I add them for later processing - logs.push(...logsByTag); - - // I fetch the indexed tagging secret corresponding to the log as I need that to evaluate whether - // a new largest index have been found. - const secretCorrespondingToLog = secretsForTheWholeWindow[logIndex]; - const initialIndex = initialIndexesMap[secretCorrespondingToLog.appTaggingSecret.toString()]; - - if ( - secretCorrespondingToLog.index > initialIndex && - (!newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()] || - secretCorrespondingToLog.index > newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()]) - ) { - // We have found a new largest index so we store it for later processing (storing it in db + fetching - // the difference of the window sets of current and the next iteration) - newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()] = secretCorrespondingToLog.index; + // If we find a log with an index greater than the one stored in the db, we want to fetch logs for tags that + // correspond to the difference of the window sets and then we want to store the new largest index in the db. + const newLargestIndexMap: { [k: string]: number } = {}; + + // Fetch the logs for the tags and iterate over them + const logsByTags = await this.aztecNode.getLogsByTags(tagsForTheWholeWindow); + + logsByTags.forEach((logsByTag, logIndex) => { + // What do I even want to do here? + // I want to check if the log for the given tag exists + // --> if it does, I want to: + // - add it to the logs to process + // - if it corresponds to an index > current one in the db + // - I want update it in the db + // - I want to fetch the difference of the window sets + if (logsByTag.length > 0) { + // The logs for the given tag exist so I add them for later processing + logs.push(...logsByTag); + + // I fetch the indexed tagging secret corresponding to the log as I need that to evaluate whether + // a new largest index have been found. + const secretCorrespondingToLog = secretsForTheWholeWindow[logIndex]; + const initialIndex = initialIndexesMap[secretCorrespondingToLog.appTaggingSecret.toString()]; + + if ( + secretCorrespondingToLog.index > initialIndex && + (!newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()] || + secretCorrespondingToLog.index > + newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()]) + ) { + // We have found a new largest index so we store it for later processing (storing it in the db + fetching + // the difference of the window sets of current and the next iteration) + newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()] = secretCorrespondingToLog.index; + } } - } - }); + }); + + // Now based on the new largest indexes we found, I will construct a new secrets and windows + } } return newLogsMap; } diff --git a/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts b/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts index 5e7d22e37a53..dab351652028 100644 --- a/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts +++ b/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts @@ -19,16 +19,13 @@ export function getLeftMostIndexedTaggingSecrets( ); } -export function getIndexedTaggingSecretsForTheWholeWindow( - app: AztecAddress, - recipient: AztecAddress, - storedIndexedTaggingSecrets: IndexedTaggingSecret[], - windowHalfSize: number, +export function getIndexedTaggingSecretsForTheWindow( + secretsAndWindows: { appTaggingSecret: Fr; leftMostIndex: number; rightMostIndex: number }[], ): IndexedTaggingSecret[] { const secrets: IndexedTaggingSecret[] = []; - for (const storedIndexedTaggingSecret of storedIndexedTaggingSecrets) { - for (let i = storedIndexedTaggingSecret.index - windowHalfSize; i <= storedIndexedTaggingSecret.index + windowHalfSize; i++) { - secrets.push(new IndexedTaggingSecret(storedIndexedTaggingSecret.appTaggingSecret, i)); + for (const secretAndWindow of secretsAndWindows) { + for (let i = secretAndWindow.leftMostIndex; i <= secretAndWindow.rightMostIndex; i++) { + secrets.push(new IndexedTaggingSecret(secretAndWindow.appTaggingSecret, i)); } } return secrets; From 232995045bb7d9c90658ca6da6ec4a1430980c22 Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 13 Dec 2024 19:38:28 +0000 Subject: [PATCH 04/10] WIP --- .../pxe/src/simulator_oracle/index.ts | 63 ++++++++++++++----- .../pxe/src/simulator_oracle/tagging_utils.ts | 40 +----------- 2 files changed, 48 insertions(+), 55 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index e2dc803aee79..a8f5cd235aba 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -38,12 +38,7 @@ import { type IncomingNoteDao } from '../database/incoming_note_dao.js'; import { type PxeDatabase } from '../database/index.js'; import { produceNoteDaos } from '../note_decryption_utils/produce_note_daos.js'; import { getAcirSimulator } from '../simulator/index.js'; -import { - getIndexedTaggingSecretsForTheWindow, - getInitialIndexesMap, - getLeftMostIndexedTaggingSecrets, - getRightMostIndexes, -} from './tagging_utils.js'; +import { getIndexedTaggingSecretsForTheWindow, getInitialIndexesMap } from './tagging_utils.js'; /** * A data oracle that provides information needed for simulating a transaction. @@ -430,9 +425,10 @@ export class SimulatorOracle implements DBOracle { const WINDOW_HALF_SIZE = 10; const recipients = scopes ? scopes : await this.keyStore.getAccounts(); - // A map of never-before-seen logs going from recipient address to logs + // A map of logs going from recipient address to logs. Note that the logs might have been processed before as we + // due to us having a sliding window that "looks back" for logs as well. const newLogsMap = new Map(); - const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); + // const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); for (const recipient of recipients) { const logs: TxScopedL2Log[] = []; // Ideally this algorithm would be implemented in noir, exposing its building blocks as oracles. @@ -452,10 +448,13 @@ export class SimulatorOracle implements DBOracle { // so we might have missed some logs. For these reasons, we have to look both back and ahead of // the stored index. + // As we iterate we store the largest index we have seen for a given secret to later on store it in the db. + const newLargestIndexMapToStore: { [k: string]: number } = {}; + // The initial/unmodified indexes of the secrets stored in a key-value map where key is the app tagging secret. const initialIndexesMap = getInitialIndexesMap(secrets); - const secretsAndWindows = secrets.map(secret => { + let secretsAndWindows = secrets.map(secret => { return { appTaggingSecret: secret.appTaggingSecret, leftMostIndex: secret.index - WINDOW_HALF_SIZE, @@ -469,9 +468,9 @@ export class SimulatorOracle implements DBOracle { secret.computeSiloedTag(recipient, contractAddress), ); - // If we find a log with an index greater than the one stored in the db, we want to fetch logs for tags that - // correspond to the difference of the window sets and then we want to store the new largest index in the db. - const newLargestIndexMap: { [k: string]: number } = {}; + // We store the new largest indexes we find in the iteration in the following map to later on construct + // a new set of secrets and windows to fetch logs for. + const newLargestIndexMapForIteration: { [k: string]: number } = {}; // Fetch the logs for the tags and iterate over them const logsByTags = await this.aztecNode.getLogsByTags(tagsForTheWholeWindow); @@ -495,19 +494,51 @@ export class SimulatorOracle implements DBOracle { if ( secretCorrespondingToLog.index > initialIndex && - (!newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()] || + (!newLargestIndexMapForIteration[secretCorrespondingToLog.appTaggingSecret.toString()] || secretCorrespondingToLog.index > - newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()]) + newLargestIndexMapForIteration[secretCorrespondingToLog.appTaggingSecret.toString()]) ) { // We have found a new largest index so we store it for later processing (storing it in the db + fetching // the difference of the window sets of current and the next iteration) - newLargestIndexMap[secretCorrespondingToLog.appTaggingSecret.toString()] = secretCorrespondingToLog.index; + newLargestIndexMapForIteration[secretCorrespondingToLog.appTaggingSecret.toString()] = + secretCorrespondingToLog.index; } } }); - // Now based on the new largest indexes we found, I will construct a new secrets and windows + // Now based on the new largest indexes we found, I will construct a new secrets and windows set to fetch logs + // for. Note that we it's very unlikely that a new log from the current window would appear between + // the iterations so we fetch the logs only for the difference of the window sets. + const newSecretsAndWindows = []; + for (const [appTaggingSecret, newIndex] of Object.entries(newLargestIndexMapForIteration)) { + const secret = secrets.find(secret => secret.appTaggingSecret.toString() === appTaggingSecret); + if (secret) { + newSecretsAndWindows.push({ + appTaggingSecret: secret.appTaggingSecret, + // We set the left most index to the new largest index + 1 to avoid fetching the same logs again + leftMostIndex: newIndex + 1, + rightMostIndex: newIndex + WINDOW_HALF_SIZE, + }); + + // We store the new largest index in the map to store it in the db. + newLargestIndexMapToStore[appTaggingSecret] = newIndex; + } else { + throw new Error( + `Secret not found for appTaggingSecret ${appTaggingSecret}. This is a bug as it should never happen!`, + ); + } + } + + // Now we set the new secrets and windows and proceed to the next iteration + secretsAndWindows = newSecretsAndWindows; } + + // At this point we have processed all the logs for the recipient so we store the new largest indexes in the db + await this.db.setTaggingSecretsIndexesAsRecipient( + Object.entries(newLargestIndexMapToStore).map( + ([appTaggingSecret, index]) => new IndexedTaggingSecret(Fr.fromHexString(appTaggingSecret), index), + ), + ); } return newLogsMap; } diff --git a/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts b/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts index dab351652028..82759eaf038f 100644 --- a/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts +++ b/yarn-project/pxe/src/simulator_oracle/tagging_utils.ts @@ -1,23 +1,4 @@ -import { AztecAddress, Fr, IndexedTaggingSecret } from '@aztec/circuits.js'; - -/** - * Gets indexed tagging secrets with leftmost indexes. - * @param indexedTaggingSecrets - The indexed tagging secrets to get the leftmost indexed tagging secrets from. - * @param windowHalfSize- The half size of the window to slide over the tagging secret indexes. - * @returns The leftmost indexed tagging secrets. - */ -export function getLeftMostIndexedTaggingSecrets( - indexedTaggingSecrets: IndexedTaggingSecret[], - windowHalfSize: number, -): IndexedTaggingSecret[] { - return indexedTaggingSecrets.map( - indexedTaggingSecret => - new IndexedTaggingSecret( - indexedTaggingSecret.appTaggingSecret, - Math.max(0, indexedTaggingSecret.index - windowHalfSize), - ), - ); -} +import { type Fr, IndexedTaggingSecret } from '@aztec/circuits.js'; export function getIndexedTaggingSecretsForTheWindow( secretsAndWindows: { appTaggingSecret: Fr; leftMostIndex: number; rightMostIndex: number }[], @@ -31,25 +12,6 @@ export function getIndexedTaggingSecretsForTheWindow( return secrets; } -/** - * Creates a map from app tagging secret to rightmost index. - * @param indexedTaggingSecrets - The indexed tagging secrets to get the rightmost indexes from. - * @param windowHalfSize- The half size of the window to slide over the tagging secret indexes. - * @returns The map from app tagging secret to rightmost index. - */ -export function getRightMostIndexes( - indexedTaggingSecrets: IndexedTaggingSecret[], - windowHalfSize: number, -): { [k: string]: number } { - const rightMostIndexes: { [k: string]: number } = {}; - - for (const indexedTaggingSecret of indexedTaggingSecrets) { - rightMostIndexes[indexedTaggingSecret.appTaggingSecret.toString()] = indexedTaggingSecret.index + windowHalfSize; - } - - return rightMostIndexes; -} - /** * Creates a map from app tagging secret to initial index. * @param indexedTaggingSecrets - The indexed tagging secrets to get the initial indexes from. From 47095b91371f3ba51febc26a9d07be292e348dc5 Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 13 Dec 2024 19:38:53 +0000 Subject: [PATCH 05/10] WIP --- yarn-project/pxe/src/simulator_oracle/index.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index a8f5cd235aba..6b5f6399102f 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -476,13 +476,6 @@ export class SimulatorOracle implements DBOracle { const logsByTags = await this.aztecNode.getLogsByTags(tagsForTheWholeWindow); logsByTags.forEach((logsByTag, logIndex) => { - // What do I even want to do here? - // I want to check if the log for the given tag exists - // --> if it does, I want to: - // - add it to the logs to process - // - if it corresponds to an index > current one in the db - // - I want update it in the db - // - I want to fetch the difference of the window sets if (logsByTag.length > 0) { // The logs for the given tag exist so I add them for later processing logs.push(...logsByTag); From 7a8cb24c0e839b28c65b1599fe4abbc813abfc1f Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 13 Dec 2024 19:56:33 +0000 Subject: [PATCH 06/10] WIP --- .../src/structs/indexed_tagging_secret.ts | 6 +++++- yarn-project/pxe/src/simulator_oracle/index.ts | 18 +++++++++++------- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/yarn-project/circuits.js/src/structs/indexed_tagging_secret.ts b/yarn-project/circuits.js/src/structs/indexed_tagging_secret.ts index 4a884180eb8f..37b98d62282d 100644 --- a/yarn-project/circuits.js/src/structs/indexed_tagging_secret.ts +++ b/yarn-project/circuits.js/src/structs/indexed_tagging_secret.ts @@ -3,7 +3,11 @@ import { poseidon2Hash } from '@aztec/foundation/crypto'; import { Fr } from '@aztec/foundation/fields'; export class IndexedTaggingSecret { - constructor(public appTaggingSecret: Fr, public index: number) {} + constructor(public appTaggingSecret: Fr, public index: number) { + if (index < 0) { + throw new Error('IndexedTaggingSecret index out of bounds'); + } + } toFields(): Fr[] { return [this.appTaggingSecret, new Fr(this.index)]; diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 6b5f6399102f..63699a5446c0 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -425,12 +425,13 @@ export class SimulatorOracle implements DBOracle { const WINDOW_HALF_SIZE = 10; const recipients = scopes ? scopes : await this.keyStore.getAccounts(); - // A map of logs going from recipient address to logs. Note that the logs might have been processed before as we - // due to us having a sliding window that "looks back" for logs as well. - const newLogsMap = new Map(); + // A map of logs going from recipient address to logs. Note that the logs might have been processed before + // due to us having a sliding window that "looks back" for logs as well. (We look back as there is no guarantee + // that a logs will be received ordered by a given tax index and that the tags won't be reused). + const logsMap = new Map(); // const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); for (const recipient of recipients) { - const logs: TxScopedL2Log[] = []; + const logsForRecipient: TxScopedL2Log[] = []; // Ideally this algorithm would be implemented in noir, exposing its building blocks as oracles. // However it is impossible at the moment due to the language not supporting nested slices. // This nesting is necessary because for a given set of tags we don't @@ -457,7 +458,7 @@ export class SimulatorOracle implements DBOracle { let secretsAndWindows = secrets.map(secret => { return { appTaggingSecret: secret.appTaggingSecret, - leftMostIndex: secret.index - WINDOW_HALF_SIZE, + leftMostIndex: Math.max(0, secret.index - WINDOW_HALF_SIZE), rightMostIndex: secret.index + WINDOW_HALF_SIZE, }; }); @@ -478,7 +479,7 @@ export class SimulatorOracle implements DBOracle { logsByTags.forEach((logsByTag, logIndex) => { if (logsByTag.length > 0) { // The logs for the given tag exist so I add them for later processing - logs.push(...logsByTag); + logsForRecipient.push(...logsByTag); // I fetch the indexed tagging secret corresponding to the log as I need that to evaluate whether // a new largest index have been found. @@ -526,6 +527,9 @@ export class SimulatorOracle implements DBOracle { secretsAndWindows = newSecretsAndWindows; } + // We filter the logs by block number and store them in the map + logsMap.set(recipient.toString(), logsForRecipient.filter(log => log.blockNumber <= maxBlockNumber)); + // At this point we have processed all the logs for the recipient so we store the new largest indexes in the db await this.db.setTaggingSecretsIndexesAsRecipient( Object.entries(newLargestIndexMapToStore).map( @@ -533,7 +537,7 @@ export class SimulatorOracle implements DBOracle { ), ); } - return newLogsMap; + return logsMap; } /** From 598c3ec3cac144ee18e7067fee31e2d7cb48a9af Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 13 Dec 2024 21:39:54 +0000 Subject: [PATCH 07/10] newLargestInde issue fix --- yarn-project/pxe/src/simulator_oracle/index.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 63699a5446c0..58caf75dcd74 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -487,16 +487,18 @@ export class SimulatorOracle implements DBOracle { const initialIndex = initialIndexesMap[secretCorrespondingToLog.appTaggingSecret.toString()]; if ( - secretCorrespondingToLog.index > initialIndex && - (!newLargestIndexMapForIteration[secretCorrespondingToLog.appTaggingSecret.toString()] || - secretCorrespondingToLog.index > + secretCorrespondingToLog.index >= initialIndex && + (newLargestIndexMapForIteration[secretCorrespondingToLog.appTaggingSecret.toString()] === undefined || + secretCorrespondingToLog.index >= newLargestIndexMapForIteration[secretCorrespondingToLog.appTaggingSecret.toString()]) ) { // We have found a new largest index so we store it for later processing (storing it in the db + fetching // the difference of the window sets of current and the next iteration) newLargestIndexMapForIteration[secretCorrespondingToLog.appTaggingSecret.toString()] = - secretCorrespondingToLog.index; + secretCorrespondingToLog.index + 1; } + + } }); @@ -510,7 +512,7 @@ export class SimulatorOracle implements DBOracle { newSecretsAndWindows.push({ appTaggingSecret: secret.appTaggingSecret, // We set the left most index to the new largest index + 1 to avoid fetching the same logs again - leftMostIndex: newIndex + 1, + leftMostIndex: newIndex, rightMostIndex: newIndex + WINDOW_HALF_SIZE, }); @@ -528,7 +530,10 @@ export class SimulatorOracle implements DBOracle { } // We filter the logs by block number and store them in the map - logsMap.set(recipient.toString(), logsForRecipient.filter(log => log.blockNumber <= maxBlockNumber)); + logsMap.set( + recipient.toString(), + logsForRecipient.filter(log => log.blockNumber <= maxBlockNumber), + ); // At this point we have processed all the logs for the recipient so we store the new largest indexes in the db await this.db.setTaggingSecretsIndexesAsRecipient( From 42750b14eed1e07e1003069ce35a6e1a59e87e7b Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 13 Dec 2024 21:53:30 +0000 Subject: [PATCH 08/10] docs cleanup --- .../pxe/src/simulator_oracle/index.ts | 78 +++++++++++-------- 1 file changed, 45 insertions(+), 33 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 58caf75dcd74..9a04f4cdae7a 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -424,37 +424,32 @@ export class SimulatorOracle implements DBOracle { // Half the size of the window we slide over the tagging secret indexes. const WINDOW_HALF_SIZE = 10; + // Ideally this algorithm would be implemented in noir, exposing its building blocks as oracles. + // However it is impossible at the moment due to the language not supporting nested slices. + // This nesting is necessary because for a given set of tags we don't + // know how many logs we will get back. Furthermore, these logs are of undetermined + // length, since we don't really know the note they correspond to until we decrypt them. + const recipients = scopes ? scopes : await this.keyStore.getAccounts(); // A map of logs going from recipient address to logs. Note that the logs might have been processed before // due to us having a sliding window that "looks back" for logs as well. (We look back as there is no guarantee // that a logs will be received ordered by a given tax index and that the tags won't be reused). const logsMap = new Map(); - // const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); + const contractName = await this.contractDataOracle.getDebugContractName(contractAddress); for (const recipient of recipients) { const logsForRecipient: TxScopedL2Log[] = []; - // Ideally this algorithm would be implemented in noir, exposing its building blocks as oracles. - // However it is impossible at the moment due to the language not supporting nested slices. - // This nesting is necessary because for a given set of tags we don't - // know how many logs we will get back. Furthermore, these logs are of undetermined - // length, since we don't really know the note they correspond to until we decrypt them. - // 1. Get all the secrets for the recipient and sender pairs (#9365) + // Get all the secrets for the recipient and sender pairs (#9365) const secrets = await this.#getIndexedTaggingSecretsForContacts(contractAddress, recipient); - // 1.1 Set up a sliding window with an offset. Chances are the sender might have messed up - // and inadvertently incremented their index without us getting any logs (for example, in case - // of a revert). If we stopped looking for logs the first time we don't receive any logs for a tag, - // we might never receive anything from that sender again. - // Also there's a possibility that we have advanced our index, but the sender has reused it, - // so we might have missed some logs. For these reasons, we have to look both back and ahead of - // the stored index. - - // As we iterate we store the largest index we have seen for a given secret to later on store it in the db. - const newLargestIndexMapToStore: { [k: string]: number } = {}; - - // The initial/unmodified indexes of the secrets stored in a key-value map where key is the app tagging secret. - const initialIndexesMap = getInitialIndexesMap(secrets); - + // We fetch logs for a window of indexes in a range: + // . + // + // We use this window approach because it could happen that a sender might have messed up and inadvertently + // incremented their index without us getting any logs (for example, in case of a revert). If we stopped looking + // for logs the first time we don't receive any logs for a tag, we might never receive anything from that sender again. + // Also there's a possibility that we have advanced our index, but the sender has reused it, so we might have missed + // some logs. For these reasons, we have to look both back and ahead of the stored index. let secretsAndWindows = secrets.map(secret => { return { appTaggingSecret: secret.appTaggingSecret, @@ -463,6 +458,12 @@ export class SimulatorOracle implements DBOracle { }; }); + // As we iterate we store the largest index we have seen for a given secret to later on store it in the db. + const newLargestIndexMapToStore: { [k: string]: number } = {}; + + // The initial/unmodified indexes of the secrets stored in a key-value map where key is the app tagging secret. + const initialIndexesMap = getInitialIndexesMap(secrets); + while (secretsAndWindows.length > 0) { const secretsForTheWholeWindow = getIndexedTaggingSecretsForTheWindow(secretsAndWindows); const tagsForTheWholeWindow = secretsForTheWholeWindow.map(secret => @@ -478,14 +479,21 @@ export class SimulatorOracle implements DBOracle { logsByTags.forEach((logsByTag, logIndex) => { if (logsByTag.length > 0) { - // The logs for the given tag exist so I add them for later processing + // The logs for the given tag exist so we store them for later processing logsForRecipient.push(...logsByTag); - // I fetch the indexed tagging secret corresponding to the log as I need that to evaluate whether + // We the indexed tagging secret corresponding to the log as I need that to evaluate whether // a new largest index have been found. const secretCorrespondingToLog = secretsForTheWholeWindow[logIndex]; const initialIndex = initialIndexesMap[secretCorrespondingToLog.appTaggingSecret.toString()]; + this.log.debug(`Found ${logsByTag.length} logs as recipient ${recipient}`, { + recipient, + secret: secretCorrespondingToLog.appTaggingSecret, + contractName, + contractAddress, + }); + if ( secretCorrespondingToLog.index >= initialIndex && (newLargestIndexMapForIteration[secretCorrespondingToLog.appTaggingSecret.toString()] === undefined || @@ -496,27 +504,31 @@ export class SimulatorOracle implements DBOracle { // the difference of the window sets of current and the next iteration) newLargestIndexMapForIteration[secretCorrespondingToLog.appTaggingSecret.toString()] = secretCorrespondingToLog.index + 1; - } - + this.log.debug( + `Incrementing index to ${ + secretCorrespondingToLog.index + 1 + } at contract ${contractName}(${contractAddress})`, + ); + } } }); - // Now based on the new largest indexes we found, I will construct a new secrets and windows set to fetch logs - // for. Note that we it's very unlikely that a new log from the current window would appear between - // the iterations so we fetch the logs only for the difference of the window sets. + // Now based on the new largest indexes we found, we will construct a new secrets and windows set to fetch logs + // for. Note that it's very unlikely that a new log from the current window would appear between the iterations + // so we fetch the logs only for the difference of the window sets. const newSecretsAndWindows = []; for (const [appTaggingSecret, newIndex] of Object.entries(newLargestIndexMapForIteration)) { const secret = secrets.find(secret => secret.appTaggingSecret.toString() === appTaggingSecret); if (secret) { newSecretsAndWindows.push({ appTaggingSecret: secret.appTaggingSecret, - // We set the left most index to the new largest index + 1 to avoid fetching the same logs again + // We set the left most index to the new index to avoid fetching the same logs again leftMostIndex: newIndex, rightMostIndex: newIndex + WINDOW_HALF_SIZE, }); - // We store the new largest index in the map to store it in the db. + // We store the new largest index in the map to later store it in the db. newLargestIndexMapToStore[appTaggingSecret] = newIndex; } else { throw new Error( @@ -525,17 +537,17 @@ export class SimulatorOracle implements DBOracle { } } - // Now we set the new secrets and windows and proceed to the next iteration + // Now we set the new secrets and windows and proceed to the next iteration. secretsAndWindows = newSecretsAndWindows; } - // We filter the logs by block number and store them in the map + // We filter the logs by block number and store them in the map. logsMap.set( recipient.toString(), logsForRecipient.filter(log => log.blockNumber <= maxBlockNumber), ); - // At this point we have processed all the logs for the recipient so we store the new largest indexes in the db + // At this point we have processed all the logs for the recipient so we store the new largest indexes in the db. await this.db.setTaggingSecretsIndexesAsRecipient( Object.entries(newLargestIndexMapToStore).map( ([appTaggingSecret, index]) => new IndexedTaggingSecret(Fr.fromHexString(appTaggingSecret), index), From 64e72c5a9dc8d418760e4c1a1fcaa5ee4681427b Mon Sep 17 00:00:00 2001 From: benesjan Date: Fri, 13 Dec 2024 21:55:49 +0000 Subject: [PATCH 09/10] fixed simulator test --- .../simulator_oracle/simulator_oracle.test.ts | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index 1e88eaab3634..b5bfdd0adb55 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -233,7 +233,8 @@ describe('Simulator oracle', () => { const senderOffset = 0; generateMockLogs(senderOffset); const syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3); - // We expect to have all logs intended for the recipient, one per sender + 1 with a duplicated tag for the first one + half of the logs for the second index + // We expect to have all logs intended for the recipient, one per sender + 1 with a duplicated tag for the first + // one + half of the logs for the second index expect(syncedLogs.get(recipient.address.toString())).toHaveLength(NUM_SENDERS + 1 + NUM_SENDERS / 2); // Recompute the secrets (as recipient) to ensure indexes are updated @@ -252,9 +253,9 @@ describe('Simulator oracle', () => { expect(indexes).toHaveLength(NUM_SENDERS); expect(indexes).toEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]); - // We should have called the node 12 times: - // 2 times with logs (sliding the window) + 10 times with no results (window size) - expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2 + SENDER_OFFSET_WINDOW_SIZE); + // We should have called the node 2 times: + // 2 times: first time during initial request, second time after pushing the edge of the window once + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2); }); it('should sync tagged logs as senders', async () => { @@ -332,9 +333,9 @@ describe('Simulator oracle', () => { expect(indexes).toHaveLength(NUM_SENDERS); expect(indexes).toEqual([6, 6, 6, 6, 6, 7, 7, 7, 7, 7]); - // We should have called the node 17 times: - // 5 times with no results (sender offset) + 2 times with logs (sliding the window) + 10 times with no results (window size) - expect(aztecNode.getLogsByTags.mock.calls.length).toBe(5 + 2 + SENDER_OFFSET_WINDOW_SIZE); + // We should have called the node 2 times: + // 2 times: first time during initial request, second time after pushing the edge of the window once + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2); }); it("should sync tagged logs for which indexes are not updated if they're inside the window", async () => { @@ -358,16 +359,16 @@ describe('Simulator oracle', () => { expect(syncedLogs.get(recipient.address.toString())).toHaveLength(NUM_SENDERS + 1 + NUM_SENDERS / 2); // First sender should have 2 logs, but keep index 2 since they were built using the same tag - // Next 4 senders hould also have index 2 = offset + 1 + // Next 4 senders should also have index 2 = offset + 1 // Last 5 senders should have index 3 = offset + 2 const indexes = await database.getTaggingSecretsIndexesAsRecipient(secrets); expect(indexes).toHaveLength(NUM_SENDERS); expect(indexes).toEqual([2, 2, 2, 2, 2, 3, 3, 3, 3, 3]); - // We should have called the node 13 times: - // 1 time without logs + 2 times with logs (sliding the window) + 10 times with no results (window size) - expect(aztecNode.getLogsByTags.mock.calls.length).toBe(3 + SENDER_OFFSET_WINDOW_SIZE); + // We should have called the node 2 times: + // first time during initial request, second time after pushing the edge of the window once + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2); }); it("should not sync tagged logs for which indexes are not updated if they're outside the window", async () => { @@ -396,9 +397,8 @@ describe('Simulator oracle', () => { expect(indexes).toHaveLength(NUM_SENDERS); expect(indexes).toEqual([11, 11, 11, 11, 11, 11, 11, 11, 11, 11]); - // We should have called the node SENDER_OFFSET_WINDOW_SIZE + 1 (with logs) + SENDER_OFFSET_WINDOW_SIZE: - // Once for index 1 (NUM_SENDERS/2 logs) + 2 times the sliding window (no logs each time) - expect(aztecNode.getLogsByTags.mock.calls.length).toBe(1 + 2 * SENDER_OFFSET_WINDOW_SIZE); + // We should have called the node once and that is only for the first window + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(1); }); it('should sync tagged logs from scratch after a DB wipe', async () => { @@ -420,8 +420,9 @@ describe('Simulator oracle', () => { // No logs should be synced since we start from index 2 = 12 - window_size expect(syncedLogs.get(recipient.address.toString())).toHaveLength(0); - // We should have called the node 21 times (window size + current_index + window size) - expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2 * SENDER_OFFSET_WINDOW_SIZE + 1); + // Since no logs were synced, window edge hash not been pushed and for this reason we should have called + // the node only once for the initial window + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(1); aztecNode.getLogsByTags.mockClear(); @@ -431,16 +432,16 @@ describe('Simulator oracle', () => { syncedLogs = await simulatorOracle.syncTaggedLogs(contractAddress, 3); // First sender should have 2 logs, but keep index 1 since they were built using the same tag - // Next 4 senders hould also have index 1 = offset + 1 + // Next 4 senders should also have index 1 = offset + 1 // Last 5 senders should have index 2 = offset + 2 const indexes = await database.getTaggingSecretsIndexesAsRecipient(secrets); expect(indexes).toHaveLength(NUM_SENDERS); expect(indexes).toEqual([1, 1, 1, 1, 1, 2, 2, 2, 2, 2]); - // We should have called the node 12 times: - // 2 times with logs (sliding the window) + 10 times with no results (window size) - expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2 + SENDER_OFFSET_WINDOW_SIZE); + // We should have called the node 2 times: + // first time during initial request, second time after pushing the edge of the window once + expect(aztecNode.getLogsByTags.mock.calls.length).toBe(2); }); it('should not sync tagged logs with a blockNumber > maxBlockNumber', async () => { From 148e6de0d8e8424557235126d0e5f31357ac3986 Mon Sep 17 00:00:00 2001 From: benesjan Date: Mon, 16 Dec 2024 14:57:27 +0000 Subject: [PATCH 10/10] comment fix --- yarn-project/pxe/src/simulator_oracle/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 9a04f4cdae7a..7fc05853ed6d 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -482,7 +482,7 @@ export class SimulatorOracle implements DBOracle { // The logs for the given tag exist so we store them for later processing logsForRecipient.push(...logsByTag); - // We the indexed tagging secret corresponding to the log as I need that to evaluate whether + // We retrieve the indexed tagging secret corresponding to the log as I need that to evaluate whether // a new largest index have been found. const secretCorrespondingToLog = secretsForTheWholeWindow[logIndex]; const initialIndex = initialIndexesMap[secretCorrespondingToLog.appTaggingSecret.toString()];