From 8b964a605bb20170f8db6843ed1a1b094ef62148 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 14 Aug 2024 10:07:20 +0000 Subject: [PATCH 1/7] Fixed TF --- yarn-project/aztec/terraform/prover-node/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf index 36e2cd02291e..3631dc4ffee8 100644 --- a/yarn-project/aztec/terraform/prover-node/main.tf +++ b/yarn-project/aztec/terraform/prover-node/main.tf @@ -300,7 +300,7 @@ resource "aws_ecs_service" "aztec-prover-node" { # Configure ALB to route /aztec-prover-node to server. resource "aws_alb_target_group" "aztec-prover-node-http" { count = local.node_count - name = "${var.DEPLOY_TAG}-node-${count.index + 1}-http-target" + name = "${var.DEPLOY_TAG}-prover-node-${count.index + 1}-http-target" port = 80 protocol = "HTTP" target_type = "ip" From 5e48887c34d242a46275c59eeccdc8fa6c162a80 Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 14 Aug 2024 10:36:03 +0000 Subject: [PATCH 2/7] More TF fixes --- yarn-project/aztec/terraform/prover-node/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf index 3631dc4ffee8..e5e340b5d307 100644 --- a/yarn-project/aztec/terraform/prover-node/main.tf +++ b/yarn-project/aztec/terraform/prover-node/main.tf @@ -300,7 +300,7 @@ resource "aws_ecs_service" "aztec-prover-node" { # Configure ALB to route /aztec-prover-node to server. resource "aws_alb_target_group" "aztec-prover-node-http" { count = local.node_count - name = "${var.DEPLOY_TAG}-prover-node-${count.index + 1}-http-target" + name = "${var.DEPLOY_TAG}-prover-${count.index + 1}-target" port = 80 protocol = "HTTP" target_type = "ip" From 197e68d4c3a2518ba026e37b495dea50d8ed551b Mon Sep 17 00:00:00 2001 From: PhilWindle Date: Wed, 14 Aug 2024 11:38:40 +0000 Subject: [PATCH 3/7] TF fix --- yarn-project/aztec/terraform/prover-node/main.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf index e5e340b5d307..b6579790971a 100644 --- a/yarn-project/aztec/terraform/prover-node/main.tf +++ b/yarn-project/aztec/terraform/prover-node/main.tf @@ -182,7 +182,7 @@ resource "aws_ecs_task_definition" "aztec-prover-node" { environment = [ // General { name = "NODE_ENV", value = "production" }, - { name = "LOG_LEVEL", value = "info" }, + { name = "LOG_LEVEL", value = "debug" }, { name = "DEBUG", value = "aztec:*,-json-rpc:json_proxy:*,-aztec:avm_simulator:*" }, { name = "DEPLOY_TAG", value = var.DEPLOY_TAG }, { name = "NETWORK_NAME", value = "${var.DEPLOY_TAG}" }, @@ -208,8 +208,8 @@ resource "aws_ecs_task_definition" "aztec-prover-node" { { name = "PROVER_AGENT_ENABLED", value = "false" }, { name = "PROVER_AGENT_CONCURRENCY", value = "0" }, { name = "PROVER_REAL_PROOFS", value = tostring(var.PROVING_ENABLED) }, - { name = "BB_WORKING_DIRECTORY", value = "${local.data_dir}/node_${count.index + 1}/temp" }, - { name = "ACVM_WORKING_DIRECTORY", value = "${local.data_dir}/node_${count.index + 1}/temp" }, + { name = "BB_WORKING_DIRECTORY", value = "${local.data_dir}/prover_node_${count.index + 1}/temp" }, + { name = "ACVM_WORKING_DIRECTORY", value = "${local.data_dir}/prover_node_${count.index + 1}/temp" }, // Metrics { name = "OTEL_EXPORTER_OTLP_ENDPOINT", value = "http://aztec-otel.local:4318" }, From 5a0cc08287766953fb8ef05008053513ffe12149 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 14 Aug 2024 10:41:37 -0300 Subject: [PATCH 4/7] fix: Use data dir for lmdb forks In case tmpdir() is not available, rely on the data dir for creating forks of the store. --- yarn-project/kv-store/src/lmdb/store.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts index cb6ee87d7d77..dfeca64fd00d 100644 --- a/yarn-project/kv-store/src/lmdb/store.ts +++ b/yarn-project/kv-store/src/lmdb/store.ts @@ -3,7 +3,7 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { mkdtemp } from 'fs/promises'; import { type Database, type Key, type RootDatabase, open } from 'lmdb'; import { tmpdir } from 'os'; -import { join } from 'path'; +import { dirname, join } from 'path'; import { type AztecArray } from '../interfaces/array.js'; import { type AztecCounter } from '../interfaces/counter.js'; @@ -25,7 +25,7 @@ export class AztecLmdbStore implements AztecKVStore { #data: Database; #multiMapData: Database; - constructor(rootDb: RootDatabase, public readonly isEphemeral: boolean) { + constructor(rootDb: RootDatabase, public readonly isEphemeral: boolean, private path?: string) { this.#rootDb = rootDb; // big bucket to store all the data @@ -61,7 +61,7 @@ export class AztecLmdbStore implements AztecKVStore { ): AztecLmdbStore { log.info(`Opening LMDB database at ${path || 'temporary location'}`); const rootDb = open({ path, noSync: ephemeral }); - return new AztecLmdbStore(rootDb, ephemeral); + return new AztecLmdbStore(rootDb, ephemeral, path); } /** @@ -69,10 +69,11 @@ export class AztecLmdbStore implements AztecKVStore { * @returns A new AztecLmdbStore. */ async fork() { - const forkPath = join(await mkdtemp(join(tmpdir(), 'aztec-store-fork-')), 'root.mdb'); + const baseDir = this.path ? dirname(this.path) : tmpdir(); + const forkPath = join(await mkdtemp(join(baseDir, 'aztec-store-fork-')), 'root.mdb'); await this.#rootDb.backup(forkPath, false); const forkDb = open(forkPath, { noSync: this.isEphemeral }); - return new AztecLmdbStore(forkDb, this.isEphemeral); + return new AztecLmdbStore(forkDb, this.isEphemeral, forkPath); } /** From d52a647a1843035957b24f9dd79c21f4eee33e5b Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 14 Aug 2024 12:04:38 -0300 Subject: [PATCH 5/7] chore: Fork logs and prover job catch Add logs to forking and a catch to the prover job. --- yarn-project/kv-store/src/lmdb/store.ts | 6 ++- yarn-project/prover-node/src/prover-node.ts | 50 ++++++++++++--------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts index dfeca64fd00d..75cd4b65c066 100644 --- a/yarn-project/kv-store/src/lmdb/store.ts +++ b/yarn-project/kv-store/src/lmdb/store.ts @@ -24,6 +24,7 @@ export class AztecLmdbStore implements AztecKVStore { #rootDb: RootDatabase; #data: Database; #multiMapData: Database; + #log = createDebugLogger('aztec:kv-store:lmdb'); constructor(rootDb: RootDatabase, public readonly isEphemeral: boolean, private path?: string) { this.#rootDb = rootDb; @@ -59,7 +60,7 @@ export class AztecLmdbStore implements AztecKVStore { ephemeral: boolean = false, log = createDebugLogger('aztec:kv-store:lmdb'), ): AztecLmdbStore { - log.info(`Opening LMDB database at ${path || 'temporary location'}`); + log.verbose(`Opening LMDB database at ${path || 'temporary location'}`); const rootDb = open({ path, noSync: ephemeral }); return new AztecLmdbStore(rootDb, ephemeral, path); } @@ -70,9 +71,12 @@ export class AztecLmdbStore implements AztecKVStore { */ async fork() { const baseDir = this.path ? dirname(this.path) : tmpdir(); + this.#log.debug(`Forking store with basedir ${baseDir}`); const forkPath = join(await mkdtemp(join(baseDir, 'aztec-store-fork-')), 'root.mdb'); + this.#log.verbose(`Forking store to ${forkPath}`); await this.#rootDb.backup(forkPath, false); const forkDb = open(forkPath, { noSync: this.isEphemeral }); + this.#log.debug(`Forked store at ${forkPath} opened successfully`); return new AztecLmdbStore(forkDb, this.isEphemeral, forkPath); } diff --git a/yarn-project/prover-node/src/prover-node.ts b/yarn-project/prover-node/src/prover-node.ts index ca4b30429131..b5d535c227fe 100644 --- a/yarn-project/prover-node/src/prover-node.ts +++ b/yarn-project/prover-node/src/prover-node.ts @@ -63,28 +63,36 @@ export class ProverNode { * Checks whether there are new blocks to prove, proves them, and submits them. */ protected async work() { - if (this.options.disableAutomaticProving) { - return; + try { + if (this.options.disableAutomaticProving) { + return; + } + + const [latestBlockNumber, latestProvenBlockNumber] = await Promise.all([ + this.l2BlockSource.getBlockNumber(), + this.l2BlockSource.getProvenBlockNumber(), + ]); + + // Consider both the latest block we are proving and the last block proven on the chain + const latestBlockBeingProven = this.latestBlockWeAreProving ?? 0; + const latestProven = Math.max(latestBlockBeingProven, latestProvenBlockNumber); + if (latestProven >= latestBlockNumber) { + this.log.debug(`No new blocks to prove`, { + latestBlockNumber, + latestProvenBlockNumber, + latestBlockBeingProven, + }); + return; + } + + const fromBlock = latestProven + 1; + const toBlock = fromBlock; // We only prove one block at a time for now + + await this.startProof(fromBlock, toBlock); + this.latestBlockWeAreProving = toBlock; + } catch (err) { + this.log.error(`Error in prover node work`, err); } - - const [latestBlockNumber, latestProvenBlockNumber] = await Promise.all([ - this.l2BlockSource.getBlockNumber(), - this.l2BlockSource.getProvenBlockNumber(), - ]); - - // Consider both the latest block we are proving and the last block proven on the chain - const latestBlockBeingProven = this.latestBlockWeAreProving ?? 0; - const latestProven = Math.max(latestBlockBeingProven, latestProvenBlockNumber); - if (latestProven >= latestBlockNumber) { - this.log.debug(`No new blocks to prove`, { latestBlockNumber, latestProvenBlockNumber, latestBlockBeingProven }); - return; - } - - const fromBlock = latestProven + 1; - const toBlock = fromBlock; // We only prove one block at a time for now - - await this.startProof(fromBlock, toBlock); - this.latestBlockWeAreProving = toBlock; } /** From 79c7af13d24d4d17171476a477e84beaf705d5ff Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 14 Aug 2024 10:58:27 -0300 Subject: [PATCH 6/7] chore: Enable execute command on aws ecs services This allows executing commands from the AWS CLI directly on the containers for troubleshooting. --- yarn-project/aztec-faucet/terraform/main.tf | 1 + yarn-project/aztec/terraform/bot/main.tf | 1 + yarn-project/aztec/terraform/node/main.tf | 1 + yarn-project/aztec/terraform/prover-node/main.tf | 2 +- yarn-project/aztec/terraform/prover/main.tf | 1 + yarn-project/aztec/terraform/pxe/main.tf | 1 + 6 files changed, 6 insertions(+), 1 deletion(-) diff --git a/yarn-project/aztec-faucet/terraform/main.tf b/yarn-project/aztec-faucet/terraform/main.tf index 691401b4ef18..2326d9e3e4ef 100644 --- a/yarn-project/aztec-faucet/terraform/main.tf +++ b/yarn-project/aztec-faucet/terraform/main.tf @@ -171,6 +171,7 @@ resource "aws_ecs_service" "aztec-faucet" { deployment_maximum_percent = 100 deployment_minimum_healthy_percent = 0 platform_version = "1.4.0" + enable_execute_command = true network_configuration { subnets = [ diff --git a/yarn-project/aztec/terraform/bot/main.tf b/yarn-project/aztec/terraform/bot/main.tf index fee1a564b0ff..4871cd18eef7 100644 --- a/yarn-project/aztec/terraform/bot/main.tf +++ b/yarn-project/aztec/terraform/bot/main.tf @@ -192,6 +192,7 @@ resource "aws_ecs_service" "aztec-bot" { deployment_maximum_percent = 100 deployment_minimum_healthy_percent = 0 force_new_deployment = true + enable_execute_command = true network_configuration { subnets = [ diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf index 4bbe91821693..e7f7b040d7de 100644 --- a/yarn-project/aztec/terraform/node/main.tf +++ b/yarn-project/aztec/terraform/node/main.tf @@ -402,6 +402,7 @@ resource "aws_ecs_service" "aztec-node" { deployment_minimum_healthy_percent = 0 platform_version = "1.4.0" force_new_deployment = true + enable_execute_command = true network_configuration { diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf index b6579790971a..f4cc86600222 100644 --- a/yarn-project/aztec/terraform/prover-node/main.tf +++ b/yarn-project/aztec/terraform/prover-node/main.tf @@ -272,7 +272,7 @@ resource "aws_ecs_service" "aztec-prover-node" { deployment_minimum_healthy_percent = 0 platform_version = "1.4.0" force_new_deployment = true - + enable_execute_command = true network_configuration { assign_public_ip = true diff --git a/yarn-project/aztec/terraform/prover/main.tf b/yarn-project/aztec/terraform/prover/main.tf index 25056aa95b89..b02bf12957b9 100644 --- a/yarn-project/aztec/terraform/prover/main.tf +++ b/yarn-project/aztec/terraform/prover/main.tf @@ -307,6 +307,7 @@ resource "aws_ecs_service" "aztec-proving-agent" { desired_count = local.agents_per_prover deployment_maximum_percent = 100 deployment_minimum_healthy_percent = 0 + enable_execute_command = true #platform_version = "1.4.0" # Associate the EC2 capacity provider diff --git a/yarn-project/aztec/terraform/pxe/main.tf b/yarn-project/aztec/terraform/pxe/main.tf index 5e94191744ca..9a0908f8bfb5 100644 --- a/yarn-project/aztec/terraform/pxe/main.tf +++ b/yarn-project/aztec/terraform/pxe/main.tf @@ -179,6 +179,7 @@ resource "aws_ecs_service" "aztec-pxe" { deployment_minimum_healthy_percent = 0 platform_version = "1.4.0" force_new_deployment = true + enable_execute_command = true network_configuration { subnets = [ From 4e1148a7475f0eea13e30a0bf6f9b6fb2b3bcaee Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 14 Aug 2024 16:08:17 -0300 Subject: [PATCH 7/7] fix: Forking world state in prover-node Attempt to fix the `Error in prover node work: Error No such file or directory` error in prover-node when forking. Tested by connecting to the container, modifying the js, and running prover-node manually. --- yarn-project/kv-store/src/lmdb/store.test.ts | 2 +- yarn-project/kv-store/src/lmdb/store.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/yarn-project/kv-store/src/lmdb/store.test.ts b/yarn-project/kv-store/src/lmdb/store.test.ts index f6babd0cb677..b57091d3bc0e 100644 --- a/yarn-project/kv-store/src/lmdb/store.test.ts +++ b/yarn-project/kv-store/src/lmdb/store.test.ts @@ -17,7 +17,7 @@ describe('AztecLmdbStore', () => { }; it('forks a persistent store', async () => { - const path = join(await mkdtemp(join(tmpdir(), 'aztec-store-test-')), 'main.mdb'); + const path = await mkdtemp(join(tmpdir(), 'aztec-store-test-')); const store = AztecLmdbStore.open(path, false); await itForks(store); }); diff --git a/yarn-project/kv-store/src/lmdb/store.ts b/yarn-project/kv-store/src/lmdb/store.ts index 75cd4b65c066..348f78ba1f6a 100644 --- a/yarn-project/kv-store/src/lmdb/store.ts +++ b/yarn-project/kv-store/src/lmdb/store.ts @@ -72,7 +72,7 @@ export class AztecLmdbStore implements AztecKVStore { async fork() { const baseDir = this.path ? dirname(this.path) : tmpdir(); this.#log.debug(`Forking store with basedir ${baseDir}`); - const forkPath = join(await mkdtemp(join(baseDir, 'aztec-store-fork-')), 'root.mdb'); + const forkPath = (await mkdtemp(join(baseDir, 'aztec-store-fork-'))) + (this.isEphemeral ? '/data.mdb' : ''); this.#log.verbose(`Forking store to ${forkPath}`); await this.#rootDb.backup(forkPath, false); const forkDb = open(forkPath, { noSync: this.isEphemeral });