chore: rc-2 update & beacon node fix for sepolia

.github/workflows/network-deploy.yml

@@ -115,6 +115,9 @@ jobs:
       REGION: us-west1-a
       TF_STATE_BUCKET: aztec-terraform
       GKE_CLUSTER_CONTEXT: "gke_testnet-440309_us-west1-a_${{ inputs.cluster }}"
+      GCP_API_KEY_HEADER: "X-goog-api-key"
+      EXTERNAL_ETHEREUM_HOST: "https://json-rpc.${{ secrets.SEPOLIA_EXTERNAL_HOST }}"
+      EXTERNAL_ETHEREUM_CONSENSUS_HOST: "https://beacon.${{ secrets.SEPOLIA_EXTERNAL_HOST }}"
 
     steps:
       - name: Checkout code
@@ -150,12 +153,14 @@ jobs:
 
       - name: Grab the boot node deployment mnemonic
         id: get-mnemonic
+        if: ${{ inputs.sepolia_deployment != 'true' }}
         run: |
           echo "::add-mask::$(gcloud secrets versions access latest --secret=${{ env.DEPLOYMENT_MNEMONIC_SECRET_NAME }})"
           echo "mnemonic=$(gcloud secrets versions access latest --secret=${{ env.DEPLOYMENT_MNEMONIC_SECRET_NAME }})" >> "$GITHUB_OUTPUT"
 
       - name: Generate eth devnet config
         id: generate-eth-devnet-config
+        if: ${{ inputs.sepolia_deployment != 'true' }}
         run: |
           REPO=$(git rev-parse --show-toplevel)
 
@@ -191,8 +196,11 @@ jobs:
               -var="VALIDATOR_KEYS=${{ secrets.SEPOLIA_VALIDATOR_KEYS }}" \
               -var="BOOT_NODE_SEQ_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_BOOT_NODE_SEQ_PUBLISHER_PRIVATE_KEY }}" \
               -var="PROVER_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_PROVER_PUBLISHER_PRIVATE_KEY }}" \
-              -var="ETHEREUM_EXTERNAL_HOST=${{ secrets.SEPOLIA_EXTERNAL_HOST }}" \
-            -lock=${{ inputs.respect_tf_lock }}
+              -var="EXTERNAL_ETHEREUM_HOST=${{ env.EXTERNAL_ETHEREUM_HOST }}?key=${{ secrets.SEPOLIA_API_KEY }}" \
+              -var="EXTERNAL_ETHEREUM_CONSENSUS_HOST=${{ env.EXTERNAL_ETHEREUM_CONSENSUS_HOST }}" \
+              -var="EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY=${{ secrets.SEPOLIA_API_KEY }}" \
+              -var="EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY_HEADER=${{ env.GCP_API_KEY_HEADER }}" \
+              -lock=${{ inputs.respect_tf_lock }}
           else
             terraform destroy -auto-approve \
               -var="RELEASE_NAME=${{ env.NAMESPACE }}" \
@@ -217,7 +225,10 @@ jobs:
               -var="VALIDATOR_KEYS=${{ secrets.SEPOLIA_VALIDATOR_KEYS }}" \
               -var="BOOT_NODE_SEQ_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_BOOT_NODE_SEQ_PUBLISHER_PRIVATE_KEY }}" \
               -var="PROVER_PUBLISHER_PRIVATE_KEY=${{ secrets.SEPOLIA_PROVER_PUBLISHER_PRIVATE_KEY }}" \
-              -var="ETHEREUM_EXTERNAL_HOST=${{ secrets.SEPOLIA_EXTERNAL_HOST }}" \
+              -var="EXTERNAL_ETHEREUM_HOST=${{ env.EXTERNAL_ETHEREUM_HOST }}?key=${{ secrets.SEPOLIA_API_KEY }}" \
+              -var="EXTERNAL_ETHEREUM_CONSENSUS_HOST=${{ env.EXTERNAL_ETHEREUM_CONSENSUS_HOST }}" \
+              -var="EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY=${{ secrets.SEPOLIA_API_KEY }}" \
+              -var="EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY_HEADER=${{ env.GCP_API_KEY_HEADER }}" \
               -out=tfplan \
               -lock=${{ inputs.respect_tf_lock }}
           else

spartan/aztec-network/files/config/setup-service-addresses.sh

@@ -131,6 +131,8 @@ fi
 # Write addresses to file for sourcing
 echo "export ETHEREUM_HOST=${ETHEREUM_ADDR}" >> /shared/config/service-addresses
 echo "export L1_CONSENSUS_HOST_URL=${ETHEREUM_CONSENSUS_ADDR}" >> /shared/config/service-addresses
+echo "export L1_CONSENSUS_HOST_API_KEY=${EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY}" >> /shared/config/service-addresses
+echo "export L1_CONSENSUS_HOST_API_KEY_HEADER=${EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY_HEADER}" >> /shared/config/service-addresses
 echo "export BOOT_NODE_HOST=${BOOT_NODE_ADDR}" >> /shared/config/service-addresses
 echo "export PROVER_NODE_HOST=${PROVER_NODE_ADDR}" >> /shared/config/service-addresses
 echo "export PROVER_BROKER_HOST=${PROVER_BROKER_ADDR}" >> /shared/config/service-addresses

spartan/aztec-network/templates/_helpers.tpl

@@ -147,6 +147,10 @@ Service Address Setup Container
       value: "{{ .Values.ethereum.execution.service.port }}"
     - name: EXTERNAL_ETHEREUM_CONSENSUS_HOST
       value: "{{ .Values.ethereum.beacon.externalHost }}"
+    - name: EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY
+      value: "{{ .Values.ethereum.beacon.apiKey }}"
+    - name: EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY_HEADER
+      value: "{{ .Values.ethereum.beacon.apiKeyHeader }}"
     - name: ETHEREUM_CONSENSUS_PORT
       value: "{{ .Values.ethereum.beacon.service.port }}"
     - name: EXTERNAL_BOOT_NODE_HOST

spartan/aztec-network/values.yaml

@@ -239,6 +239,9 @@ ethereum:
         cpu: "1"
     storageSize: "80Gi"
   beacon:
+    externalHost: ""
+    apiKey: ""
+    apiKeyHeader: ""
     service:
       port: 5052
       targetPort: 5052

spartan/aztec-network/values/rc-2.yaml

@@ -18,6 +18,10 @@ ethereum:
   chainId: "11155111"
   deployL1ContractsPrivateKey:
   externalHost:
+  beacon:
+    externalHost:
+    apiKey: ""
+    apiKeyHeader: ""
 
 validator:
   l1FixedPriorityFeePerGas: 2
@@ -101,7 +105,7 @@ proverAgent:
 
 bot:
   replicas: 4
-  followChain: "PENDING"
+  followChain: "NONE"
   enabled: true
   txIntervalSeconds: 0
   privateTransfersPerTx: 1

spartan/aztec-network/values/sepolia-48-validators-with-proving-and-metrics.yaml

@@ -12,12 +12,16 @@ images:
 
 network:
   setupL2Contracts: false
-  public: false
+  public: true
 
 ethereum:
   chainId: "11155111"
   deployL1ContractsPrivateKey:
   externalHost:
+  beacon:
+    externalHost:
+    apiKey: ""
+    apiKeyHeader: ""
 
 validator:
   l1FixedPriorityFeePerGas: 2
@@ -101,7 +105,7 @@ proverAgent:
 
 bot:
   replicas: 4
-  followChain: "PENDING"
+  followChain: "NONE"
   enabled: true
   txIntervalSeconds: 0
   privateTransfersPerTx: 1

spartan/terraform/deploy-release/main.tf

@@ -88,10 +88,34 @@ resource "helm_release" "aztec-gke-cluster" {
   }
 
   dynamic "set" {
-    for_each = var.ETHEREUM_EXTERNAL_HOST != "" ? toset(["iterate"]) : toset([])
+    for_each = var.EXTERNAL_ETHEREUM_HOST != "" ? toset(["iterate"]) : toset([])
     content {
       name  = "ethereum.externalHost"
-      value = var.ETHEREUM_EXTERNAL_HOST
+      value = var.EXTERNAL_ETHEREUM_HOST
+    }
+  }
+
+  dynamic "set" {
+    for_each = var.EXTERNAL_ETHEREUM_CONSENSUS_HOST != "" ? toset(["iterate"]) : toset([])
+    content {
+      name  = "ethereum.beacon.externalHost"
+      value = var.EXTERNAL_ETHEREUM_CONSENSUS_HOST
+    }
+  }
+
+  dynamic "set" {
+    for_each = var.EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY != "" ? toset(["iterate"]) : toset([])
+    content {
+      name  = "ethereum.beacon.apiKey"
+      value = var.EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY
+    }
+  }
+
+  dynamic "set" {
+    for_each = var.EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY_HEADER != "" ? toset(["iterate"]) : toset([])
+    content {
+      name  = "ethereum.beacon.apiKeyHeader"
+      value = var.EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY_HEADER
     }
   }
 

spartan/terraform/deploy-release/variables.tf

@@ -54,11 +54,30 @@ variable "PROVER_PUBLISHER_PRIVATE_KEY" {
   default     = ""
 }
 
-variable "ETHEREUM_EXTERNAL_HOST" {
+variable "EXTERNAL_ETHEREUM_HOST" {
   description = "External host to use for the ethereum node"
   type        = string
   default     = ""
 }
+
+variable "EXTERNAL_ETHEREUM_CONSENSUS_HOST" {
+  description = "External host to use for the ethereum consensus node"
+  type        = string
+  default     = ""
+}
+
+variable "EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY" {
+  description = "API key to use for the ethereum consensus node"
+  type        = string
+  default     = ""
+}
+
+variable "EXTERNAL_ETHEREUM_CONSENSUS_HOST_API_KEY_HEADER" {
+  description = "API key header to use for the ethereum consensus node"
+  type        = string
+  default     = ""
+}
+
 variable "L1_DEPLOYMENT_SALT" {
   description = "Salt to use for the L1 contract deployments"
   type        = string

yarn-project/aztec-node/src/aztec-node/config.ts

@@ -1,5 +1,5 @@
 import { type ArchiverConfig, archiverConfigMappings } from '@aztec/archiver/config';
-import { type BlobSinkConfig } from '@aztec/blob-sink/client';
+import { type BlobSinkConfig, blobSinkConfigMapping } from '@aztec/blob-sink/client';
 import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings } from '@aztec/foundation/config';
 import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config';
 import { type P2PConfig, p2pConfigMappings } from '@aztec/p2p/config';
@@ -37,6 +37,7 @@ export const aztecNodeConfigMappings: ConfigMappingsType<AztecNodeConfig> = {
   ...worldStateConfigMappings,
   ...p2pConfigMappings,
   ...dataConfigMappings,
+  ...blobSinkConfigMapping,
   disableValidator: {
     env: 'VALIDATOR_DISABLED',
     description: 'Whether the validator is disabled for this node.',

yarn-project/blob-sink/src/client/config.ts

@@ -18,6 +18,16 @@ export interface BlobSinkConfig {
    * The URL of the L1 consensus client
    */
   l1ConsensusHostUrl?: string;
+
+  /**
+   * The API key for the L1 consensus client. Added end of URL as "?key=<api-key>" unless a header is defined
+   */
+  l1ConsensusHostApiKey?: string;
+
+  /**
+   * The header name for the L1 consensus client API key, if needed. Added as "<api-key-header>: <api-key>"
+   */
+  l1ConsensusHostApiKeyHeader?: string;
 }
 
 export const blobSinkConfigMapping: ConfigMappingsType<BlobSinkConfig> = {
@@ -33,6 +43,16 @@ export const blobSinkConfigMapping: ConfigMappingsType<BlobSinkConfig> = {
     env: 'L1_CONSENSUS_HOST_URL',
     description: 'The URL of the L1 consensus client',
   },
+  l1ConsensusHostApiKey: {
+    env: 'L1_CONSENSUS_HOST_API_KEY',
+    description:
+      'The API key for the L1 consensus client, if needed. Added end of URL as "?key=<api-key>" unless a header is defined',
+  },
+  l1ConsensusHostApiKeyHeader: {
+    env: 'L1_CONSENSUS_HOST_API_KEY_HEADER',
+    description:
+      'The header name for the L1 consensus client API key, if needed. Added as "<api-key-header>: <api-key>"',
+  },
 };
 
 /**

yarn-project/blob-sink/src/client/http.ts

@@ -80,7 +80,10 @@ export class HttpBlobSinkClient implements BlobSinkClientInterface {
 
     if (blobs.length == 0 && this.config.l1ConsensusHostUrl) {
       // The beacon api can query by slot number, so we get that first
-      this.log.debug('Getting slot number from consensus host');
+      this.log.debug('Getting slot number from consensus host', {
+        blockHash,
+        consensusHostUrl: this.config.l1ConsensusHostUrl,
+      });
       const slotNumber = await this.getSlotNumber(blockHash);
       if (slotNumber) {
         const blobs = await this.getBlobSidecarFrom(this.config.l1ConsensusHostUrl, slotNumber, indices);
@@ -106,12 +109,16 @@ export class HttpBlobSinkClient implements BlobSinkClientInterface {
   ): Promise<Blob[]> {
     // TODO(md): right now we assume all blobs are ours, this will not yet work on sepolia
     try {
-      let url = `${hostUrl}/eth/v1/beacon/blob_sidecars/${blockHashOrSlot}`;
+      let baseUrl = `${hostUrl}/eth/v1/beacon/blob_sidecars/${blockHashOrSlot}`;
       if (indices && indices.length > 0) {
-        url += `?indices=${indices.join(',')}`;
+        baseUrl += `?indices=${indices.join(',')}`;
       }
 
-      const res = await this.fetch(url);
+      const { url, ...options } = getBeaconNodeFetchOptions(baseUrl, this.config);
+
+      this.log.debug(`Fetching blob sidecar from ${url} with options`, options);
+
+      const res = await this.fetch(url, options);
 
       if (res.ok) {
         const body = await res.json();
@@ -183,7 +190,12 @@ export class HttpBlobSinkClient implements BlobSinkClientInterface {
 
     // Query beacon chain to get the slot number for that block root
     try {
-      const res = await this.fetch(`${this.config.l1ConsensusHostUrl}/eth/v1/beacon/headers/${parentBeaconBlockRoot}`);
+      const { url, ...options } = getBeaconNodeFetchOptions(
+        `${this.config.l1ConsensusHostUrl}/eth/v1/beacon/headers/${parentBeaconBlockRoot}`,
+        this.config,
+      );
+      const res = await this.fetch(url, options);
+
       if (res.ok) {
         const body = await res.json();
 
@@ -210,3 +222,19 @@ function filterRelevantBlobs(blobs: Blob[], blobHashes: Buffer[]): Blob[] {
     return blobHashes.some(hash => hash.equals(blobHash));
   });
 }
+
+function getBeaconNodeFetchOptions(url: string, config: BlobSinkConfig) {
+  let formattedUrl = url;
+  if (config.l1ConsensusHostApiKey && !config.l1ConsensusHostApiKeyHeader) {
+    formattedUrl += `${formattedUrl.includes('?') ? '&' : '?'}key=${config.l1ConsensusHostApiKey}`;
+  }
+  return {
+    url: formattedUrl,
+    ...(config.l1ConsensusHostApiKey &&
+      config.l1ConsensusHostApiKeyHeader && {
+        headers: {
+          [config.l1ConsensusHostApiKeyHeader]: config.l1ConsensusHostApiKey,
+        },
+      }),
+  };
+}

yarn-project/foundation/src/config/env_var.ts

@@ -54,6 +54,8 @@ export type EnvVar =
   | 'INBOX_CONTRACT_ADDRESS'
   | 'L1_CHAIN_ID'
   | 'L1_CONSENSUS_HOST_URL'
+  | 'L1_CONSENSUS_HOST_API_KEY'
+  | 'L1_CONSENSUS_HOST_API_KEY_HEADER'
   | 'L1_PRIVATE_KEY'
   | 'L2_QUEUE_SIZE'
   | 'LOG_ELAPSED_TIME'