From 62adc45ffe7a02f750de94d7dc77f75496e0cef9 Mon Sep 17 00:00:00 2001 From: Charlie <5764343+charlielye@users.noreply.github.com> Date: Tue, 9 Jun 2026 16:50:50 +0000 Subject: [PATCH 01/12] feat(ci3): give single-instance PR runs a parent dashboard log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Merge-queue runs route through `multi_job_run`, which pipes the runner-side orchestration into a parent dashboard log (`cache_log "CI run" $RUN_ID`) โ€” so the spot/instance request is visible on ci.aztec-labs.com. Single-instance PR modes called `bootstrap_ec2` directly, so that output only reached the GitHub Actions console; you had to leave the dashboard to see which instance was created. Route the PR-facing single-instance modes (fast/docs/barretenberg/ barretenberg-full, full/full-no-test-cache, chonk-input-update) through `multi_job_run` with a single job, matching merge-queue. The job id is kept as `x-$cmd` so the `ci/` GitHub status check name is unchanged. socket-fix keeps its raw (un-denoised) output but now pipes through `cache_log` so it too gets a parent log. --- ci.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/ci.sh b/ci.sh index 26b0dfecb105..316abf4c6d86 100755 --- a/ci.sh +++ b/ci.sh @@ -112,27 +112,30 @@ case "$cmd" in ;; fast|docs|barretenberg|barretenberg-full) export CI_DASHBOARD="prs" - export JOB_ID="x-$cmd" - bootstrap_ec2 "./bootstrap.sh ci-$cmd" + # Route through multi_job_run (even for a single instance) so the runner-side + # orchestration โ€” including the spot/instance request โ€” is captured into a + # parent dashboard log, matching merge-queue. The job id stays "x-$cmd" so the + # GitHub status check name is unchanged. + multi_job_run "x-$cmd amd64 ci-$cmd" ;; socket-fix) export CI_DASHBOARD="prs" export JOB_ID="x-socket-fix" export INSTANCE_POSTFIX="socket-fix" export CPUS=16 - bootstrap_ec2 "./bootstrap.sh ci-socket-fix $*" + # Capture the runner-side output (incl. instance request) to a parent dashboard + # log. No denoise here: this is an interactive debug mode where raw output matters. + PARENT_LOG_ID=$RUN_ID bootstrap_ec2 "./bootstrap.sh ci-socket-fix $*" 2>&1 | DUP=1 cache_log "CI run" $RUN_ID ;; full|full-no-test-cache) export CI_DASHBOARD="prs" - export JOB_ID="x-$cmd" export AWS_SHUTDOWN_TIME=75 - bootstrap_ec2 "./bootstrap.sh ci-$cmd" + multi_job_run "x-$cmd amd64 ci-$cmd" ;; chonk-input-update) export CI_DASHBOARD="prs" - export JOB_ID="x-$cmd" export AWS_SHUTDOWN_TIME=90 - bootstrap_ec2 "./bootstrap.sh ci-chonk-input-update" + multi_job_run "x-$cmd amd64 ci-chonk-input-update" ;; barretenberg-debug) export CI_DASHBOARD="nightly" From 01f1bc5e9a8032d7cb9320131173531c940bb6b9 Mon Sep 17 00:00:00 2001 From: alexghr <3816165+alexghr@users.noreply.github.com> Date: Tue, 9 Jun 2026 17:04:10 +0000 Subject: [PATCH 02/12] chore: deployments Fix A-1163 --- .github/workflows/ci3.yml | 12 ++- .github/workflows/deploy-network.yml | 24 ++--- .github/workflows/deploy-next-net.yml | 43 +++------ .github/workflows/deploy-staging-internal.yml | 87 +++++++++++++++++ ...-staging.yml => deploy-staging-public.yml} | 12 +-- .../workflows/ensure-funded-environment.yml | 5 +- .../workflows/ensure-funded-environments.yml | 3 +- .github/workflows/network-healthcheck.yml | 4 +- .github/workflows/nightly-bench-10tps.yml | 2 +- .../workflows/nightly-release-tag-v4-next.yml | 33 ------- .github/workflows/nightly-release-tag.yml | 42 ++++++-- .github/workflows/nightly-spartan-bench.yml | 68 ++++++++----- .github/workflows/test-network-scenarios.yml | 7 ++ .github/workflows/weekly-proving-bench.yml | 2 +- spartan/.gitignore | 5 +- spartan/environments/next-net.env | 4 +- .../{staging.env => staging-internal.env} | 22 ++--- spartan/environments/staging-public.env | 96 +++++++++++++++++++ spartan/terraform/gke-cluster/iam.tf | 24 +++++ 19 files changed, 345 insertions(+), 150 deletions(-) create mode 100644 .github/workflows/deploy-staging-internal.yml rename .github/workflows/{deploy-staging.yml => deploy-staging-public.yml} (89%) delete mode 100644 .github/workflows/nightly-release-tag-v4-next.yml rename spartan/environments/{staging.env => staging-internal.env} (77%) create mode 100644 spartan/environments/staging-public.env diff --git a/.github/workflows/ci3.yml b/.github/workflows/ci3.yml index 8cd1128fddbc..d1c42e337e99 100644 --- a/.github/workflows/ci3.yml +++ b/.github/workflows/ci3.yml @@ -250,6 +250,7 @@ jobs: DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + INTERNAL_DOCKER_REGISTRY: ${{ secrets.INTERNAL_DOCKER_REGISTRY }} CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }} CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }} RUN_ID: ${{ github.run_id }} @@ -265,7 +266,16 @@ jobs: tag="${tag#v}" major_version=$(./ci3/semver major "$tag") namespace="v${major_version}-scenario" - docker_image="aztecprotocol/aztec:${tag}" + case "$GITHUB_REPOSITORY" in + AztecProtocol/aztec-packages-private) + image_registry="${INTERNAL_DOCKER_REGISTRY%/}" + echo "::add-mask::${image_registry}" + ;; + *) + image_registry="aztecprotocol" + ;; + esac + docker_image="${image_registry}/aztec:${tag}" else # branch name namespace=pr-$(echo "$HEAD_REF" | sed 's/[^a-z0-9-]/-/g' | cut -c1-20 | sed 's/-*$//') diff --git a/.github/workflows/deploy-network.yml b/.github/workflows/deploy-network.yml index c1d0130b0256..3b96f8654db8 100644 --- a/.github/workflows/deploy-network.yml +++ b/.github/workflows/deploy-network.yml @@ -47,11 +47,11 @@ on: description: "Source tag that triggered this deploy" required: false type: string - notify_on_failure: - description: "Whether this workflow should send its own failure notification" + skip_notify_on_failure: + description: "Whether this workflow should skip its own failure notification" required: false type: boolean - default: true + default: false workflow_dispatch: inputs: network: @@ -60,7 +60,8 @@ on: type: choice options: - next-net - - staging + - staging-public + - staging-internal - testnet - mainnet semver: @@ -97,11 +98,6 @@ on: description: "Source tag that triggered this deploy" required: false type: string - notify_on_failure: - description: "Whether this workflow should send its own failure notification" - required: false - type: boolean - default: true concurrency: group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.aztec_docker_image || inputs.semver }}-${{ github.ref || github.ref_name }} @@ -254,8 +250,6 @@ jobs: VALIDATOR_HA_DOCKER_IMAGE: ${{ inputs.ha_docker_image || '' }} run: | echo "Deploying network: ${{ inputs.network }}" - echo "Using image: $AZTEC_DOCKER_IMAGE" - echo "Using prover image: $PROVER_AGENT_DOCKER_IMAGE" echo "Using branch/ref: ${{ steps.checkout-ref.outputs.ref }}" cd spartan @@ -280,7 +274,6 @@ jobs: echo "| Item | Value |" echo "|------|-------|" echo "| Network | \`${{ inputs.network }}\` |" - echo "| Docker Image | \`${{ env.AZTEC_DOCKER_IMAGE }}\` |" echo "| Ref | \`${{ steps.checkout-ref.outputs.ref }}\` |" if [[ -n "${{ inputs.source_tag }}" ]]; then echo "| Source Tag | [\`${{ inputs.source_tag }}\`](https://github.com/${{ github.repository }}/releases/tag/${{ inputs.source_tag }}) |" @@ -288,7 +281,7 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" - name: Notify Slack and dispatch ClaudeBox on failure - if: failure() && inputs.notify_on_failure + if: failure() && !inputs.skip_notify_on_failure env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} @@ -300,7 +293,7 @@ jobs: CHANNEL="#alerts-${{ inputs.network }}" RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - TEXT="Deploy Network workflow FAILED for *${{ inputs.network }}* (image ${{ env.AZTEC_DOCKER_IMAGE }}): <${RUN_URL}|View Run> (๐Ÿค–)" + TEXT="Deploy Network workflow FAILED for *${{ inputs.network }}*: <${RUN_URL}|View Run> (๐Ÿค–)" # Post to Slack and capture timestamp for permalink RESP=$(curl -sS -X POST https://slack.com/api/chat.postMessage \ @@ -318,11 +311,10 @@ jobs: fi # Dispatch ClaudeBox to investigate the failure - PROMPT="Deployment of ${{ inputs.network }} (image ${{ env.AZTEC_DOCKER_IMAGE }}) failed. \ + PROMPT="Deployment of ${{ inputs.network }} failed. \ Follow .claude/claudebox/deploy-investigation.md to investigate. \ GitHub Actions run: ${RUN_URL}. \ Network: ${{ inputs.network }}. \ - Docker image: ${{ env.AZTEC_DOCKER_IMAGE }}. \ Git ref: ${{ steps.checkout-ref.outputs.ref }}. \ Namespace: ${{ inputs.namespace || inputs.network }}. \ Deploy contracts: ${{ inputs.deploy_contracts }}." diff --git a/.github/workflows/deploy-next-net.yml b/.github/workflows/deploy-next-net.yml index fc40cec81619..b06d064986c9 100644 --- a/.github/workflows/deploy-next-net.yml +++ b/.github/workflows/deploy-next-net.yml @@ -1,6 +1,6 @@ # Deploy next-net environment # This workflow deploys the next-net environment with a specified version -# Runs nightly with the latest nightly tag, or can be manually triggered with any image +# Runs from the private repo's next branch with the latest nightly tag, or a manually supplied tag name: Deploy Next Net on: @@ -10,7 +10,7 @@ on: workflow_dispatch: inputs: image_tag: - description: "Docker image tag (e.g., 2.3.4, 3.0.0-nightly.20251004-amd64, or leave empty for latest nightly)" + description: "Docker image tag/semver (e.g., 6.0.0-nightly.20260609, or leave empty for latest nightly)" required: false type: string deploy_contracts: @@ -25,57 +25,42 @@ concurrency: jobs: get-image-tag: - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest outputs: - tag: ${{ steps.determine_tag.outputs.TAG }} semver: ${{ steps.determine_tag.outputs.SEMVER }} steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: next - name: Determine image tag id: determine_tag run: | if [[ -n "${{ inputs.image_tag }}" ]]; then - # Manual trigger with specified tag - TAG="${{ inputs.image_tag }}" - echo "Using manually specified tag: $TAG" - - # Extract semver (remove -amd64 suffix if present) - SEMVER=$(echo "$TAG" | sed 's/-amd64$//') + SEMVER="${{ inputs.image_tag }}" + SEMVER="${SEMVER#v}" + SEMVER="${SEMVER%-amd64}" + echo "Using manually specified tag: $SEMVER" else - # Scheduled nightly run - get latest nightly tag current_version=$(jq -r '."."' .release-please-manifest.json) echo "Current version: $current_version" - # Format the tag as: -nightly.-amd64 - nightly_semver="${current_version}-nightly.$(date -u +%Y%m%d)" - nightly_tag="${nightly_semver}-amd64" - - # Check if the tag exists on docker hub - TAGS=$(curl -s https://registry.hub.docker.com/v2/repositories/aztecprotocol/aztec/tags/$nightly_tag) - if [[ "$TAGS" != *"not found"* ]]; then - TAG="$nightly_tag" - SEMVER="$nightly_semver" - echo "Using nightly tag: $TAG" - else - echo "Error: Tag $nightly_tag not published to docker hub" - exit 1 - fi + SEMVER="${current_version}-nightly.$(date -u +%Y%m%d)" + echo "Using nightly tag: $SEMVER" fi - echo "TAG=$TAG" >> "$GITHUB_OUTPUT" echo "SEMVER=$SEMVER" >> "$GITHUB_OUTPUT" deploy-next-net: needs: get-image-tag - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.repository == 'AztecProtocol/aztec-packages-private' }} uses: ./.github/workflows/deploy-network.yml with: network: next-net semver: ${{ needs.get-image-tag.outputs.semver }} - aztec_docker_image: "aztecprotocol/aztec:${{ needs.get-image-tag.outputs.tag }}" - ref: ${{ github.ref }} + ref: next + use_internal_docker_registry: true deploy_contracts: ${{ github.event_name == 'schedule' || inputs.deploy_contracts == true }} secrets: inherit diff --git a/.github/workflows/deploy-staging-internal.yml b/.github/workflows/deploy-staging-internal.yml new file mode 100644 index 000000000000..d5f3eb127984 --- /dev/null +++ b/.github/workflows/deploy-staging-internal.yml @@ -0,0 +1,87 @@ +name: Deploy to staging internal + +on: + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + tag: + description: "Tag to deploy (v5.0.0 or any v5.0.0 prerelease). Leave empty to use today's nightly tag." + required: false + type: string + deploy_contracts: + description: "Redeploy rollup contracts before deploying infra." + required: false + type: boolean + default: false + +concurrency: + group: deploy-staging-internal + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + +jobs: + determine-tag: + runs-on: ubuntu-latest + if: >- + (github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule') && + (github.event_name != 'schedule' || + github.repository == 'AztecProtocol/aztec-packages-private') + outputs: + tag: ${{ steps.resolve.outputs.tag }} + semver: ${{ steps.resolve.outputs.semver }} + steps: + - name: Checkout v5-next + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: v5-next + token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Resolve nightly tag + id: resolve + run: | + if [[ -n "${{ inputs.tag }}" ]]; then + TAG="${{ inputs.tag }}" + echo "Using manually specified tag: $TAG" + else + TAG="v5.0.0-nightly.$(date -u +%Y%m%d)" + echo "Using today's nightly tag: $TAG" + fi + + SEMVER="${TAG#v}" + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "semver=$SEMVER" >> $GITHUB_OUTPUT + + wait-for-ci3: + needs: determine-tag + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages-private' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + fetch-depth: 1 + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: 22 + + - name: Wait for CI3 + run: spartan/scripts/wait_for_ci3.ts "${{ needs.determine-tag.outputs.tag }}" + + deploy: + needs: [determine-tag, wait-for-ci3] + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages-private' }} + uses: ./.github/workflows/deploy-network.yml + with: + network: staging-internal + semver: ${{ needs.determine-tag.outputs.semver }} + source_tag: ${{ needs.determine-tag.outputs.tag }} + deploy_contracts: ${{ inputs.deploy_contracts == true }} + use_internal_docker_registry: true + secrets: inherit diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging-public.yml similarity index 89% rename from .github/workflows/deploy-staging.yml rename to .github/workflows/deploy-staging-public.yml index 0c5b38f6d65d..8aaea198d33c 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging-public.yml @@ -1,4 +1,4 @@ -name: Deploy to staging +name: Deploy to staging public on: schedule: @@ -16,7 +16,7 @@ on: default: false concurrency: - group: deploy-staging + group: deploy-staging-public cancel-in-progress: true env: @@ -52,11 +52,6 @@ jobs: echo "Using today's nightly tag: $TAG" fi - if [[ ! "$TAG" =~ ^v5\.0\.0($|-.*) ]]; then - echo "Error: Staging deploys require a v5.0.0 tag or prerelease, got '$TAG'" - exit 1 - fi - SEMVER="${TAG#v}" echo "tag=$TAG" >> $GITHUB_OUTPUT echo "semver=$SEMVER" >> $GITHUB_OUTPUT @@ -84,9 +79,8 @@ jobs: if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} uses: ./.github/workflows/deploy-network.yml with: - network: staging + network: staging-public semver: ${{ needs.determine-tag.outputs.semver }} - use_internal_docker_registry: true source_tag: ${{ needs.determine-tag.outputs.tag }} deploy_contracts: ${{ inputs.deploy_contracts == true }} secrets: inherit diff --git a/.github/workflows/ensure-funded-environment.yml b/.github/workflows/ensure-funded-environment.yml index 89a98760a424..63639ce8c701 100644 --- a/.github/workflows/ensure-funded-environment.yml +++ b/.github/workflows/ensure-funded-environment.yml @@ -6,7 +6,7 @@ on: workflow_call: inputs: environment: - description: 'Environment to fund (e.g., staging, next-net, testnet)' + description: 'Environment to fund (e.g., staging-public, staging-internal, next-net, testnet)' required: true type: string low_watermark: @@ -30,7 +30,8 @@ on: required: true type: choice options: - - staging + - staging-public + - staging-internal - next-net - testnet - devnet diff --git a/.github/workflows/ensure-funded-environments.yml b/.github/workflows/ensure-funded-environments.yml index a8dd9de4bfa7..3378300f1701 100644 --- a/.github/workflows/ensure-funded-environments.yml +++ b/.github/workflows/ensure-funded-environments.yml @@ -50,7 +50,8 @@ jobs: run: | # Static environments static='[ - {"environment":"staging"}, + {"environment":"staging-public"}, + {"environment":"staging-internal"}, {"environment":"next-net"}, {"environment":"testnet"} ]' diff --git a/.github/workflows/network-healthcheck.yml b/.github/workflows/network-healthcheck.yml index eae21f5bea36..7347e74d1387 100644 --- a/.github/workflows/network-healthcheck.yml +++ b/.github/workflows/network-healthcheck.yml @@ -9,7 +9,7 @@ on: description: 'Comma-separated list of networks to check (default: all)' required: false type: string - default: 'testnet,mainnet,staging,next-net' + default: 'testnet,mainnet,staging-public,staging-internal,next-net' jobs: healthcheck: @@ -25,7 +25,7 @@ jobs: GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} CI: "1" run: | - NETWORKS="${{ inputs.networks || 'next-net,staging,testnet,mainnet' }}" + NETWORKS="${{ inputs.networks || 'next-net,staging-public,staging-internal,testnet,mainnet' }}" PROMPT="Run a network healthcheck for: ${NETWORKS}. diff --git a/.github/workflows/nightly-bench-10tps.yml b/.github/workflows/nightly-bench-10tps.yml index 3f0c096b6980..f06e34fa6198 100644 --- a/.github/workflows/nightly-bench-10tps.yml +++ b/.github/workflows/nightly-bench-10tps.yml @@ -100,7 +100,7 @@ jobs: namespace: bench-10tps aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + skip_notify_on_failure: true secrets: inherit wait-for-first-l2-block: diff --git a/.github/workflows/nightly-release-tag-v4-next.yml b/.github/workflows/nightly-release-tag-v4-next.yml deleted file mode 100644 index 73d53386b326..000000000000 --- a/.github/workflows/nightly-release-tag-v4-next.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Nightly Release Tag (v4-next) -on: - schedule: - # Run the workflow every night at 5:00 AM UTC. - - cron: "0 5 * * *" - workflow_dispatch: {} - -permissions: - contents: write - -concurrency: - group: ${{ github.workflow }}-v4-next - -jobs: - nightly-release-tag-v4-next: - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: v4-next - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - - - name: Create v4-next Nightly Tag - run: | - git config --global user.email "tech@aztecprotocol.com" - git config --global user.name "AztecBot" - current_version=$(jq -r '."."' .release-please-manifest.json) - echo "Current version: $current_version" - nightly_tag="v${current_version}-nightly.$(date -u +%Y%m%d)" - echo "v4-next nightly tag: $nightly_tag" - git tag -a "$nightly_tag" -m "$nightly_tag" - git push origin "$nightly_tag" diff --git a/.github/workflows/nightly-release-tag.yml b/.github/workflows/nightly-release-tag.yml index 4953711fe4f5..c0627849dcc2 100644 --- a/.github/workflows/nightly-release-tag.yml +++ b/.github/workflows/nightly-release-tag.yml @@ -3,26 +3,27 @@ on: schedule: # Run the workflow every night at 2:00 AM UTC. - cron: "0 2 * * *" + workflow_dispatch: {} -# Add permissions for the GitHub Actions bot to push tags permissions: contents: write -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - jobs: nightly-release-tag: - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + branch: [next, v5-next] + concurrency: + group: ${{ github.workflow }}-${{ github.repository }}-${{ matrix.branch }} steps: - # Check out the repository so we can read files and create tags. - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: + ref: ${{ matrix.branch }} token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - # Extract the current release version from the manifest. - # Then, create a nightly tag using the current version and the current UTC date. - name: Create Nightly Tag run: | git config --global user.email "tech@aztecprotocol.com" @@ -30,7 +31,28 @@ jobs: current_version=$(jq -r '."."' .release-please-manifest.json) echo "Current version: $current_version" nightly_tag="v${current_version}-nightly.$(date -u +%Y%m%d)" - echo "Nightly tag: $nightly_tag" - # Tag and push. + echo "${{ matrix.branch }} nightly tag: $nightly_tag" + git tag -a "$nightly_tag" -m "$nightly_tag" + git push origin "$nightly_tag" + + nightly-release-tag-v4-next: + if: ${{ github.repository == 'AztecProtocol/aztec-packages' }} + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.repository }}-v4-next + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: v4-next + token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + + - name: Create v4-next Nightly Tag + run: | + git config --global user.email "tech@aztecprotocol.com" + git config --global user.name "AztecBot" + current_version=$(jq -r '."."' .release-please-manifest.json) + echo "Current version: $current_version" + nightly_tag="v${current_version}-nightly.$(date -u +%Y%m%d)" + echo "v4-next nightly tag: $nightly_tag" git tag -a "$nightly_tag" -m "$nightly_tag" git push origin "$nightly_tag" diff --git a/.github/workflows/nightly-spartan-bench.yml b/.github/workflows/nightly-spartan-bench.yml index 166427975ee5..5399e4302365 100644 --- a/.github/workflows/nightly-spartan-bench.yml +++ b/.github/workflows/nightly-spartan-bench.yml @@ -16,33 +16,47 @@ concurrency: jobs: select-image: - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest outputs: nightly_tag: ${{ steps.nightly-tag.outputs.nightly_tag }} docker_image: ${{ steps.nightly-tag.outputs.docker_image }} source_ref: ${{ steps.nightly-tag.outputs.source_ref }} + use_internal_docker_registry: ${{ steps.nightly-tag.outputs.use_internal_docker_registry }} steps: - name: Checkout uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: - ref: next + ref: ${{ github.repository == 'AztecProtocol/aztec-packages-private' && github.event_name == 'schedule' && 'v5-next' || 'next' }} - name: Determine nightly tag id: nightly-tag run: | + USE_INTERNAL_DOCKER_REGISTRY="${{ github.repository == 'AztecProtocol/aztec-packages-private' && 'true' || 'false' }}" + if [[ -n "${{ inputs.nightly_tag }}" ]]; then nightly_tag="${{ inputs.nightly_tag }}" else current_version=$(jq -r '."."' .release-please-manifest.json) nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)" fi - docker_image="aztecprotocol/aztec:${nightly_tag}" + source_ref="v${nightly_tag}" + + if [[ "$USE_INTERNAL_DOCKER_REGISTRY" == "true" ]]; then + docker_image="" + image_label="internal/aztec:${nightly_tag}" + else + docker_image="aztecprotocol/aztec:${nightly_tag}" + image_label="$docker_image" + fi + echo "nightly_tag=$nightly_tag" >> "$GITHUB_OUTPUT" echo "docker_image=$docker_image" >> "$GITHUB_OUTPUT" echo "source_ref=$source_ref" >> "$GITHUB_OUTPUT" + echo "use_internal_docker_registry=$USE_INTERNAL_DOCKER_REGISTRY" >> "$GITHUB_OUTPUT" echo "Using nightly tag: $nightly_tag" + echo "Using image: $image_label" echo "Using source ref: $source_ref" - name: Verify source git ref @@ -54,6 +68,7 @@ jobs: echo "Nightly source commit: $source_sha" - name: Check if Docker image exists + if: steps.nightly-tag.outputs.use_internal_docker_registry != 'true' run: | DOCKER_IMAGE="${{ steps.nightly-tag.outputs.docker_image }}" echo "Checking if Docker image exists: $DOCKER_IMAGE" @@ -69,21 +84,22 @@ jobs: # --------------------------------------------------------------------------- deploy-bench-network: needs: select-image - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} uses: ./.github/workflows/deploy-network.yml with: network: tps-scenario namespace: nightly-bench - aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} + semver: ${{ needs.select-image.outputs.nightly_tag }} ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + use_internal_docker_registry: ${{ needs.select-image.outputs.use_internal_docker_registry == 'true' }} + skip_notify_on_failure: true secrets: inherit wait-bench-l2-block: needs: - select-image - deploy-bench-network - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -114,7 +130,7 @@ jobs: needs: - select-image - wait-bench-l2-block - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -168,7 +184,7 @@ jobs: max-items-in-chart: 100 cleanup-bench: - if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-bench-network @@ -193,7 +209,7 @@ jobs: run: ./.github/ci3.sh network-teardown tps-scenario nightly-bench notify-bench-failure: - if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-bench-network @@ -224,21 +240,22 @@ jobs: # --------------------------------------------------------------------------- deploy-proving-network: needs: select-image - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} uses: ./.github/workflows/deploy-network.yml with: network: prove-n-tps-fake namespace: prove-n-tps-fake - aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} + semver: ${{ needs.select-image.outputs.nightly_tag }} ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + use_internal_docker_registry: ${{ needs.select-image.outputs.use_internal_docker_registry == 'true' }} + skip_notify_on_failure: true secrets: inherit wait-proving-l2-block: needs: - select-image - deploy-proving-network - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -269,7 +286,7 @@ jobs: needs: - select-image - wait-proving-l2-block - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -322,7 +339,7 @@ jobs: max-items-in-chart: 100 cleanup-proving: - if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-proving-network @@ -347,7 +364,7 @@ jobs: run: ./.github/ci3.sh network-teardown prove-n-tps-fake prove-n-tps-fake notify-proving-failure: - if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-proving-network @@ -378,21 +395,22 @@ jobs: # --------------------------------------------------------------------------- deploy-block-capacity-network: needs: select-image - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} uses: ./.github/workflows/deploy-network.yml with: network: block-capacity namespace: nightly-block-capacity - aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} + semver: ${{ needs.select-image.outputs.nightly_tag }} ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + use_internal_docker_registry: ${{ needs.select-image.outputs.use_internal_docker_registry == 'true' }} + skip_notify_on_failure: true secrets: inherit wait-block-capacity-l2-block: needs: - select-image - deploy-block-capacity-network - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -423,7 +441,7 @@ jobs: needs: - select-image - wait-block-capacity-l2-block - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -476,7 +494,7 @@ jobs: max-items-in-chart: 100 cleanup-block-capacity: - if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-block-capacity-network @@ -501,7 +519,7 @@ jobs: run: ./.github/ci3.sh network-teardown block-capacity nightly-block-capacity notify-block-capacity-failure: - if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-block-capacity-network @@ -530,7 +548,7 @@ jobs: status: runs-on: ubuntu-latest needs: [benchmark, proving-benchmark, block-capacity-benchmark] - if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} steps: - name: Check benchmark results run: | diff --git a/.github/workflows/test-network-scenarios.yml b/.github/workflows/test-network-scenarios.yml index 2393ba576920..fa23bca63f2b 100644 --- a/.github/workflows/test-network-scenarios.yml +++ b/.github/workflows/test-network-scenarios.yml @@ -10,6 +10,11 @@ on: description: Complete docker image to use (e.g., aztecprotocol/aztec:some-tag) required: true type: string + ref: + description: Git ref to checkout for the scenario runner. + required: false + type: string + default: next namespace: description: Kubernetes namespace to use (will be sanitized for k8s naming) required: true @@ -39,6 +44,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ inputs.ref }} - name: Run Network Scenarios timeout-minutes: 350 diff --git a/.github/workflows/weekly-proving-bench.yml b/.github/workflows/weekly-proving-bench.yml index 0353eaa4d08c..c8ce2b663e96 100644 --- a/.github/workflows/weekly-proving-bench.yml +++ b/.github/workflows/weekly-proving-bench.yml @@ -74,7 +74,7 @@ jobs: namespace: prove-n-tps-real aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + skip_notify_on_failure: true secrets: inherit wait-for-first-l2-block: diff --git a/spartan/.gitignore b/spartan/.gitignore index da7cd59acdb2..b7719acadf31 100644 --- a/spartan/.gitignore +++ b/spartan/.gitignore @@ -23,9 +23,8 @@ environments/* !environments/next-scenario.env !environments/scenario.local.env !environments/source-env.sh -!environments/staging-ignition.env -!environments/staging.env -!environments/staging.local.env +!environments/staging-public.env +!environments/staging-internal.env !environments/testnet-canary.env !environments/testnet.env !environments/mainnet.env diff --git a/spartan/environments/next-net.env b/spartan/environments/next-net.env index cac6310416e4..94d2694e93a7 100644 --- a/spartan/environments/next-net.env +++ b/spartan/environments/next-net.env @@ -22,9 +22,7 @@ P2P_HOSTPORT_ENABLED=false STORE_SNAPSHOT_URL= BLOB_BUCKET_DIRECTORY=${BLOB_BUCKET_DIRECTORY:-next-net/blobs} BLOB_FILE_STORE_URLS="," -TX_FILE_STORE_ENABLED=true -TX_FILE_STORE_BUCKET_DIRECTORY=${TX_FILE_STORE_BUCKET_DIRECTORY:-next-net/txs} -TX_COLLECTION_FILE_STORE_URLS="https://aztec-labs-snapshots.com/${TX_FILE_STORE_BUCKET_DIRECTORY}" +TX_FILE_STORE_ENABLED=false R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET PROVER_FAILED_PROOF_STORE=gs://aztec-develop/next-net/failed-proofs diff --git a/spartan/environments/staging.env b/spartan/environments/staging-internal.env similarity index 77% rename from spartan/environments/staging.env rename to spartan/environments/staging-internal.env index 3193c558f6b6..a201feba0344 100644 --- a/spartan/environments/staging.env +++ b/spartan/environments/staging-internal.env @@ -1,7 +1,7 @@ # Deployment CREATE_ETH_DEVNET=false NETWORK=staging -NAMESPACE=${NAMESPACE:-staging} +NAMESPACE=${NAMESPACE:-staging-internal} GCP_REGION=us-west1-a CLUSTER=aztec-gke-public RESOURCE_PROFILE=prod @@ -15,17 +15,12 @@ ETHEREUM_CONSENSUS_HOST_API_KEY_HEADERS=REPLACE_WITH_GCP_SECRET FUNDING_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET LABS_INFRA_MNEMONIC=REPLACE_WITH_GCP_SECRET ROLLUP_DEPLOYMENT_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET -VERIFY_CONTRACTS=true -ETHERSCAN_API_KEY=REPLACE_WITH_GCP_SECRET CREATE_ROLLUP_CONTRACTS=${CREATE_ROLLUP_CONTRACTS:-false} # Storage -SNAPSHOT_BUCKET_DIRECTORY=${SNAPSHOT_BUCKET_DIRECTORY:-staging} -BLOB_BUCKET_DIRECTORY=${BLOB_BUCKET_DIRECTORY:-staging/blobs} -BLOB_FILE_STORE_URLS="," -TX_FILE_STORE_ENABLED=true -TX_FILE_STORE_BUCKET_DIRECTORY=${TX_FILE_STORE_BUCKET_DIRECTORY:-staging/txs} -TX_COLLECTION_FILE_STORE_URLS="https://aztec-labs-snapshots.com/${TX_FILE_STORE_BUCKET_DIRECTORY}" +SNAPSHOT_BUCKET_DIRECTORY=${SNAPSHOT_BUCKET_DIRECTORY:-staging-internal/snapshots} +SYNC_SNAPSHOT_URLS="https://aztec-labs-snapshots.com/${SNAPSHOT_BUCKET_DIRECTORY}" +TX_FILE_STORE_ENABLED=false R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET @@ -58,17 +53,16 @@ VALIDATOR_REPLICAS=2 VALIDATORS_PER_NODE=32 VALIDATOR_PUBLISHERS_PER_REPLICA=4 VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 -VALIDATOR_HA_REPLICAS=1 -VALIDATOR_HA_REPLICA_COUNT=2 VALIDATOR_RESOURCE_PROFILE="prod" + RPC_REPLICAS=1 # Prover PROVER_RESOURCE_PROFILE="prod" PUBLISHERS_PER_PROVER=2 PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 -PROVER_FAILED_PROOF_STORE=gs://aztec-develop/staging/failed-proofs -L1_TX_FAILED_STORE=gs://aztec-develop/staging/failed-l1-txs +# PROVER_FAILED_PROOF_STORE=gs://aztec-develop/staging/failed-proofs +# L1_TX_FAILED_STORE=gs://aztec-develop/staging/failed-l1-txs PROVER_AGENT_KEDA_ENABLED=true PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET PROVER_AGENT_KEDA_MIN_REPLICAS=0 @@ -99,4 +93,4 @@ BOT_CROSS_CHAIN_FOLLOW_CHAIN=PROPOSED # Observability OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET -LOG_LEVEL="debug; info: simulator, json-rpc" +LOG_LEVEL="info" diff --git a/spartan/environments/staging-public.env b/spartan/environments/staging-public.env new file mode 100644 index 000000000000..843d09d6925c --- /dev/null +++ b/spartan/environments/staging-public.env @@ -0,0 +1,96 @@ +# Deployment +CREATE_ETH_DEVNET=false +NETWORK=staging +NAMESPACE=${NAMESPACE:-staging-public} +GCP_REGION=us-west1-a +CLUSTER=aztec-gke-public +RESOURCE_PROFILE=prod + +# Ethereum +ETHEREUM_CHAIN_ID=11155111 +ETHEREUM_RPC_URLS=REPLACE_WITH_GCP_SECRET +ETHEREUM_CONSENSUS_HOST_URLS=REPLACE_WITH_GCP_SECRET +ETHEREUM_CONSENSUS_HOST_API_KEYS=REPLACE_WITH_GCP_SECRET +ETHEREUM_CONSENSUS_HOST_API_KEY_HEADERS=REPLACE_WITH_GCP_SECRET +FUNDING_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET +LABS_INFRA_MNEMONIC=REPLACE_WITH_GCP_SECRET +ROLLUP_DEPLOYMENT_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET +CREATE_ROLLUP_CONTRACTS=${CREATE_ROLLUP_CONTRACTS:-false} + +# Storage +SNAPSHOT_BUCKET_DIRECTORY=${SNAPSHOT_BUCKET_DIRECTORY:-staging-public/snapshots} +SYNC_SNAPSHOT_URLS="https://aztec-labs-snapshots.com/${SNAPSHOT_BUCKET_DIRECTORY}" +TX_FILE_STORE_ENABLED=false +R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET +R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET + +# Network features +DEPLOY_INTERNAL_BOOTNODE=true +P2P_PUBLIC_IP=false +P2P_HOSTPORT_ENABLED=false +P2P_TX_POOL_DELETE_TXS_AFTER_REORG=true +TEST_ACCOUNTS=false +SPONSORED_FPC=false + +# Rollup +AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=2 +AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=2 +AZTEC_INBOX_LAG=2 +AZTEC_MANA_TARGET=75000000 +AZTEC_PROVING_COST_PER_MANA=12500000 + +# Sequencer +SEQ_MAX_L2_BLOCK_GAS=150000000 +SEQ_MIN_TX_PER_BLOCK=1 +SEQ_MAX_TX_PER_CHECKPOINT=7 # 0.1 TPS +SEQ_BUILD_CHECKPOINT_IF_EMPTY=true +SEQ_BLOCK_DURATION_MS=6000 +SEQ_L1_PUBLISHING_TIME_ALLOWANCE_IN_SLOT=36 +SEQ_ENABLE_PROPOSER_PIPELINING=true + +# Validators and RPC +VALIDATOR_REPLICAS=2 +VALIDATORS_PER_NODE=32 +VALIDATOR_PUBLISHERS_PER_REPLICA=4 +VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 +VALIDATOR_RESOURCE_PROFILE="prod" + +RPC_REPLICAS=1 + +# Prover +PROVER_RESOURCE_PROFILE="prod" +PUBLISHERS_PER_PROVER=2 +PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 +# PROVER_FAILED_PROOF_STORE=gs://aztec-develop/staging/failed-proofs +# L1_TX_FAILED_STORE=gs://aztec-develop/staging/failed-l1-txs +PROVER_AGENT_KEDA_ENABLED=true +PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET +PROVER_AGENT_KEDA_MIN_REPLICAS=0 +PROVER_AGENT_KEDA_MAX_REPLICAS=8 +PROVER_AGENT_KEDA_SCALING_BANDS='[ + { + queueSize = 0 + replicas = 4 + }, + { + queueSize = 100 + replicas = 8 + } +]' + +# Bots +BOT_DA_GAS_LIMIT=100000 +BOT_L2_GAS_LIMIT=6540000 +BOT_TRANSFERS_REPLICAS=1 +BOT_TRANSFERS_TX_INTERVAL_SECONDS=250 +BOT_TRANSFERS_FOLLOW_CHAIN=PROPOSED +BOT_SWAPS_REPLICAS=0 +BOT_SWAPS_FOLLOW_CHAIN=PROPOSED +BOT_SWAPS_TX_INTERVAL_SECONDS=350 +BOT_CROSS_CHAIN_REPLICAS=0 +BOT_CROSS_CHAIN_TX_INTERVAL_SECONDS=250 +BOT_CROSS_CHAIN_FOLLOW_CHAIN=PROPOSED + +# Observability +OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET +LOG_LEVEL="info" diff --git a/spartan/terraform/gke-cluster/iam.tf b/spartan/terraform/gke-cluster/iam.tf index 10663f891ed2..ded765f27248 100644 --- a/spartan/terraform/gke-cluster/iam.tf +++ b/spartan/terraform/gke-cluster/iam.tf @@ -48,6 +48,30 @@ resource "google_service_account" "ci" { description = "Service account for CI jobs that publish Docker images" } +resource "google_project_iam_member" "ci_network_deploy_project_roles" { + for_each = toset([ + "roles/container.developer", + "roles/secretmanager.secretAccessor", + "roles/compute.loadBalancerAdmin" + ]) + project = var.project + role = each.key + member = "serviceAccount:${google_service_account.ci.email}" +} + +resource "google_storage_bucket_iam_member" "ci_terraform_state_object_user" { + bucket = "aztec-terraform" + role = "roles/storage.objectUser" + member = "serviceAccount:${google_service_account.ci.email}" +} + +resource "google_dns_managed_zone_iam_member" "ci_rpc_dns_admin" { + project = var.project + managed_zone = "rpc-aztec-labs-com" + role = "roles/dns.admin" + member = "serviceAccount:${google_service_account.ci.email}" +} + resource "google_service_account" "npm_registry_reader" { account_id = var.npm_registry_reader_service_account_id display_name = "npm Registry Reader Service Account" From 664db0fde7091c53f90c145357773041bb2ae719 Mon Sep 17 00:00:00 2001 From: Charlie <5764343+charlielye@users.noreply.github.com> Date: Wed, 10 Jun 2026 12:41:33 +0000 Subject: [PATCH 03/12] fix(ci3): scope build-instance name by repo to stop cross-repo reaping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bootstrap_ec2 terminates any existing instance sharing the target Name tag, to reap orphans left by a cancelled GA run on the same ref. But the name was just _[_postfix], with no repo component โ€” so aztec-packages and aztec-packages-private, which build the same tags/refs concurrently under the same OIDC role, computed identical names and reaped each other's live instances. Observed: nightly tag v5.0.0-nightly.20260610 built in both repos; the public run's pre-launch reap terminated the private run's in-progress arm64 release instance ~7 min in, failing that build. Prefix the instance name with the repo basename (GITHUB_REPOSITORY##*/, default aztec-packages). The key stays stable across re-runs within a repo, so the intended orphan cleanup still works; it only stops the two repos from colliding. ci.sh's helper instance_name (shell/kill/get-ip) is kept in sync. --- ci.sh | 6 +++++- ci3/bootstrap_ec2 | 13 ++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/ci.sh b/ci.sh index 26b0dfecb105..101409c887ce 100755 --- a/ci.sh +++ b/ci.sh @@ -53,7 +53,11 @@ function print_usage { [ -n "$cmd" ] && shift -instance_name=${INSTANCE_NAME:-$(echo -n "$BRANCH" | tr -c 'a-zA-Z0-9-' '_')_${arch}} +# Keep this in sync with bootstrap_ec2's instance_name scheme (repo-scoped) so the +# shell/kill/get-ip helpers find instances launched by a CI run for this repo. +repo=${GITHUB_REPOSITORY##*/} +repo=${repo:-aztec-packages} +instance_name=${INSTANCE_NAME:-${repo}_$(echo -n "$BRANCH" | tr -c 'a-zA-Z0-9-' '_')_${arch}} [ -n "${INSTANCE_POSTFIX:-}" ] && instance_name+="_$INSTANCE_POSTFIX" function get_ip_for_instance { diff --git a/ci3/bootstrap_ec2 b/ci3/bootstrap_ec2 index a15113a4c126..f5ce6c62868b 100755 --- a/ci3/bootstrap_ec2 +++ b/ci3/bootstrap_ec2 @@ -65,11 +65,18 @@ if [[ "$(git fetch origin --negotiate-only --negotiation-tip="$current_commit")" fi # Our instance_name acts as a uniqueness key for the instance. -# Instances are terminated if they exist with the same name. +# Instances are terminated if they exist with the same name; this reaps orphans +# left when a GA run is cancelled (e.g. by a new push) on the same ref. +# Scope the key to the repo: aztec-packages and aztec-packages-private can build +# the same tag/ref concurrently under the same role, and must not reap each +# other's instances. The key stays stable across re-runs within a repo, so the +# orphan cleanup still works. +repo=${GITHUB_REPOSITORY##*/} +repo=${repo:-aztec-packages} if [[ "$REF_NAME" =~ ^gh-readonly-queue/.*(pr-[0-9]+) ]]; then - instance_name="${BASH_REMATCH[1]}_$arch" + instance_name="${repo}_${BASH_REMATCH[1]}_$arch" else - instance_name=$(echo -n "$REF_NAME" | head -c 50 | tr -c 'a-zA-Z0-9-' '_')_$arch + instance_name="${repo}_$(echo -n "$REF_NAME" | head -c 50 | tr -c 'a-zA-Z0-9-' '_')_$arch" fi state_dir=$(mktemp -d /tmp/aws_request_instance.XXXXXX) From 8bfd77df3b4b7ccef48999489736ad4f84b0a7e8 Mon Sep 17 00:00:00 2001 From: Josh Crites Date: Wed, 10 Jun 2026 12:26:55 -0400 Subject: [PATCH 04/12] docs: update sponsored fpc address --- .claude/skills/release-docs/SKILL.md | 10 ++++++---- .claude/skills/release-network-docs/SKILL.md | 3 ++- .../version-v4.3.1/getting_started_on_testnet.md | 2 +- docs/docs-developers/getting_started_on_testnet.md | 2 +- docs/docs/networks.md | 2 +- 5 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.claude/skills/release-docs/SKILL.md b/.claude/skills/release-docs/SKILL.md index 6eb78293f32d..75c305769bb5 100644 --- a/.claude/skills/release-docs/SKILL.md +++ b/.claude/skills/release-docs/SKILL.md @@ -79,14 +79,14 @@ returned by the RPC (e.g. the network is still running `4.1.3` but the user wants to prepare docs for `4.2.0`), this is a **pre-release** docs preparation. Ask the user to confirm the target version, then use that version instead of `nodeVersion` throughout the remaining steps. The git tag for the target version -must still exist. Contract addresses from the RPC reflect the *current* network +must still exist. Contract addresses from the RPC reflect the _current_ network state (the old version); they are still valid if the upgrade reuses the same contracts, but ask the user to confirm whether any addresses will change at upgrade time. **Run all work on the tag, not `next`.** Cut on the tag so the snapshot reflects what shipped. Then stash, switch to `next`, pop. Backport any newer -docs from `next` into the snapshot as an explicit step *after* the cut. +docs from `next` into the snapshot as an explicit step _after_ the cut. ### Unversioned root pages @@ -116,9 +116,9 @@ VERSION= bash -i <(curl -sL https://install.aztec.network/) aztec get-canonical-sponsored-fpc-address ``` -Store the address for updating docs. +Store the address for updating docs. Be sure to update the address with the appropriate value wherever it appears in the versioned docs. -**Note:** The Sponsored FPC is only deployed on devnet. For mainnet and testnet releases, +**Note:** The Sponsored FPC is deployed on testnet and devnet. For mainnet releases, mark the SponsoredFPC row as "Not deployed" in the L2 Contract Addresses table. ### Step 5: Update Version Configs @@ -234,6 +234,7 @@ docs (Step 13), the generated content is included in the snapshot automatically. 1. **Triage existing TBD items.** Not all items under `## TBD` necessarily belong to the current release. Review each entry and decide whether it: + - Shipped in this release โ†’ move it under the new `## ` heading - Targets a future major version โ†’ move it under a new `## Unreleased (v)` heading (create this heading if it doesn't exist, placed between `## TBD` and @@ -250,6 +251,7 @@ docs (Step 13), the generated content is included in the snapshot automatically. 4. Check for missing migration items by analyzing the diff between the previous release tag and the new one: + ```bash git diff v..v -- yarn-project/ noir-projects/ ``` diff --git a/.claude/skills/release-network-docs/SKILL.md b/.claude/skills/release-network-docs/SKILL.md index eeaf2b6ce24e..2c0b07599e2d 100644 --- a/.claude/skills/release-network-docs/SKILL.md +++ b/.claude/skills/release-network-docs/SKILL.md @@ -144,13 +144,14 @@ in each table: - **L1 Contract Addresses table**: update all addresses from the RPC response, on-chain queries, and any additional addresses provided by the user. + - Mainnet: use `https://etherscan.io/address/0xADDR` link format - Testnet: use `https://sepolia.etherscan.io/address/0xADDR` link format - For contracts that are not deployed on this network, use `N/A` - **L2 Contract Addresses table**: update if any canonical protocol contract addresses changed (check the `protocolContractAddresses` from the RPC - response). SponsoredFPC is always "Not deployed" on mainnet and testnet. + response). SponsoredFPC is always "Not deployed" on mainnet. Also grep for any old addresses that may appear elsewhere in the docs: diff --git a/docs/developer_versioned_docs/version-v4.3.1/getting_started_on_testnet.md b/docs/developer_versioned_docs/version-v4.3.1/getting_started_on_testnet.md index 9d755acd9ae0..6bc673862b36 100644 --- a/docs/developer_versioned_docs/version-v4.3.1/getting_started_on_testnet.md +++ b/docs/developer_versioned_docs/version-v4.3.1/getting_started_on_testnet.md @@ -53,7 +53,7 @@ Set the required environment variables: ```bash export NODE_URL=https://rpc.testnet.aztec-labs.com -export SPONSORED_FPC_ADDRESS=0x254082b62f9108d044b8998f212bb145619d91bfcd049461d74babb840181257 +export SPONSORED_FPC_ADDRESS=0x08b888c4be63ed67f61a622fdd013ea028326bac22a8982a3b5a7e9ec62f765b ``` ### Step 2: Register the Sponsored FPC diff --git a/docs/docs-developers/getting_started_on_testnet.md b/docs/docs-developers/getting_started_on_testnet.md index 5f66aff0aa21..674850a5fab2 100644 --- a/docs/docs-developers/getting_started_on_testnet.md +++ b/docs/docs-developers/getting_started_on_testnet.md @@ -53,7 +53,7 @@ Set the required environment variables: ```bash export NODE_URL=https://rpc.testnet.aztec-labs.com -export SPONSORED_FPC_ADDRESS=0x254082b62f9108d044b8998f212bb145619d91bfcd049461d74babb840181257 +export SPONSORED_FPC_ADDRESS=0x08b888c4be63ed67f61a622fdd013ea028326bac22a8982a3b5a7e9ec62f765b ``` ### Step 2: Register the Sponsored FPC diff --git a/docs/docs/networks.md b/docs/docs/networks.md index ffa2d8261fe4..7c3a140925ca 100644 --- a/docs/docs/networks.md +++ b/docs/docs/networks.md @@ -85,7 +85,7 @@ Stable releases target roughly one per month, typically mid-month. Dates are not | **Class Registry** | `0x0000000000000000000000000000000000000000000000000000000000000003` | `0x0000000000000000000000000000000000000000000000000000000000000003` | | **MultiCall Entrypoint** | `0x0000000000000000000000000000000000000000000000000000000000000004` | `0x0000000000000000000000000000000000000000000000000000000000000004` | | **Fee Juice** | `0x0000000000000000000000000000000000000000000000000000000000000005` | `0x0000000000000000000000000000000000000000000000000000000000000005` | -| **SponsoredFPC** | Not deployed | `0x254082b62f9108d044b8998f212bb145619d91bfcd049461d74babb840181257` | +| **SponsoredFPC** | Not deployed | `0x08b888c4be63ed67f61a622fdd013ea028326bac22a8982a3b5a7e9ec62f765b` | ## Governance parameters From 2ed6d2b99bb8b9289f3ba6d5e2f56a25dc08623e Mon Sep 17 00:00:00 2001 From: Josh Crites Date: Wed, 10 Jun 2026 16:41:51 -0400 Subject: [PATCH 05/12] add reminder about funding testnet fpc --- .claude/skills/release-docs/SKILL.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.claude/skills/release-docs/SKILL.md b/.claude/skills/release-docs/SKILL.md index 75c305769bb5..88d7a10b63b1 100644 --- a/.claude/skills/release-docs/SKILL.md +++ b/.claude/skills/release-docs/SKILL.md @@ -120,6 +120,7 @@ Store the address for updating docs. Be sure to update the address with the appr **Note:** The Sponsored FPC is deployed on testnet and devnet. For mainnet releases, mark the SponsoredFPC row as "Not deployed" in the L2 Contract Addresses table. +If the Sponsored FPC address changes for a testnet release, send a reminder that the new address must be funded on testnet. ### Step 5: Update Version Configs From 56168b316689dab063d7bddd9bef273b6692b368 Mon Sep 17 00:00:00 2001 From: AztecBot <49558828+AztecBot@users.noreply.github.com> Date: Thu, 11 Jun 2026 05:56:16 +0000 Subject: [PATCH 06/12] chore(ci): public nightlies skip scenario tests and the next tag ## What Two changes scoped to the **public** repo (`AztecProtocol/aztec-packages`) nightly flow, plus a follow-up tightening of the scenario-test trigger. Private tagging is unchanged. ### 1. Network scenario tests run only on the private v5-next nightly `ci3.yml`'s `ci-network-scenario` job fired on any current nightly tag in both repos. Private produces both a `next` (v6) and a `v5-next` (v5) nightly tag, so simply gating to the private repo still ran scenarios against the v6 nightly. The nightly-triggered path is now gated to **private repo + a `v5.` nightly tag**: ```yaml ( needs.validate-nightly-tag.outputs.is_current == 'true' && github.repository == 'AztecProtocol/aztec-packages-private' && startsWith(github.ref_name, 'v5.') ) || contains(github.event.pull_request.labels.*.name, 'ci-network-scenario') ``` `v5-next` is at `5.x.x` (tag `v5.x.x-nightly.*`) and `next` is at `6.x.x` (tag `v6.x.x-nightly.*`), so `startsWith(github.ref_name, 'v5.')` selects the v5-next nightly only. The manual PR-label path (`ci-network-scenario`) is preserved for ad-hoc dev runs. ### 2. Stop tagging `next` with a nightly tag in public `nightly-release-tag.yml`'s matrix tagged `[next, v5-next]` in both repos. The branch list is now repo-dependent: private keeps `[next, v5-next]`, public tags only `v5-next` (and `v4-next` via its existing dedicated job). Net result: **public tags `v4-next` + `v5-next` only**, private is untouched. ## Why Nightly network scenario tests should run only against the private v5-next nightly, and public should not produce a `next` nightly tag. --- .github/workflows/ci3.yml | 8 ++++++-- .github/workflows/nightly-release-tag.yml | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci3.yml b/.github/workflows/ci3.yml index d1c42e337e99..3234f4d9bf7e 100644 --- a/.github/workflows/ci3.yml +++ b/.github/workflows/ci3.yml @@ -208,7 +208,7 @@ jobs: fail-fast: false matrix: test_set: ["1", "2"] - # We run on current nightly tags only, or when the ci-network-scenario label is present in a PR. + # We run on the current v5-next nightly tag (private repo only), or when the ci-network-scenario label is present in a PR. needs: [ci, validate-nightly-tag] if: | always() @@ -216,7 +216,11 @@ jobs: && github.event.pull_request.head.repo.fork != true && github.event.pull_request.draft == false && ( - needs.validate-nightly-tag.outputs.is_current == 'true' + ( + needs.validate-nightly-tag.outputs.is_current == 'true' + && github.repository == 'AztecProtocol/aztec-packages-private' + && startsWith(github.ref_name, 'v5.') + ) || contains(github.event.pull_request.labels.*.name, 'ci-network-scenario') ) steps: diff --git a/.github/workflows/nightly-release-tag.yml b/.github/workflows/nightly-release-tag.yml index c0627849dcc2..e330145308ee 100644 --- a/.github/workflows/nightly-release-tag.yml +++ b/.github/workflows/nightly-release-tag.yml @@ -15,7 +15,9 @@ jobs: strategy: fail-fast: false matrix: - branch: [next, v5-next] + # Public only tags v5-next here (v4-next is handled in its own job below); + # next is tagged on the private repo only. + branch: ${{ github.repository == 'AztecProtocol/aztec-packages-private' && fromJSON('["next", "v5-next"]') || fromJSON('["v5-next"]') }} concurrency: group: ${{ github.workflow }}-${{ github.repository }}-${{ matrix.branch }} steps: From 3d6efdf765cf1a3d4e7b48d11b3d0192d59c22a1 Mon Sep 17 00:00:00 2001 From: AztecBot <49558828+AztecBot@users.noreply.github.com> Date: Thu, 11 Jun 2026 10:38:31 +0000 Subject: [PATCH 07/12] fix(spartan): wait_for_ci3 finds aged runs and proceeds once CI3 completes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem A devnet deploy failed waiting on CI3 for two distinct reasons: 1. **Lookup window bug.** The script used `gh run list --workflow ci3.yml` (which returns only ~20 newest runs) and filtered by `headSha` client-side. By the time the deploy polled, the 03:04 nightly run had aged off that first page, so the match never fired and the script timed out โ€” even though the run existed. 2. **Conclusion gated the deploy.** Even once found, `gh run watch --exit-status` would fail the deploy if the CI3 nightly itself was red (e.g. #2208). The nightly bundles many jobs, so an unrelated red job blocked release even though the release build was fine. ## Fix 1. Query `repos//actions/workflows/ci3.yml/runs?head_sha=` via `gh api`, which filters server-side by SHA and finds the run instantly no matter how old it is. 2. Drop `--exit-status` from `gh run watch` (so the whole-run conclusion no longer gates), and instead gate specifically on the two release jobs โ€” the `./bootstrap.sh ci-release` builds on amd64 (`ci/x-release`) and arm64 (`ci/a-release`). These are posted as **GitHub commit statuses** on the tag's commit by `ci3/bootstrap_ec2` (`post_github_status ci/`). The script now waits for both statuses to reach a terminal state (polling, since the runner posts them asynchronously) and fails only if either is not `success`. It still fails if no CI3 run ever appears for the tag. The deploy now proceeds iff CI3 ran **and** both release-build jobs succeeded, independent of unrelated nightly failures. --- spartan/scripts/wait_for_ci3.ts | 90 +++++++++++++++++++++++++++++++-- 1 file changed, 87 insertions(+), 3 deletions(-) diff --git a/spartan/scripts/wait_for_ci3.ts b/spartan/scripts/wait_for_ci3.ts index e0a77e415147..afa42d21b1f7 100755 --- a/spartan/scripts/wait_for_ci3.ts +++ b/spartan/scripts/wait_for_ci3.ts @@ -13,6 +13,13 @@ * 1. Resolves the tag's SHA via `gh api` * 2. Polls for up to 10 minutes for a CI3 run matching that SHA * 3. Uses `gh run watch` to stream the run to completion + * 4. Gates the deploy on the two release jobs (`ci/x-release`, `ci/a-release`) + * succeeding, rather than on the overall run conclusion + * + * The overall CI3 nightly conclusion does NOT gate the deploy: it bundles many + * jobs and an unrelated red job would otherwise block release. We only care + * that the release-build jobs (`./bootstrap.sh ci-release` on amd64 + arm64, + * reported as commit statuses on the tag's commit) succeeded. * * Writes run_id to GITHUB_OUTPUT when running in CI. */ @@ -65,8 +72,12 @@ async function main(): Promise { let runId = ""; for (let i = 1; i <= maxAttempts; i++) { + // Query the workflow's runs filtered server-side by head_sha. This finds + // the run no matter how far down the run history it has aged โ€” unlike + // `gh run list` (which only returns ~20 newest and would miss an older + // nightly run that has since been pushed off the first page). const result = execSync( - `gh run list --repo ${repo} --workflow ci3.yml --json headSha,databaseId --jq '.[] | select(.headSha == "${commitSha}") | .databaseId'`, + `gh api "repos/${repo}/actions/workflows/ci3.yml/runs?head_sha=${commitSha}" --jq '.workflow_runs[].id'`, { encoding: "utf-8" }, ).trim(); @@ -92,11 +103,84 @@ async function main(): Promise { // 3. Write output for CI writeGithubOutputs({ run_id: runId }); - // 4. Watch the run to completion + // 4. Watch the run to completion. + // + // Deliberately omit `--exit-status`: we don't gate on the overall run + // conclusion. The CI3 nightly bundles many jobs, and an unrelated red job + // (e.g. a flaky nightly test) would otherwise fail this step and block the + // deploy. `gh run watch` (without the flag) exits 0 once the run reaches a + // completed status; we gate on the specific release jobs below instead. console.log(`Watching CI3 run ${runId}...`); - execSync(`gh run watch ${runId} --repo ${repo} --exit-status`, { + execSync(`gh run watch ${runId} --repo ${repo}`, { stdio: "inherit", }); + + // 5. Gate the deploy on the two release jobs. + // + // The release flow (ci.sh `release`) runs `./bootstrap.sh ci-release` on an + // amd64 (x-release) and an arm64 (a-release) EC2 instance. Each posts a + // GitHub commit status `ci/` on the tag's commit (see + // ci3/bootstrap_ec2 -> post_github_status). We require both to be `success`; + // those are the jobs that actually build and publish the release artifacts. + const requiredContexts = ["ci/x-release", "ci/a-release"]; + await gateOnCommitStatuses(commitSha, requiredContexts); +} + +/** + * Wait for the given commit-status contexts to reach a terminal state on the + * commit, then fail (exit 1) unless all of them are `success`. Polls briefly + * because the runner posts these statuses asynchronously, so they may land a + * moment after the workflow run completes. + */ +async function gateOnCommitStatuses( + commitSha: string, + requiredContexts: string[], +): Promise { + const terminal = new Set(["success", "failure", "error"]); + const maxAttempts = 30; // up to ~5 minutes at 10s intervals + let states: Record = {}; + + for (let i = 1; i <= maxAttempts; i++) { + // The combined endpoint returns the latest status per context. + const statuses: Array<{ context: string; state: string }> = JSON.parse( + execSync( + `gh api repos/${repo}/commits/${commitSha}/status --jq '[.statuses[] | {context, state}]'`, + { encoding: "utf-8" }, + ), + ); + + states = {}; + for (const { context, state } of statuses) { + states[context] = state; + } + + const allTerminal = requiredContexts.every((c) => + terminal.has(states[c]), + ); + if (allTerminal) { + break; + } + + const missing = requiredContexts.filter((c) => !terminal.has(states[c])); + console.log( + `Attempt ${i}/${maxAttempts}: waiting on release statuses ${missing.join(", ")} (current: ${missing.map((c) => `${c}=${states[c] ?? "absent"}`).join(", ")})...`, + ); + await sleep(10_000); + } + + const failed = requiredContexts.filter((c) => states[c] !== "success"); + for (const context of requiredContexts) { + console.log(`Release status ${context}: ${states[context] ?? "absent"}`); + } + + if (failed.length > 0) { + console.error( + `Error: release job(s) did not succeed: ${failed.map((c) => `${c}=${states[c] ?? "absent"}`).join(", ")}`, + ); + process.exit(1); + } + + console.log("All release jobs succeeded."); } main(); From 5bb74e91ea655a3d79bbd440e2ac5b00386e3321 Mon Sep 17 00:00:00 2001 From: AztecBot <49558828+AztecBot@users.noreply.github.com> Date: Thu, 11 Jun 2026 14:04:43 +0000 Subject: [PATCH 08/12] chore: deploy staging from `next` scripts/helm MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## What Make all three nightly deployments run the deploy from the tip of `next` (latest scripts + helm) while keeping the correct image for each target network. ### Deploy ref โ†’ `next` - `deploy-staging-internal.yml`, `deploy-staging-public.yml`: pass `ref: next` to `deploy-network.yml` so the `spartan/` deploy scripts and helm charts come from `next`. - `deploy-next-net.yml` already passed `ref: next` (unchanged). ### `determine-tag` job (staging) - Checkout a single commit at the tip of `next` (`ref: next`, `fetch-depth: 1`) instead of `v5-next` with full history. - Tag resolution: if an explicit `tag` input is given, use it as-is. Otherwise construct `v5.0.0-nightly.` and verify it actually exists with `git ls-remote --exit-code --tags origin`, failing the deploy early if the nightly tag is missing rather than proceeding to deploy a non-existent image. ## Why `deploy-network.yml` checks out `inputs.ref` to run the deploy scripts/helm; when unset it falls back to `github.ref` (default branch on `schedule`, dispatch branch on `workflow_dispatch`), making the scripts/helm implicit and branch-dependent. Pinning to `next` keeps staging on the latest infra while `semver`/`source_tag` continue to select the v5-line image (`v5.0.0-nightly.`), which is the correct image for the staging networks. The `v5.0.0-nightly.` tag is created on both the public and private repos (the nightly tagger tags `v5-next` on both), so the `git ls-remote origin` check resolves against whichever repo the workflow runs in. --- .github/workflows/deploy-staging-internal.yml | 12 +++++++++--- .github/workflows/deploy-staging-public.yml | 12 +++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.github/workflows/deploy-staging-internal.yml b/.github/workflows/deploy-staging-internal.yml index d5f3eb127984..18194d96f9eb 100644 --- a/.github/workflows/deploy-staging-internal.yml +++ b/.github/workflows/deploy-staging-internal.yml @@ -34,12 +34,12 @@ jobs: tag: ${{ steps.resolve.outputs.tag }} semver: ${{ steps.resolve.outputs.semver }} steps: - - name: Checkout v5-next + - name: Checkout next uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: - ref: v5-next + ref: next token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - fetch-depth: 0 + fetch-depth: 1 - name: Resolve nightly tag id: resolve @@ -50,6 +50,11 @@ jobs: else TAG="v5.0.0-nightly.$(date -u +%Y%m%d)" echo "Using today's nightly tag: $TAG" + if ! git ls-remote --exit-code --tags origin "refs/tags/$TAG" >/dev/null; then + echo "Error: nightly tag $TAG does not exist on origin" + exit 1 + fi + echo "Confirmed nightly tag $TAG exists on origin" fi SEMVER="${TAG#v}" @@ -82,6 +87,7 @@ jobs: network: staging-internal semver: ${{ needs.determine-tag.outputs.semver }} source_tag: ${{ needs.determine-tag.outputs.tag }} + ref: next deploy_contracts: ${{ inputs.deploy_contracts == true }} use_internal_docker_registry: true secrets: inherit diff --git a/.github/workflows/deploy-staging-public.yml b/.github/workflows/deploy-staging-public.yml index 8aaea198d33c..65c094f9534e 100644 --- a/.github/workflows/deploy-staging-public.yml +++ b/.github/workflows/deploy-staging-public.yml @@ -34,12 +34,12 @@ jobs: tag: ${{ steps.resolve.outputs.tag }} semver: ${{ steps.resolve.outputs.semver }} steps: - - name: Checkout v5-next + - name: Checkout next uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: - ref: v5-next + ref: next token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - fetch-depth: 0 + fetch-depth: 1 - name: Resolve nightly tag id: resolve @@ -50,6 +50,11 @@ jobs: else TAG="v5.0.0-nightly.$(date -u +%Y%m%d)" echo "Using today's nightly tag: $TAG" + if ! git ls-remote --exit-code --tags origin "refs/tags/$TAG" >/dev/null; then + echo "Error: nightly tag $TAG does not exist on origin" + exit 1 + fi + echo "Confirmed nightly tag $TAG exists on origin" fi SEMVER="${TAG#v}" @@ -82,5 +87,6 @@ jobs: network: staging-public semver: ${{ needs.determine-tag.outputs.semver }} source_tag: ${{ needs.determine-tag.outputs.tag }} + ref: next deploy_contracts: ${{ inputs.deploy_contracts == true }} secrets: inherit From a62c05977ff856b150d8d6e162b615a86d20e258 Mon Sep 17 00:00:00 2001 From: AztecBot Date: Thu, 11 Jun 2026 16:51:24 +0000 Subject: [PATCH 09/12] docs(CLAUDE.md): discourage unprompted subagents and dynamic workflows --- CLAUDE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 96b74c043987..d4df966e403d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,6 +83,10 @@ Tests should validate behavior, not mock call-count. Prefer `expect(result).toEq Before writing a new helper, utility, or component, search for an existing one with Grep or Glob. Reuse or refactor to a shared module; do not introduce a parallel implementation. + +Do the work in this session by default. Do not spawn parallel subagents (the Agent/Task tool) or launch dynamic workflows (the Workflow tool) unless the user explicitly asks for it. Each extra agent multiplies token spend โ€” roughly 2x for one helper and far more when a request fans out to many โ€” and the user cannot see the fan-out coming or stop it; a single prompt that quietly started ~30 agents has exhausted an operator's budget. Searching the codebase, summarizing, researching, and ordinary multi-file edits are inline work: run the tool calls yourself. Reach for a subagent only when the user requested orchestration, or when one clearly-scoped read-heavy helper genuinely needs isolation from the main context โ€” prefer a single agent over many, and never start a dynamic workflow by default. If a task would benefit from parallel agents but the user has not asked, either do it directly or describe the multi-agent option and ask before spending the budget. + + Preserve existing `// TODO`, `// TODO(name)`, and `// NOTE:` comments unless the current task is to resolve them. A "tidy up" refactor that deletes another author's deferred-work markers destroys context that is not recoverable from git history. From 8aed7c54bfd2c8c80c35ac507031dda2588718b8 Mon Sep 17 00:00:00 2001 From: AztecBot Date: Thu, 11 Jun 2026 19:08:02 +0000 Subject: [PATCH 10/12] docs: update Aztec & Noir Developer Office Hours Google Meet link --- docs/docs-developers/docs/resources/community_calls.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs-developers/docs/resources/community_calls.md b/docs/docs-developers/docs/resources/community_calls.md index e4b1ce02e3f1..954031a15068 100644 --- a/docs/docs-developers/docs/resources/community_calls.md +++ b/docs/docs-developers/docs/resources/community_calls.md @@ -21,7 +21,7 @@ Every week you can join office hours and ecosystem calls to get unblocked, learn ## Aztec & Noir Developer Office Hours - **When:** Thursdays ยท 14:00 - 15:00 UTC -- **Where:** [Google Meet](https://meet.google.com/sdd-rdsr-shu) +- **Where:** [Google Meet](https://meet.google.com/vev-waao-mab) - **For:** Developers building with Aztec.nr smart contracts or writing and debugging Noir. Bring your questions about syntax, tooling, patterns, or protocol-level topics. Share a project you're working on, or just hang out with the Aztec Labs Dev Rel team and other devs. --- From c11b0849ecb03494bedb6d1f923ae70c91d2089c Mon Sep 17 00:00:00 2001 From: AztecBot Date: Thu, 11 Jun 2026 19:24:58 +0000 Subject: [PATCH 11/12] update PR #24033 --- .../version-v4.3.1/docs/resources/community_calls.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/developer_versioned_docs/version-v4.3.1/docs/resources/community_calls.md b/docs/developer_versioned_docs/version-v4.3.1/docs/resources/community_calls.md index e4b1ce02e3f1..954031a15068 100644 --- a/docs/developer_versioned_docs/version-v4.3.1/docs/resources/community_calls.md +++ b/docs/developer_versioned_docs/version-v4.3.1/docs/resources/community_calls.md @@ -21,7 +21,7 @@ Every week you can join office hours and ecosystem calls to get unblocked, learn ## Aztec & Noir Developer Office Hours - **When:** Thursdays ยท 14:00 - 15:00 UTC -- **Where:** [Google Meet](https://meet.google.com/sdd-rdsr-shu) +- **Where:** [Google Meet](https://meet.google.com/vev-waao-mab) - **For:** Developers building with Aztec.nr smart contracts or writing and debugging Noir. Bring your questions about syntax, tooling, patterns, or protocol-level topics. Share a project you're working on, or just hang out with the Aztec Labs Dev Rel team and other devs. --- From 3b462ff658f9465ac50f3fd65d75f6b7fb0aacfb Mon Sep 17 00:00:00 2001 From: AztecBot Date: Thu, 11 Jun 2026 21:46:47 +0000 Subject: [PATCH 12/12] fix(ci): refresh grind launcher checkout to origin/next before launching --- ci3/dashboard/rk.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/ci3/dashboard/rk.py b/ci3/dashboard/rk.py index ac1f40e3f50d..f3354d146268 100644 --- a/ci3/dashboard/rk.py +++ b/ci3/dashboard/rk.py @@ -556,6 +556,26 @@ def make_options(param_name, options, current_value, suffix=''): # Dashboard server needs local repo checkout at REPO_PATH repo_path = os.environ.get('REPO_PATH') if repo_path: + # Refresh the launcher checkout to current origin/next before launching. + # REPO_PATH only supplies the orchestration scripts (ci.sh/bootstrap_ec2); + # the grind target commit is checked out on the remote box. The launcher + # must stay current so grind uses the same transport (SSM) as the rest of + # CI -- a drifted checkout silently falls back to the retired SSH path and + # every instance times out waiting for SSH. + refresh = subprocess.run( + ['git', '-C', repo_path, 'fetch', '--quiet', 'origin', 'next'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) + if refresh.returncode == 0: + refresh = subprocess.run( + ['git', '-C', repo_path, 'checkout', '--quiet', '--force', 'origin/next'], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True + ) + if refresh.returncode != 0: + r.setex(run_id, 86400, + f'Failed to refresh launcher checkout at {repo_path}:\n{refresh.stdout}\n'.encode()) + return redirect(f'/{run_id}') + subprocess.Popen( ['bash', '-c', f'cd {repo_path} && RUN_ID={run_id} CPUS={cpus} ./ci.sh grind-test {shlex.quote(full_cmd)} {grind_time} {jobs_pct} {memsuspend_pct} {commit}'], stdout=subprocess.DEVNULL,