From 01f1bc5e9a8032d7cb9320131173531c940bb6b9 Mon Sep 17 00:00:00 2001 From: alexghr <3816165+alexghr@users.noreply.github.com> Date: Tue, 9 Jun 2026 17:04:10 +0000 Subject: [PATCH] chore: deployments Fix A-1163 --- .github/workflows/ci3.yml | 12 ++- .github/workflows/deploy-network.yml | 24 ++--- .github/workflows/deploy-next-net.yml | 43 +++------ .github/workflows/deploy-staging-internal.yml | 87 +++++++++++++++++ ...-staging.yml => deploy-staging-public.yml} | 12 +-- .../workflows/ensure-funded-environment.yml | 5 +- .../workflows/ensure-funded-environments.yml | 3 +- .github/workflows/network-healthcheck.yml | 4 +- .github/workflows/nightly-bench-10tps.yml | 2 +- .../workflows/nightly-release-tag-v4-next.yml | 33 ------- .github/workflows/nightly-release-tag.yml | 42 ++++++-- .github/workflows/nightly-spartan-bench.yml | 68 ++++++++----- .github/workflows/test-network-scenarios.yml | 7 ++ .github/workflows/weekly-proving-bench.yml | 2 +- spartan/.gitignore | 5 +- spartan/environments/next-net.env | 4 +- .../{staging.env => staging-internal.env} | 22 ++--- spartan/environments/staging-public.env | 96 +++++++++++++++++++ spartan/terraform/gke-cluster/iam.tf | 24 +++++ 19 files changed, 345 insertions(+), 150 deletions(-) create mode 100644 .github/workflows/deploy-staging-internal.yml rename .github/workflows/{deploy-staging.yml => deploy-staging-public.yml} (89%) delete mode 100644 .github/workflows/nightly-release-tag-v4-next.yml rename spartan/environments/{staging.env => staging-internal.env} (77%) create mode 100644 spartan/environments/staging-public.env diff --git a/.github/workflows/ci3.yml b/.github/workflows/ci3.yml index 8cd1128fddbc..d1c42e337e99 100644 --- a/.github/workflows/ci3.yml +++ b/.github/workflows/ci3.yml @@ -250,6 +250,7 @@ jobs: DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + INTERNAL_DOCKER_REGISTRY: ${{ secrets.INTERNAL_DOCKER_REGISTRY }} CI3_INSTANCE_PROFILE_NAME: ${{ secrets.CI3_INSTANCE_PROFILE_NAME }} CI3_SECURITY_GROUP_ID: ${{ secrets.CI3_SECURITY_GROUP_ID }} RUN_ID: ${{ github.run_id }} @@ -265,7 +266,16 @@ jobs: tag="${tag#v}" major_version=$(./ci3/semver major "$tag") namespace="v${major_version}-scenario" - docker_image="aztecprotocol/aztec:${tag}" + case "$GITHUB_REPOSITORY" in + AztecProtocol/aztec-packages-private) + image_registry="${INTERNAL_DOCKER_REGISTRY%/}" + echo "::add-mask::${image_registry}" + ;; + *) + image_registry="aztecprotocol" + ;; + esac + docker_image="${image_registry}/aztec:${tag}" else # branch name namespace=pr-$(echo "$HEAD_REF" | sed 's/[^a-z0-9-]/-/g' | cut -c1-20 | sed 's/-*$//') diff --git a/.github/workflows/deploy-network.yml b/.github/workflows/deploy-network.yml index c1d0130b0256..3b96f8654db8 100644 --- a/.github/workflows/deploy-network.yml +++ b/.github/workflows/deploy-network.yml @@ -47,11 +47,11 @@ on: description: "Source tag that triggered this deploy" required: false type: string - notify_on_failure: - description: "Whether this workflow should send its own failure notification" + skip_notify_on_failure: + description: "Whether this workflow should skip its own failure notification" required: false type: boolean - default: true + default: false workflow_dispatch: inputs: network: @@ -60,7 +60,8 @@ on: type: choice options: - next-net - - staging + - staging-public + - staging-internal - testnet - mainnet semver: @@ -97,11 +98,6 @@ on: description: "Source tag that triggered this deploy" required: false type: string - notify_on_failure: - description: "Whether this workflow should send its own failure notification" - required: false - type: boolean - default: true concurrency: group: deploy-network-${{ inputs.network }}-${{ inputs.namespace || inputs.network }}-${{ inputs.aztec_docker_image || inputs.semver }}-${{ github.ref || github.ref_name }} @@ -254,8 +250,6 @@ jobs: VALIDATOR_HA_DOCKER_IMAGE: ${{ inputs.ha_docker_image || '' }} run: | echo "Deploying network: ${{ inputs.network }}" - echo "Using image: $AZTEC_DOCKER_IMAGE" - echo "Using prover image: $PROVER_AGENT_DOCKER_IMAGE" echo "Using branch/ref: ${{ steps.checkout-ref.outputs.ref }}" cd spartan @@ -280,7 +274,6 @@ jobs: echo "| Item | Value |" echo "|------|-------|" echo "| Network | \`${{ inputs.network }}\` |" - echo "| Docker Image | \`${{ env.AZTEC_DOCKER_IMAGE }}\` |" echo "| Ref | \`${{ steps.checkout-ref.outputs.ref }}\` |" if [[ -n "${{ inputs.source_tag }}" ]]; then echo "| Source Tag | [\`${{ inputs.source_tag }}\`](https://github.com/${{ github.repository }}/releases/tag/${{ inputs.source_tag }}) |" @@ -288,7 +281,7 @@ jobs: } >> "$GITHUB_STEP_SUMMARY" - name: Notify Slack and dispatch ClaudeBox on failure - if: failure() && inputs.notify_on_failure + if: failure() && !inputs.skip_notify_on_failure env: SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} @@ -300,7 +293,7 @@ jobs: CHANNEL="#alerts-${{ inputs.network }}" RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" - TEXT="Deploy Network workflow FAILED for *${{ inputs.network }}* (image ${{ env.AZTEC_DOCKER_IMAGE }}): <${RUN_URL}|View Run> (🤖)" + TEXT="Deploy Network workflow FAILED for *${{ inputs.network }}*: <${RUN_URL}|View Run> (🤖)" # Post to Slack and capture timestamp for permalink RESP=$(curl -sS -X POST https://slack.com/api/chat.postMessage \ @@ -318,11 +311,10 @@ jobs: fi # Dispatch ClaudeBox to investigate the failure - PROMPT="Deployment of ${{ inputs.network }} (image ${{ env.AZTEC_DOCKER_IMAGE }}) failed. \ + PROMPT="Deployment of ${{ inputs.network }} failed. \ Follow .claude/claudebox/deploy-investigation.md to investigate. \ GitHub Actions run: ${RUN_URL}. \ Network: ${{ inputs.network }}. \ - Docker image: ${{ env.AZTEC_DOCKER_IMAGE }}. \ Git ref: ${{ steps.checkout-ref.outputs.ref }}. \ Namespace: ${{ inputs.namespace || inputs.network }}. \ Deploy contracts: ${{ inputs.deploy_contracts }}." diff --git a/.github/workflows/deploy-next-net.yml b/.github/workflows/deploy-next-net.yml index fc40cec81619..b06d064986c9 100644 --- a/.github/workflows/deploy-next-net.yml +++ b/.github/workflows/deploy-next-net.yml @@ -1,6 +1,6 @@ # Deploy next-net environment # This workflow deploys the next-net environment with a specified version -# Runs nightly with the latest nightly tag, or can be manually triggered with any image +# Runs from the private repo's next branch with the latest nightly tag, or a manually supplied tag name: Deploy Next Net on: @@ -10,7 +10,7 @@ on: workflow_dispatch: inputs: image_tag: - description: "Docker image tag (e.g., 2.3.4, 3.0.0-nightly.20251004-amd64, or leave empty for latest nightly)" + description: "Docker image tag/semver (e.g., 6.0.0-nightly.20260609, or leave empty for latest nightly)" required: false type: string deploy_contracts: @@ -25,57 +25,42 @@ concurrency: jobs: get-image-tag: - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest outputs: - tag: ${{ steps.determine_tag.outputs.TAG }} semver: ${{ steps.determine_tag.outputs.SEMVER }} steps: - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: next - name: Determine image tag id: determine_tag run: | if [[ -n "${{ inputs.image_tag }}" ]]; then - # Manual trigger with specified tag - TAG="${{ inputs.image_tag }}" - echo "Using manually specified tag: $TAG" - - # Extract semver (remove -amd64 suffix if present) - SEMVER=$(echo "$TAG" | sed 's/-amd64$//') + SEMVER="${{ inputs.image_tag }}" + SEMVER="${SEMVER#v}" + SEMVER="${SEMVER%-amd64}" + echo "Using manually specified tag: $SEMVER" else - # Scheduled nightly run - get latest nightly tag current_version=$(jq -r '."."' .release-please-manifest.json) echo "Current version: $current_version" - # Format the tag as: -nightly.-amd64 - nightly_semver="${current_version}-nightly.$(date -u +%Y%m%d)" - nightly_tag="${nightly_semver}-amd64" - - # Check if the tag exists on docker hub - TAGS=$(curl -s https://registry.hub.docker.com/v2/repositories/aztecprotocol/aztec/tags/$nightly_tag) - if [[ "$TAGS" != *"not found"* ]]; then - TAG="$nightly_tag" - SEMVER="$nightly_semver" - echo "Using nightly tag: $TAG" - else - echo "Error: Tag $nightly_tag not published to docker hub" - exit 1 - fi + SEMVER="${current_version}-nightly.$(date -u +%Y%m%d)" + echo "Using nightly tag: $SEMVER" fi - echo "TAG=$TAG" >> "$GITHUB_OUTPUT" echo "SEMVER=$SEMVER" >> "$GITHUB_OUTPUT" deploy-next-net: needs: get-image-tag - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.repository == 'AztecProtocol/aztec-packages-private' }} uses: ./.github/workflows/deploy-network.yml with: network: next-net semver: ${{ needs.get-image-tag.outputs.semver }} - aztec_docker_image: "aztecprotocol/aztec:${{ needs.get-image-tag.outputs.tag }}" - ref: ${{ github.ref }} + ref: next + use_internal_docker_registry: true deploy_contracts: ${{ github.event_name == 'schedule' || inputs.deploy_contracts == true }} secrets: inherit diff --git a/.github/workflows/deploy-staging-internal.yml b/.github/workflows/deploy-staging-internal.yml new file mode 100644 index 000000000000..d5f3eb127984 --- /dev/null +++ b/.github/workflows/deploy-staging-internal.yml @@ -0,0 +1,87 @@ +name: Deploy to staging internal + +on: + schedule: + - cron: "0 7 * * *" + workflow_dispatch: + inputs: + tag: + description: "Tag to deploy (v5.0.0 or any v5.0.0 prerelease). Leave empty to use today's nightly tag." + required: false + type: string + deploy_contracts: + description: "Redeploy rollup contracts before deploying infra." + required: false + type: boolean + default: false + +concurrency: + group: deploy-staging-internal + cancel-in-progress: true + +env: + GITHUB_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + +jobs: + determine-tag: + runs-on: ubuntu-latest + if: >- + (github.event_name == 'workflow_dispatch' || + github.event_name == 'schedule') && + (github.event_name != 'schedule' || + github.repository == 'AztecProtocol/aztec-packages-private') + outputs: + tag: ${{ steps.resolve.outputs.tag }} + semver: ${{ steps.resolve.outputs.semver }} + steps: + - name: Checkout v5-next + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: v5-next + token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Resolve nightly tag + id: resolve + run: | + if [[ -n "${{ inputs.tag }}" ]]; then + TAG="${{ inputs.tag }}" + echo "Using manually specified tag: $TAG" + else + TAG="v5.0.0-nightly.$(date -u +%Y%m%d)" + echo "Using today's nightly tag: $TAG" + fi + + SEMVER="${TAG#v}" + echo "tag=$TAG" >> $GITHUB_OUTPUT + echo "semver=$SEMVER" >> $GITHUB_OUTPUT + + wait-for-ci3: + needs: determine-tag + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages-private' }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + fetch-depth: 1 + + - name: Setup Node + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e + with: + node-version: 22 + + - name: Wait for CI3 + run: spartan/scripts/wait_for_ci3.ts "${{ needs.determine-tag.outputs.tag }}" + + deploy: + needs: [determine-tag, wait-for-ci3] + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages-private' }} + uses: ./.github/workflows/deploy-network.yml + with: + network: staging-internal + semver: ${{ needs.determine-tag.outputs.semver }} + source_tag: ${{ needs.determine-tag.outputs.tag }} + deploy_contracts: ${{ inputs.deploy_contracts == true }} + use_internal_docker_registry: true + secrets: inherit diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging-public.yml similarity index 89% rename from .github/workflows/deploy-staging.yml rename to .github/workflows/deploy-staging-public.yml index 0c5b38f6d65d..8aaea198d33c 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging-public.yml @@ -1,4 +1,4 @@ -name: Deploy to staging +name: Deploy to staging public on: schedule: @@ -16,7 +16,7 @@ on: default: false concurrency: - group: deploy-staging + group: deploy-staging-public cancel-in-progress: true env: @@ -52,11 +52,6 @@ jobs: echo "Using today's nightly tag: $TAG" fi - if [[ ! "$TAG" =~ ^v5\.0\.0($|-.*) ]]; then - echo "Error: Staging deploys require a v5.0.0 tag or prerelease, got '$TAG'" - exit 1 - fi - SEMVER="${TAG#v}" echo "tag=$TAG" >> $GITHUB_OUTPUT echo "semver=$SEMVER" >> $GITHUB_OUTPUT @@ -84,9 +79,8 @@ jobs: if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} uses: ./.github/workflows/deploy-network.yml with: - network: staging + network: staging-public semver: ${{ needs.determine-tag.outputs.semver }} - use_internal_docker_registry: true source_tag: ${{ needs.determine-tag.outputs.tag }} deploy_contracts: ${{ inputs.deploy_contracts == true }} secrets: inherit diff --git a/.github/workflows/ensure-funded-environment.yml b/.github/workflows/ensure-funded-environment.yml index 89a98760a424..63639ce8c701 100644 --- a/.github/workflows/ensure-funded-environment.yml +++ b/.github/workflows/ensure-funded-environment.yml @@ -6,7 +6,7 @@ on: workflow_call: inputs: environment: - description: 'Environment to fund (e.g., staging, next-net, testnet)' + description: 'Environment to fund (e.g., staging-public, staging-internal, next-net, testnet)' required: true type: string low_watermark: @@ -30,7 +30,8 @@ on: required: true type: choice options: - - staging + - staging-public + - staging-internal - next-net - testnet - devnet diff --git a/.github/workflows/ensure-funded-environments.yml b/.github/workflows/ensure-funded-environments.yml index a8dd9de4bfa7..3378300f1701 100644 --- a/.github/workflows/ensure-funded-environments.yml +++ b/.github/workflows/ensure-funded-environments.yml @@ -50,7 +50,8 @@ jobs: run: | # Static environments static='[ - {"environment":"staging"}, + {"environment":"staging-public"}, + {"environment":"staging-internal"}, {"environment":"next-net"}, {"environment":"testnet"} ]' diff --git a/.github/workflows/network-healthcheck.yml b/.github/workflows/network-healthcheck.yml index eae21f5bea36..7347e74d1387 100644 --- a/.github/workflows/network-healthcheck.yml +++ b/.github/workflows/network-healthcheck.yml @@ -9,7 +9,7 @@ on: description: 'Comma-separated list of networks to check (default: all)' required: false type: string - default: 'testnet,mainnet,staging,next-net' + default: 'testnet,mainnet,staging-public,staging-internal,next-net' jobs: healthcheck: @@ -25,7 +25,7 @@ jobs: GH_TOKEN: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} CI: "1" run: | - NETWORKS="${{ inputs.networks || 'next-net,staging,testnet,mainnet' }}" + NETWORKS="${{ inputs.networks || 'next-net,staging-public,staging-internal,testnet,mainnet' }}" PROMPT="Run a network healthcheck for: ${NETWORKS}. diff --git a/.github/workflows/nightly-bench-10tps.yml b/.github/workflows/nightly-bench-10tps.yml index 3f0c096b6980..f06e34fa6198 100644 --- a/.github/workflows/nightly-bench-10tps.yml +++ b/.github/workflows/nightly-bench-10tps.yml @@ -100,7 +100,7 @@ jobs: namespace: bench-10tps aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + skip_notify_on_failure: true secrets: inherit wait-for-first-l2-block: diff --git a/.github/workflows/nightly-release-tag-v4-next.yml b/.github/workflows/nightly-release-tag-v4-next.yml deleted file mode 100644 index 73d53386b326..000000000000 --- a/.github/workflows/nightly-release-tag-v4-next.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Nightly Release Tag (v4-next) -on: - schedule: - # Run the workflow every night at 5:00 AM UTC. - - cron: "0 5 * * *" - workflow_dispatch: {} - -permissions: - contents: write - -concurrency: - group: ${{ github.workflow }}-v4-next - -jobs: - nightly-release-tag-v4-next: - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 - with: - ref: v4-next - token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - - - name: Create v4-next Nightly Tag - run: | - git config --global user.email "tech@aztecprotocol.com" - git config --global user.name "AztecBot" - current_version=$(jq -r '."."' .release-please-manifest.json) - echo "Current version: $current_version" - nightly_tag="v${current_version}-nightly.$(date -u +%Y%m%d)" - echo "v4-next nightly tag: $nightly_tag" - git tag -a "$nightly_tag" -m "$nightly_tag" - git push origin "$nightly_tag" diff --git a/.github/workflows/nightly-release-tag.yml b/.github/workflows/nightly-release-tag.yml index 4953711fe4f5..c0627849dcc2 100644 --- a/.github/workflows/nightly-release-tag.yml +++ b/.github/workflows/nightly-release-tag.yml @@ -3,26 +3,27 @@ on: schedule: # Run the workflow every night at 2:00 AM UTC. - cron: "0 2 * * *" + workflow_dispatch: {} -# Add permissions for the GitHub Actions bot to push tags permissions: contents: write -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - jobs: nightly-release-tag: - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + branch: [next, v5-next] + concurrency: + group: ${{ github.workflow }}-${{ github.repository }}-${{ matrix.branch }} steps: - # Check out the repository so we can read files and create tags. - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: + ref: ${{ matrix.branch }} token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} - # Extract the current release version from the manifest. - # Then, create a nightly tag using the current version and the current UTC date. - name: Create Nightly Tag run: | git config --global user.email "tech@aztecprotocol.com" @@ -30,7 +31,28 @@ jobs: current_version=$(jq -r '."."' .release-please-manifest.json) echo "Current version: $current_version" nightly_tag="v${current_version}-nightly.$(date -u +%Y%m%d)" - echo "Nightly tag: $nightly_tag" - # Tag and push. + echo "${{ matrix.branch }} nightly tag: $nightly_tag" + git tag -a "$nightly_tag" -m "$nightly_tag" + git push origin "$nightly_tag" + + nightly-release-tag-v4-next: + if: ${{ github.repository == 'AztecProtocol/aztec-packages' }} + runs-on: ubuntu-latest + concurrency: + group: ${{ github.workflow }}-${{ github.repository }}-v4-next + steps: + - uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: v4-next + token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} + + - name: Create v4-next Nightly Tag + run: | + git config --global user.email "tech@aztecprotocol.com" + git config --global user.name "AztecBot" + current_version=$(jq -r '."."' .release-please-manifest.json) + echo "Current version: $current_version" + nightly_tag="v${current_version}-nightly.$(date -u +%Y%m%d)" + echo "v4-next nightly tag: $nightly_tag" git tag -a "$nightly_tag" -m "$nightly_tag" git push origin "$nightly_tag" diff --git a/.github/workflows/nightly-spartan-bench.yml b/.github/workflows/nightly-spartan-bench.yml index 166427975ee5..5399e4302365 100644 --- a/.github/workflows/nightly-spartan-bench.yml +++ b/.github/workflows/nightly-spartan-bench.yml @@ -16,33 +16,47 @@ concurrency: jobs: select-image: - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest outputs: nightly_tag: ${{ steps.nightly-tag.outputs.nightly_tag }} docker_image: ${{ steps.nightly-tag.outputs.docker_image }} source_ref: ${{ steps.nightly-tag.outputs.source_ref }} + use_internal_docker_registry: ${{ steps.nightly-tag.outputs.use_internal_docker_registry }} steps: - name: Checkout uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 with: - ref: next + ref: ${{ github.repository == 'AztecProtocol/aztec-packages-private' && github.event_name == 'schedule' && 'v5-next' || 'next' }} - name: Determine nightly tag id: nightly-tag run: | + USE_INTERNAL_DOCKER_REGISTRY="${{ github.repository == 'AztecProtocol/aztec-packages-private' && 'true' || 'false' }}" + if [[ -n "${{ inputs.nightly_tag }}" ]]; then nightly_tag="${{ inputs.nightly_tag }}" else current_version=$(jq -r '."."' .release-please-manifest.json) nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)" fi - docker_image="aztecprotocol/aztec:${nightly_tag}" + source_ref="v${nightly_tag}" + + if [[ "$USE_INTERNAL_DOCKER_REGISTRY" == "true" ]]; then + docker_image="" + image_label="internal/aztec:${nightly_tag}" + else + docker_image="aztecprotocol/aztec:${nightly_tag}" + image_label="$docker_image" + fi + echo "nightly_tag=$nightly_tag" >> "$GITHUB_OUTPUT" echo "docker_image=$docker_image" >> "$GITHUB_OUTPUT" echo "source_ref=$source_ref" >> "$GITHUB_OUTPUT" + echo "use_internal_docker_registry=$USE_INTERNAL_DOCKER_REGISTRY" >> "$GITHUB_OUTPUT" echo "Using nightly tag: $nightly_tag" + echo "Using image: $image_label" echo "Using source ref: $source_ref" - name: Verify source git ref @@ -54,6 +68,7 @@ jobs: echo "Nightly source commit: $source_sha" - name: Check if Docker image exists + if: steps.nightly-tag.outputs.use_internal_docker_registry != 'true' run: | DOCKER_IMAGE="${{ steps.nightly-tag.outputs.docker_image }}" echo "Checking if Docker image exists: $DOCKER_IMAGE" @@ -69,21 +84,22 @@ jobs: # --------------------------------------------------------------------------- deploy-bench-network: needs: select-image - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} uses: ./.github/workflows/deploy-network.yml with: network: tps-scenario namespace: nightly-bench - aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} + semver: ${{ needs.select-image.outputs.nightly_tag }} ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + use_internal_docker_registry: ${{ needs.select-image.outputs.use_internal_docker_registry == 'true' }} + skip_notify_on_failure: true secrets: inherit wait-bench-l2-block: needs: - select-image - deploy-bench-network - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -114,7 +130,7 @@ jobs: needs: - select-image - wait-bench-l2-block - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -168,7 +184,7 @@ jobs: max-items-in-chart: 100 cleanup-bench: - if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-bench-network @@ -193,7 +209,7 @@ jobs: run: ./.github/ci3.sh network-teardown tps-scenario nightly-bench notify-bench-failure: - if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-bench-network @@ -224,21 +240,22 @@ jobs: # --------------------------------------------------------------------------- deploy-proving-network: needs: select-image - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} uses: ./.github/workflows/deploy-network.yml with: network: prove-n-tps-fake namespace: prove-n-tps-fake - aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} + semver: ${{ needs.select-image.outputs.nightly_tag }} ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + use_internal_docker_registry: ${{ needs.select-image.outputs.use_internal_docker_registry == 'true' }} + skip_notify_on_failure: true secrets: inherit wait-proving-l2-block: needs: - select-image - deploy-proving-network - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -269,7 +286,7 @@ jobs: needs: - select-image - wait-proving-l2-block - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -322,7 +339,7 @@ jobs: max-items-in-chart: 100 cleanup-proving: - if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-proving-network @@ -347,7 +364,7 @@ jobs: run: ./.github/ci3.sh network-teardown prove-n-tps-fake prove-n-tps-fake notify-proving-failure: - if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-proving-network @@ -378,21 +395,22 @@ jobs: # --------------------------------------------------------------------------- deploy-block-capacity-network: needs: select-image - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} uses: ./.github/workflows/deploy-network.yml with: network: block-capacity namespace: nightly-block-capacity - aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} + semver: ${{ needs.select-image.outputs.nightly_tag }} ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + use_internal_docker_registry: ${{ needs.select-image.outputs.use_internal_docker_registry == 'true' }} + skip_notify_on_failure: true secrets: inherit wait-block-capacity-l2-block: needs: - select-image - deploy-block-capacity-network - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest timeout-minutes: 120 steps: @@ -423,7 +441,7 @@ jobs: needs: - select-image - wait-block-capacity-l2-block - if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' }} + if: ${{ github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private' }} runs-on: ubuntu-latest steps: - name: Checkout @@ -476,7 +494,7 @@ jobs: max-items-in-chart: 100 cleanup-block-capacity: - if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-block-capacity-network @@ -501,7 +519,7 @@ jobs: run: ./.github/ci3.sh network-teardown block-capacity nightly-block-capacity notify-block-capacity-failure: - if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && failure() && github.event_name != 'workflow_dispatch' && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} needs: - select-image - deploy-block-capacity-network @@ -530,7 +548,7 @@ jobs: status: runs-on: ubuntu-latest needs: [benchmark, proving-benchmark, block-capacity-benchmark] - if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages') }} + if: ${{ always() && (github.event_name != 'schedule' || github.repository == 'AztecProtocol/aztec-packages' || github.repository == 'AztecProtocol/aztec-packages-private') }} steps: - name: Check benchmark results run: | diff --git a/.github/workflows/test-network-scenarios.yml b/.github/workflows/test-network-scenarios.yml index 2393ba576920..fa23bca63f2b 100644 --- a/.github/workflows/test-network-scenarios.yml +++ b/.github/workflows/test-network-scenarios.yml @@ -10,6 +10,11 @@ on: description: Complete docker image to use (e.g., aztecprotocol/aztec:some-tag) required: true type: string + ref: + description: Git ref to checkout for the scenario runner. + required: false + type: string + default: next namespace: description: Kubernetes namespace to use (will be sanitized for k8s naming) required: true @@ -39,6 +44,8 @@ jobs: steps: - name: Checkout uses: actions/checkout@93cb6efe18208431cddfb8368fd83d5badbf9bfd # v5.0.1 + with: + ref: ${{ inputs.ref }} - name: Run Network Scenarios timeout-minutes: 350 diff --git a/.github/workflows/weekly-proving-bench.yml b/.github/workflows/weekly-proving-bench.yml index 0353eaa4d08c..c8ce2b663e96 100644 --- a/.github/workflows/weekly-proving-bench.yml +++ b/.github/workflows/weekly-proving-bench.yml @@ -74,7 +74,7 @@ jobs: namespace: prove-n-tps-real aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} ref: ${{ needs.select-image.outputs.source_ref }} - notify_on_failure: false + skip_notify_on_failure: true secrets: inherit wait-for-first-l2-block: diff --git a/spartan/.gitignore b/spartan/.gitignore index da7cd59acdb2..b7719acadf31 100644 --- a/spartan/.gitignore +++ b/spartan/.gitignore @@ -23,9 +23,8 @@ environments/* !environments/next-scenario.env !environments/scenario.local.env !environments/source-env.sh -!environments/staging-ignition.env -!environments/staging.env -!environments/staging.local.env +!environments/staging-public.env +!environments/staging-internal.env !environments/testnet-canary.env !environments/testnet.env !environments/mainnet.env diff --git a/spartan/environments/next-net.env b/spartan/environments/next-net.env index cac6310416e4..94d2694e93a7 100644 --- a/spartan/environments/next-net.env +++ b/spartan/environments/next-net.env @@ -22,9 +22,7 @@ P2P_HOSTPORT_ENABLED=false STORE_SNAPSHOT_URL= BLOB_BUCKET_DIRECTORY=${BLOB_BUCKET_DIRECTORY:-next-net/blobs} BLOB_FILE_STORE_URLS="," -TX_FILE_STORE_ENABLED=true -TX_FILE_STORE_BUCKET_DIRECTORY=${TX_FILE_STORE_BUCKET_DIRECTORY:-next-net/txs} -TX_COLLECTION_FILE_STORE_URLS="https://aztec-labs-snapshots.com/${TX_FILE_STORE_BUCKET_DIRECTORY}" +TX_FILE_STORE_ENABLED=false R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET PROVER_FAILED_PROOF_STORE=gs://aztec-develop/next-net/failed-proofs diff --git a/spartan/environments/staging.env b/spartan/environments/staging-internal.env similarity index 77% rename from spartan/environments/staging.env rename to spartan/environments/staging-internal.env index 3193c558f6b6..a201feba0344 100644 --- a/spartan/environments/staging.env +++ b/spartan/environments/staging-internal.env @@ -1,7 +1,7 @@ # Deployment CREATE_ETH_DEVNET=false NETWORK=staging -NAMESPACE=${NAMESPACE:-staging} +NAMESPACE=${NAMESPACE:-staging-internal} GCP_REGION=us-west1-a CLUSTER=aztec-gke-public RESOURCE_PROFILE=prod @@ -15,17 +15,12 @@ ETHEREUM_CONSENSUS_HOST_API_KEY_HEADERS=REPLACE_WITH_GCP_SECRET FUNDING_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET LABS_INFRA_MNEMONIC=REPLACE_WITH_GCP_SECRET ROLLUP_DEPLOYMENT_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET -VERIFY_CONTRACTS=true -ETHERSCAN_API_KEY=REPLACE_WITH_GCP_SECRET CREATE_ROLLUP_CONTRACTS=${CREATE_ROLLUP_CONTRACTS:-false} # Storage -SNAPSHOT_BUCKET_DIRECTORY=${SNAPSHOT_BUCKET_DIRECTORY:-staging} -BLOB_BUCKET_DIRECTORY=${BLOB_BUCKET_DIRECTORY:-staging/blobs} -BLOB_FILE_STORE_URLS="," -TX_FILE_STORE_ENABLED=true -TX_FILE_STORE_BUCKET_DIRECTORY=${TX_FILE_STORE_BUCKET_DIRECTORY:-staging/txs} -TX_COLLECTION_FILE_STORE_URLS="https://aztec-labs-snapshots.com/${TX_FILE_STORE_BUCKET_DIRECTORY}" +SNAPSHOT_BUCKET_DIRECTORY=${SNAPSHOT_BUCKET_DIRECTORY:-staging-internal/snapshots} +SYNC_SNAPSHOT_URLS="https://aztec-labs-snapshots.com/${SNAPSHOT_BUCKET_DIRECTORY}" +TX_FILE_STORE_ENABLED=false R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET @@ -58,17 +53,16 @@ VALIDATOR_REPLICAS=2 VALIDATORS_PER_NODE=32 VALIDATOR_PUBLISHERS_PER_REPLICA=4 VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 -VALIDATOR_HA_REPLICAS=1 -VALIDATOR_HA_REPLICA_COUNT=2 VALIDATOR_RESOURCE_PROFILE="prod" + RPC_REPLICAS=1 # Prover PROVER_RESOURCE_PROFILE="prod" PUBLISHERS_PER_PROVER=2 PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 -PROVER_FAILED_PROOF_STORE=gs://aztec-develop/staging/failed-proofs -L1_TX_FAILED_STORE=gs://aztec-develop/staging/failed-l1-txs +# PROVER_FAILED_PROOF_STORE=gs://aztec-develop/staging/failed-proofs +# L1_TX_FAILED_STORE=gs://aztec-develop/staging/failed-l1-txs PROVER_AGENT_KEDA_ENABLED=true PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET PROVER_AGENT_KEDA_MIN_REPLICAS=0 @@ -99,4 +93,4 @@ BOT_CROSS_CHAIN_FOLLOW_CHAIN=PROPOSED # Observability OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET -LOG_LEVEL="debug; info: simulator, json-rpc" +LOG_LEVEL="info" diff --git a/spartan/environments/staging-public.env b/spartan/environments/staging-public.env new file mode 100644 index 000000000000..843d09d6925c --- /dev/null +++ b/spartan/environments/staging-public.env @@ -0,0 +1,96 @@ +# Deployment +CREATE_ETH_DEVNET=false +NETWORK=staging +NAMESPACE=${NAMESPACE:-staging-public} +GCP_REGION=us-west1-a +CLUSTER=aztec-gke-public +RESOURCE_PROFILE=prod + +# Ethereum +ETHEREUM_CHAIN_ID=11155111 +ETHEREUM_RPC_URLS=REPLACE_WITH_GCP_SECRET +ETHEREUM_CONSENSUS_HOST_URLS=REPLACE_WITH_GCP_SECRET +ETHEREUM_CONSENSUS_HOST_API_KEYS=REPLACE_WITH_GCP_SECRET +ETHEREUM_CONSENSUS_HOST_API_KEY_HEADERS=REPLACE_WITH_GCP_SECRET +FUNDING_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET +LABS_INFRA_MNEMONIC=REPLACE_WITH_GCP_SECRET +ROLLUP_DEPLOYMENT_PRIVATE_KEY=REPLACE_WITH_GCP_SECRET +CREATE_ROLLUP_CONTRACTS=${CREATE_ROLLUP_CONTRACTS:-false} + +# Storage +SNAPSHOT_BUCKET_DIRECTORY=${SNAPSHOT_BUCKET_DIRECTORY:-staging-public/snapshots} +SYNC_SNAPSHOT_URLS="https://aztec-labs-snapshots.com/${SNAPSHOT_BUCKET_DIRECTORY}" +TX_FILE_STORE_ENABLED=false +R2_ACCESS_KEY_ID=REPLACE_WITH_GCP_SECRET +R2_SECRET_ACCESS_KEY=REPLACE_WITH_GCP_SECRET + +# Network features +DEPLOY_INTERNAL_BOOTNODE=true +P2P_PUBLIC_IP=false +P2P_HOSTPORT_ENABLED=false +P2P_TX_POOL_DELETE_TXS_AFTER_REORG=true +TEST_ACCOUNTS=false +SPONSORED_FPC=false + +# Rollup +AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=2 +AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=2 +AZTEC_INBOX_LAG=2 +AZTEC_MANA_TARGET=75000000 +AZTEC_PROVING_COST_PER_MANA=12500000 + +# Sequencer +SEQ_MAX_L2_BLOCK_GAS=150000000 +SEQ_MIN_TX_PER_BLOCK=1 +SEQ_MAX_TX_PER_CHECKPOINT=7 # 0.1 TPS +SEQ_BUILD_CHECKPOINT_IF_EMPTY=true +SEQ_BLOCK_DURATION_MS=6000 +SEQ_L1_PUBLISHING_TIME_ALLOWANCE_IN_SLOT=36 +SEQ_ENABLE_PROPOSER_PIPELINING=true + +# Validators and RPC +VALIDATOR_REPLICAS=2 +VALIDATORS_PER_NODE=32 +VALIDATOR_PUBLISHERS_PER_REPLICA=4 +VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 +VALIDATOR_RESOURCE_PROFILE="prod" + +RPC_REPLICAS=1 + +# Prover +PROVER_RESOURCE_PROFILE="prod" +PUBLISHERS_PER_PROVER=2 +PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 +# PROVER_FAILED_PROOF_STORE=gs://aztec-develop/staging/failed-proofs +# L1_TX_FAILED_STORE=gs://aztec-develop/staging/failed-l1-txs +PROVER_AGENT_KEDA_ENABLED=true +PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET +PROVER_AGENT_KEDA_MIN_REPLICAS=0 +PROVER_AGENT_KEDA_MAX_REPLICAS=8 +PROVER_AGENT_KEDA_SCALING_BANDS='[ + { + queueSize = 0 + replicas = 4 + }, + { + queueSize = 100 + replicas = 8 + } +]' + +# Bots +BOT_DA_GAS_LIMIT=100000 +BOT_L2_GAS_LIMIT=6540000 +BOT_TRANSFERS_REPLICAS=1 +BOT_TRANSFERS_TX_INTERVAL_SECONDS=250 +BOT_TRANSFERS_FOLLOW_CHAIN=PROPOSED +BOT_SWAPS_REPLICAS=0 +BOT_SWAPS_FOLLOW_CHAIN=PROPOSED +BOT_SWAPS_TX_INTERVAL_SECONDS=350 +BOT_CROSS_CHAIN_REPLICAS=0 +BOT_CROSS_CHAIN_TX_INTERVAL_SECONDS=250 +BOT_CROSS_CHAIN_FOLLOW_CHAIN=PROPOSED + +# Observability +OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET +LOG_LEVEL="info" diff --git a/spartan/terraform/gke-cluster/iam.tf b/spartan/terraform/gke-cluster/iam.tf index 10663f891ed2..ded765f27248 100644 --- a/spartan/terraform/gke-cluster/iam.tf +++ b/spartan/terraform/gke-cluster/iam.tf @@ -48,6 +48,30 @@ resource "google_service_account" "ci" { description = "Service account for CI jobs that publish Docker images" } +resource "google_project_iam_member" "ci_network_deploy_project_roles" { + for_each = toset([ + "roles/container.developer", + "roles/secretmanager.secretAccessor", + "roles/compute.loadBalancerAdmin" + ]) + project = var.project + role = each.key + member = "serviceAccount:${google_service_account.ci.email}" +} + +resource "google_storage_bucket_iam_member" "ci_terraform_state_object_user" { + bucket = "aztec-terraform" + role = "roles/storage.objectUser" + member = "serviceAccount:${google_service_account.ci.email}" +} + +resource "google_dns_managed_zone_iam_member" "ci_rpc_dns_admin" { + project = var.project + managed_zone = "rpc-aztec-labs-com" + role = "roles/dns.admin" + member = "serviceAccount:${google_service_account.ci.email}" +} + resource "google_service_account" "npm_registry_reader" { account_id = var.npm_registry_reader_service_account_id display_name = "npm Registry Reader Service Account"