diff --git a/.github/workflows/ci-amd-arm.yml b/.github/workflows/ci-amd.yml similarity index 98% rename from .github/workflows/ci-amd-arm.yml rename to .github/workflows/ci-amd.yml index ec1f955238cfb..f657b7fb314c0 100644 --- a/.github/workflows/ci-amd-arm.yml +++ b/.github/workflows/ci-amd.yml @@ -16,10 +16,15 @@ # under the License. # --- -name: Tests +# AMD CI workflow. Triggered on every PR and on push to release branches. +# Companion to `ci-arm.yml`, which runs the same pipeline on ARM as the +# scheduled canary. The two workflows are physical copies of each other +# (GitHub Actions has no cross-file YAML include), differing only in their +# trigger surface, the hardcoded `runner-type` / `platform` values in +# `build-info`'s outputs, and the architecture-specific print-platform job. +# Edit both files together when changing pipeline shape. +name: Tests (AMD) on: # yamllint disable-line rule:truthy - schedule: - - cron: '28 1,3,7,9,13,15,19,21 * * *' push: branches: - v[0-9]+-[0-9]+-test @@ -43,7 +48,7 @@ env: VERBOSE: "true" concurrency: - group: ci-amd-arm-${{ github.event.pull_request.number || github.ref }} + group: ci-amd-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: @@ -90,7 +95,9 @@ jobs: latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} - platform: ${{ steps.selective-checks.outputs.platform }} + # platform / runner-type are hardcoded per workflow file (this is the + # AMD copy). The matching ARM copy lives in ci-arm.yml. + platform: "linux/amd64" postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} @@ -120,7 +127,7 @@ jobs: run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} run-breeze-integration-tests: ${{ steps.selective-checks.outputs.run-breeze-integration-tests }} run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests }} - runner-type: ${{ steps.selective-checks.outputs.runner-type }} + runner-type: '["ubuntu-22.04"]' run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }} run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }} @@ -198,20 +205,10 @@ jobs: env: PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} - print-platform-arm: - name: "Platform: ARM" - needs: [build-info] - runs-on: ["ubuntu-22.04"] - if: needs.build-info.outputs.platform == 'linux/arm64' - steps: - - name: "Print architecture" - run: "echo '## Architecture: ARM' >> $GITHUB_STEP_SUMMARY" - - print-platform-amd: + print-platform: name: "Platform: AMD" needs: [build-info] runs-on: ["ubuntu-22.04"] - if: needs.build-info.outputs.platform == 'linux/amd64' steps: - name: "Print architecture" run: "echo '## Architecture: AMD' >> $GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ci-arm.yml b/.github/workflows/ci-arm.yml new file mode 100644 index 0000000000000..27aa6b0c8e23c --- /dev/null +++ b/.github/workflows/ci-arm.yml @@ -0,0 +1,1161 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +# ARM canary CI workflow. Triggered on the canary cron only β€” ARM runners are +# reserved for the scheduled slot. Companion to `ci-amd.yml`, which runs the +# same pipeline on AMD for every PR and push. The two workflows are physical +# copies of each other (GitHub Actions has no cross-file YAML include), +# differing only in their trigger surface, the hardcoded `runner-type` / +# `platform` values in `build-info`'s outputs, and the architecture-specific +# print-platform job. Edit both files together when changing pipeline shape. +name: Tests (ARM) +on: # yamllint disable-line rule:truthy + schedule: + - cron: '28 1,3,7,9,13,15,19,21 * * *' + workflow_dispatch: +permissions: + # All other permissions are set to none by default + contents: read +env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + VERBOSE: "true" + +concurrency: + group: ci-arm-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + build-info: + name: "Build info" + # At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners + runs-on: ["ubuntu-22.04"] + outputs: + all-python-versions-list-as-string: >- + ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} + arm-runners: ${{ steps.selective-checks.outputs.arm-runners }} + basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} + canary-run: ${{ steps.source-run-info.outputs.canary-run }} + ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} + core-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.core-test-types-list-as-strings-in-json }} + debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} + default-branch: ${{ steps.selective-checks.outputs.default-branch }} + default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} + default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} + default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} + default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} + default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} + default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} + default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} + disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} + docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} + docs-build: ${{ steps.selective-checks.outputs.docs-build }} + docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }} + excluded-providers-as-string: ${{ steps.selective-checks.outputs.excluded-providers-as-string }} + force-pip: ${{ steps.selective-checks.outputs.force-pip }} + full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} + has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} + helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} + helm-test-kubernetes-versions: ${{ steps.selective-checks.outputs.helm-test-kubernetes-versions }} + include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} + individual-providers-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-strings-in-json }} + kubernetes-combos: ${{ steps.selective-checks.outputs.kubernetes-combos }} + kubernetes-combos-list-as-string: >- + ${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} + kubernetes-versions-list-as-string: >- + ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} + latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} + mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} + mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} + # platform / runner-type are hardcoded per workflow file (this is the + # ARM copy). The matching AMD copy lives in ci-amd.yml. + platform: "linux/arm64" + postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} + postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} + prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} + # yamllint disable rule:line-length + providers-compatibility-tests-matrix: > + ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} + providers-test-types-list-as-strings-in-json: >- + ${{ steps.selective-checks.outputs.providers-test-types-list-as-strings-in-json }} + pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} + python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} + python-versions: ${{ steps.selective-checks.outputs.python-versions }} + run-airflow-ctl-tests: ${{ steps.selective-checks.outputs.run-airflow-ctl-tests }} + run-airflow-ctl-integration-tests: ${{ steps.selective-checks.outputs.run-airflow-ctl-integration-tests }} + run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} + run-api-codegen: ${{ steps.selective-checks.outputs.run-api-codegen }} + run-api-tests: ${{ steps.selective-checks.outputs.run-api-tests }} + run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} + run-go-sdk-tests: ${{ steps.selective-checks.outputs.run-go-sdk-tests }} + run-helm-tests: ${{ steps.selective-checks.outputs.run-helm-tests }} + run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} + run-mypy-providers: ${{ steps.selective-checks.outputs.run-mypy-providers }} + run-remote-logging-elasticsearch-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-elasticsearch-e2e-tests }} + run-remote-logging-opensearch-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-opensearch-e2e-tests }} + run-remote-logging-s3-e2e-tests: ${{ steps.selective-checks.outputs.run-remote-logging-s3-e2e-tests }} + run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} + run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} + run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }} + run-breeze-integration-tests: ${{ steps.selective-checks.outputs.run-breeze-integration-tests }} + run-scripts-tests: ${{ steps.selective-checks.outputs.run-scripts-tests }} + runner-type: '["ubuntu-22.04-arm"]' + run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} + run-ui-e2e-tests: ${{ steps.selective-checks.outputs.run-ui-e2e-tests }} + run-unit-tests: ${{ steps.selective-checks.outputs.run-unit-tests }} + run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} + selected-providers-list-as-string: >- + ${{ steps.selective-checks.outputs.selected-providers-list-as-string }} + shared-distributions-as-json: ${{ steps.selective-checks.outputs.shared-distributions-as-json }} + skip-prek-hooks: ${{ steps.selective-checks.outputs.skip-prek-hooks }} + skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} + source-head-repo: ${{ steps.source-run-info.outputs.head-repo }} + source-head-ref: ${{ steps.source-run-info.outputs.head-ref }} + sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} + testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }} + testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }} + use-uv: ${{ steps.selective-checks.outputs.force-pip == 'true' && 'false' || 'true' }} + upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} + is-uv-lock-update: ${{ steps.selective-checks.outputs.is-uv-lock-update }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Fetch incoming commit ${{ github.sha }} with its parent + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ github.sha }} + fetch-depth: 2 + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + id: breeze + - name: "Save github context to file" + # The github context can be very large for PRs that touch many files (e.g. Dependabot + # bumps with dozens of packages). Passing it as an env var would exceed ARG_MAX and + # prevent subsequent steps from starting bash, so we write it to a file once here and + # let downstream breeze invocations read it via GITHUB_CONTEXT_INPUT. + # + # Zizmor flags this as template-injection because attacker-controllable fields in the + # github context (PR title, body, branch names) are pasted into a `run:` script. It is + # a false positive in this specific shape: + # * the substituted JSON sits inside a SINGLE-QUOTED heredoc (`<< '__GITHUB_CONTEXT_END__'`), + # which disables ALL bash expansion -- $(...), `...`, ${...}, and even `\` are literal, + # so no shell metacharacter in the JSON can be evaluated; + # * the only remaining escape is the delimiter `__GITHUB_CONTEXT_END__` appearing on a + # line by itself in the JSON. `toJson` escapes newlines inside string values as `\n` + # (so they are NOT real newlines in the script source), and the delimiter is unique + # enough that github will never produce it as a structural line of the JSON output; + # * `cat` writes the bytes verbatim to a file -- it is not eval'd. + # Whatever junk a malicious PR puts in title/body therefore lands inside a JSON file + # consumed by breeze, never executed as shell. + shell: bash + run: | # zizmor: ignore[template-injection] + cat > "${RUNNER_TEMP}/github_context.json" << '__GITHUB_CONTEXT_END__' + ${{ toJson(github) }} + __GITHUB_CONTEXT_END__ + - name: "Get information about the Workflow" + id: source-run-info + run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} + env: + SKIP_BREEZE_SELF_UPGRADE_CHECK: "true" + GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + - name: Selective checks + id: selective-checks + env: + PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" + COMMIT_REF: "${{ github.sha }}" + VERBOSE: "false" + GITHUB_CONTEXT_INPUT: "${{ runner.temp }}/github_context.json" + run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} + - name: env + run: printenv + env: + PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} + + print-platform: + name: "Platform: ARM" + needs: [build-info] + runs-on: ["ubuntu-22.04"] + steps: + - name: "Print architecture" + run: "echo '## Architecture: ARM' >> $GITHUB_STEP_SUMMARY" + + basic-tests: + name: "Basic tests" + needs: [build-info] + uses: ./.github/workflows/basic-tests.yml + with: + runners: ${{ needs.build-info.outputs.runner-type }} + run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} + run-www-tests: ${{needs.build-info.outputs.run-www-tests}} + run-api-codegen: ${{needs.build-info.outputs.run-api-codegen}} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} + canary-run: ${{needs.build-info.outputs.canary-run}} + run-breeze-integration-tests: ${{needs.build-info.outputs.run-breeze-integration-tests}} + run-scripts-tests: ${{needs.build-info.outputs.run-scripts-tests}} + latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} + use-uv: ${{needs.build-info.outputs.use-uv}} + platform: ${{ needs.build-info.outputs.platform }} + shared-distributions-as-json: ${{needs.build-info.outputs.shared-distributions-as-json}} + + build-ci-images: + name: Build CI images + needs: [build-info] + uses: ./.github/workflows/ci-image-build.yml + permissions: + contents: read + # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. + packages: write + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + push-image: "false" + upload-image-artifact: "true" + upload-mount-cache-artifact: ${{ needs.build-info.outputs.canary-run }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + if: needs.build-info.outputs.ci-image-build == 'true' + + additional-ci-image-checks: + name: "Additional CI image checks" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/additional-ci-image-checks.yml + permissions: + contents: read + packages: write + id-token: write + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + + generate-constraints: + name: "Generate constraints" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/generate-constraints.yml + if: needs.build-info.outputs.ci-image-build == 'true' + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + generate-pypi-constraints: "true" + # generate no providers constraints only in canary builds - they take quite some time to generate + # they are not needed for regular builds, they are only needed to update constraints in canaries + generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + + ci-image-checks: + name: "CI image checks" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/ci-image-checks.yml + permissions: + id-token: write + contents: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + branch: ${{ needs.build-info.outputs.default-branch }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }} + latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} + basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + skip-prek-hooks: ${{ needs.build-info.outputs.skip-prek-hooks }} + ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + docs-build: ${{ needs.build-info.outputs.docs-build }} + run-api-codegen: ${{ needs.build-info.outputs.run-api-codegen }} + default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + source-head-repo: ${{ needs.build-info.outputs.source-head-repo }} + source-head-ref: ${{ needs.build-info.outputs.source-head-ref }} + secrets: + DOCS_AWS_ACCESS_KEY_ID: ${{ secrets.DOCS_AWS_ACCESS_KEY_ID }} + DOCS_AWS_SECRET_ACCESS_KEY: ${{ secrets.DOCS_AWS_SECRET_ACCESS_KEY }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + + mypy-providers: + timeout-minutes: 45 + name: "MyPy providers checks" + needs: [build-info, build-ci-images] + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + if: needs.build-info.outputs.run-mypy-providers == 'true' + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ needs.build-info.outputs.platform }} + python: "${{ needs.build-info.outputs.default-python-version }}" + use-uv: ${{ needs.build-info.outputs.use-uv }} + make-mnt-writeable-and-cleanup: true + id: breeze + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek + with: + python-version: ${{steps.breeze.outputs.host-python-version}} + platform: ${{ needs.build-info.outputs.platform }} + save-cache: false + - name: "MyPy checks for providers" + run: prek --color always --verbose --stage manual mypy-providers --all-files + env: + VERBOSE: "false" + COLUMNS: "202" + SKIP_GROUP_OUTPUT: "true" + DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} + RUFF_FORMAT: "github" + INCLUDE_MYPY_VOLUME: "false" + + migration-round-trip: + timeout-minutes: 20 + name: "Migration round-trip check" + needs: [build-info, build-ci-images] + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + if: needs.build-info.outputs.has-migrations == 'true' + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ needs.build-info.outputs.default-python-version }}" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" + uses: ./.github/actions/prepare_breeze_and_image + with: + platform: ${{ needs.build-info.outputs.platform }} + python: "${{ needs.build-info.outputs.default-python-version }}" + use-uv: ${{ needs.build-info.outputs.use-uv }} + make-mnt-writeable-and-cleanup: true + id: breeze + - name: "Install prek" + uses: ./.github/actions/install-prek + id: prek + with: + python-version: ${{steps.breeze.outputs.host-python-version}} + platform: ${{ needs.build-info.outputs.platform }} + save-cache: false + - name: "Migration round-trip check" + run: prek --color always --verbose --stage manual migration-round-trip --all-files + env: + VERBOSE: "false" + COLUMNS: "202" + SKIP_GROUP_OUTPUT: "true" + DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }} + + providers: + name: "provider distributions tests" + uses: ./.github/workflows/test-providers.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + if: > + needs.build-info.outputs.skip-providers-tests != 'true' && + needs.build-info.outputs.latest-versions-only != 'true' && + needs.build-info.outputs.run-unit-tests == 'true' + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }} + # yamllint disable rule:line-length + providers-compatibility-tests-matrix: > + ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + providers-test-types-list-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + + tests-helm: + name: "Helm tests" + uses: ./.github/workflows/helm-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} + helm-test-kubernetes-versions: ${{ needs.build-info.outputs.helm-test-kubernetes-versions }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: > + needs.build-info.outputs.run-helm-tests == 'true' && + needs.build-info.outputs.default-branch == 'main' && + needs.build-info.outputs.latest-versions-only != 'true' + + + tests-postgres-core: + name: "Postgres tests: core" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + backend: "postgres" + test-name: "Postgres" + test-scope: "DB" + test-group: "core" + python-versions: ${{ needs.build-info.outputs.python-versions }} + backend-versions: ${{ needs.build-info.outputs.postgres-versions }} + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.postgres-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-migration-tests: "true" + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + if: needs.build-info.outputs.run-unit-tests == 'true' + + tests-postgres-providers: + name: "Postgres tests: providers" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + backend: "postgres" + test-name: "Postgres" + test-scope: "DB" + test-group: "providers" + python-versions: ${{ needs.build-info.outputs.python-versions }} + backend-versions: ${{ needs.build-info.outputs.postgres-versions }} + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.postgres-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-migration-tests: "true" + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + if: needs.build-info.outputs.run-unit-tests == 'true' + + tests-mysql-core: + name: "MySQL tests: core" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + backend: "mysql" + test-name: "MySQL" + test-scope: "DB" + test-group: "core" + python-versions: ${{ needs.build-info.outputs.python-versions }} + backend-versions: ${{ needs.build-info.outputs.mysql-versions }} + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.mysql-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + run-migration-tests: "true" + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + if: needs.build-info.outputs.run-unit-tests == 'true' && needs.build-info.outputs.platform == 'linux/amd64' + + tests-mysql-providers: + name: "MySQL tests: providers" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + backend: "mysql" + test-name: "MySQL" + test-scope: "DB" + test-group: "providers" + python-versions: ${{ needs.build-info.outputs.python-versions }} + backend-versions: ${{ needs.build-info.outputs.mysql-versions }} + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.mysql-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + run-migration-tests: "true" + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + if: needs.build-info.outputs.run-unit-tests == 'true' && needs.build-info.outputs.platform == 'linux/amd64' + + tests-sqlite-core: + name: "Sqlite tests: core" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + backend: "sqlite" + test-name: "Sqlite" + test-name-separator: "" + test-scope: "DB" + test-group: "core" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for sqlite + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + run-migration-tests: "true" + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + if: needs.build-info.outputs.run-unit-tests == 'true' + + tests-sqlite-providers: + name: "Sqlite tests: providers" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + backend: "sqlite" + test-name: "Sqlite" + test-name-separator: "" + test-scope: "DB" + test-group: "providers" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for sqlite + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + run-migration-tests: "true" + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + if: needs.build-info.outputs.run-unit-tests == 'true' + + tests-non-db-core: + name: "Non-DB tests: core" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + backend: "sqlite" + test-name: "" + test-name-separator: "" + test-scope: "Non-DB" + test-group: "core" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for non-db + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + if: needs.build-info.outputs.run-unit-tests == 'true' + + tests-non-db-providers: + name: "Non-DB tests: providers" + uses: ./.github/workflows/run-unit-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + backend: "sqlite" + test-name: "" + test-name-separator: "" + test-scope: "Non-DB" + test-group: "providers" + python-versions: ${{ needs.build-info.outputs.python-versions }} + # No versions for non-db + backend-versions: "['']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: ${{ needs.build-info.outputs.sqlite-exclude }} + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + if: needs.build-info.outputs.run-unit-tests == 'true' + + tests-special: + name: "Special tests" + uses: ./.github/workflows/special-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + if: > + needs.build-info.outputs.run-unit-tests == 'true' && + (needs.build-info.outputs.canary-run == 'true' || + needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' || + needs.build-info.outputs.full-tests-needed == 'true') + with: + default-branch: ${{ needs.build-info.outputs.default-branch }} + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + core-test-types-list-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + providers-test-types-list-as-strings-in-json: > + ${{ needs.build-info.outputs.providers-test-types-list-as-strings-in-json }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + python-versions: ${{ needs.build-info.outputs.python-versions }} + default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + + tests-integration-system: + name: Integration and System Tests + needs: [build-info, build-ci-images] + uses: ./.github/workflows/integration-system-tests.yml + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} + testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }} + run-system-tests: ${{ needs.build-info.outputs.run-system-tests }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} + default-mysql-version: ${{ needs.build-info.outputs.default-mysql-version }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + if: needs.build-info.outputs.run-unit-tests == 'true' + + tests-with-lowest-direct-resolution-core: + name: "Low dep tests:core" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/run-unit-tests.yml + permissions: + contents: read + packages: read + if: > + needs.build-info.outputs.run-unit-tests == 'true' + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + test-name: "LowestDeps" + force-lowest-dependencies: "true" + test-scope: "All" + test-group: "core" + backend: "sqlite" + python-versions: ${{ needs.build-info.outputs.python-versions }} + backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']" + excluded-providers-as-string: "" + excludes: "[]" + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.core-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + monitor-delay-time-in-seconds: 120 + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + + tests-with-lowest-direct-resolution-providers: + name: "Low dep tests: providers" + needs: [build-info, build-ci-images] + uses: ./.github/workflows/run-unit-tests.yml + permissions: + contents: read + packages: read + if: needs.build-info.outputs.run-unit-tests == 'true' + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + test-name: "LowestDeps" + force-lowest-dependencies: "true" + test-scope: "All" + test-group: "providers" + backend: "sqlite" + python-versions: ${{ needs.build-info.outputs.python-versions }} + backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']" + excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} + excludes: "[]" + test-types-as-strings-in-json: > + ${{ needs.build-info.outputs.individual-providers-test-types-list-as-strings-in-json }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + run-coverage: ${{ needs.build-info.outputs.run-coverage }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + monitor-delay-time-in-seconds: 120 + skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + + build-prod-images: + name: Build PROD images + needs: [build-info, build-ci-images, generate-constraints] + uses: ./.github/workflows/prod-image-build.yml + permissions: + contents: read + # This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs + # from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. + packages: write + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + build-type: "Regular" + push-image: "false" + upload-image-artifact: "true" + upload-package-artifact: "true" + python-versions: ${{ needs.build-info.outputs.python-versions }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + branch: ${{ needs.build-info.outputs.default-branch }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + prod-image-build: ${{ needs.build-info.outputs.prod-image-build }} + + additional-prod-image-tests: + name: "Additional PROD image tests" + needs: [build-info, build-prod-images, generate-constraints] + uses: ./.github/workflows/additional-prod-image-tests.yml + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + default-branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + run-task-sdk-integration-tests: ${{ needs.build-info.outputs.run-task-sdk-integration-tests }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + run-remote-logging-elasticsearch-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-elasticsearch-e2e-tests }} + run-remote-logging-opensearch-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-opensearch-e2e-tests }} + run-remote-logging-s3-e2e-tests: ${{ needs.build-info.outputs.run-remote-logging-s3-e2e-tests }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }} + run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }} + if: needs.build-info.outputs.prod-image-build == 'true' + + tests-kubernetes: + name: "Kubernetes tests" + uses: ./.github/workflows/k8s-tests.yml + needs: [build-info, build-prod-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + kubernetes-combos: ${{ needs.build-info.outputs.kubernetes-combos }} + if: > + ( needs.build-info.outputs.run-kubernetes-tests == 'true' || + needs.build-info.outputs.run-helm-tests == 'true') + + + tests-task-sdk: + name: "Task SDK tests" + uses: ./.github/workflows/airflow-distributions-tests.yml + needs: [build-info, build-ci-images] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + python-versions: ${{ needs.build-info.outputs.python-versions }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + distribution-name: "task-sdk" + distribution-cmd-format: "prepare-task-sdk-distributions" + test-type: "task-sdk-tests" + use-local-venv: 'false' + test-timeout: 20 + if: needs.build-info.outputs.run-task-sdk-tests == 'true' + + tests-go-sdk: + name: "Go SDK tests" + needs: [build-info] + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + timeout-minutes: 15 + permissions: + contents: read + packages: read + if: needs.build-info.outputs.run-go-sdk-tests == 'true' + env: + GITHUB_REPOSITORY: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_USERNAME: ${{ github.actor }} + VERBOSE: "true" + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + # keep this in sync with go.mod in go-sdk/ + - name: Setup Go + uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 + with: + go-version: 1.24 + cache-dependency-path: go-sdk/go.sum + # keep this in sync with go.mod in go-sdk/ + - name: Setup Gotestsum + shell: bash + run: | + go install gotest.tools/gotestsum@c4a0df2e75a225d979a444342dd3db752b53619f # v1.13.0 + gotestsum --version + - name: "Cleanup dist files" + run: rm -fv ./dist/* + - name: Run Go tests + working-directory: ./go-sdk + run: gotestsum --format github-actions ./... + + tests-airflow-ctl: + name: "Airflow CTL tests" + uses: ./.github/workflows/airflow-distributions-tests.yml + needs: [build-info] + permissions: + contents: read + packages: read + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + python-versions: ${{ needs.build-info.outputs.python-versions }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + distribution-name: "airflow-ctl" + distribution-cmd-format: "prepare-airflow-ctl-distributions" + test-type: "airflow-ctl-tests" + use-local-venv: 'true' + test-timeout: 20 + if: needs.build-info.outputs.run-airflow-ctl-tests == 'true' + + finalize-tests: + name: Finalize tests + permissions: + contents: write + packages: write + # This will fire when all the jobs from "needs" are either successful or skipped + if: always() && !failure() && !cancelled() + needs: + - additional-ci-image-checks + - additional-prod-image-tests + - basic-tests + - build-info + - build-prod-images + - ci-image-checks + - generate-constraints + - migration-round-trip + - mypy-providers + - providers + - tests-helm + - tests-integration-system + - tests-kubernetes + - tests-mysql-core + - tests-mysql-providers + - tests-non-db-core + - tests-non-db-providers + - tests-postgres-core + - tests-postgres-providers + - tests-sqlite-core + - tests-sqlite-providers + - tests-task-sdk + - tests-airflow-ctl + - tests-go-sdk + - tests-with-lowest-direct-resolution-core + - tests-with-lowest-direct-resolution-providers + uses: ./.github/workflows/finalize-tests.yml + with: + runners: ${{ needs.build-info.outputs.runner-type }} + platform: ${{ needs.build-info.outputs.platform }} + python-versions: ${{ needs.build-info.outputs.python-versions }} + python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} + branch: ${{ needs.build-info.outputs.default-branch }} + constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} + default-python-version: "${{ needs.build-info.outputs.default-python-version }}" + upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} + include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} + docker-cache: ${{ needs.build-info.outputs.docker-cache }} + disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} + canary-run: ${{ needs.build-info.outputs.canary-run }} + use-uv: ${{ needs.build-info.outputs.use-uv }} + debug-resources: ${{ needs.build-info.outputs.debug-resources }} + + notify-slack: + name: "Notify Slack" + needs: + - build-info + - finalize-tests + if: >- + always() && + !cancelled() && + github.event_name == 'schedule' && + github.run_attempt == 1 + runs-on: ["ubuntu-22.04"] + steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Get failing jobs" + id: get-failures + shell: bash + run: | + FAILED_JOBS=$(gh run view "${{ github.run_id }}" \ + --repo "${{ github.repository }}" \ + --json jobs \ + --jq '[.jobs[] | select(.conclusion == "failure") | .name] | sort | .[]') + echo "failed-jobs<> "${GITHUB_OUTPUT}" + echo "${FAILED_JOBS}" >> "${GITHUB_OUTPUT}" + echo "EOF" >> "${GITHUB_OUTPUT}" + if [[ -n "${FAILED_JOBS}" ]]; then + echo "has-failures=true" >> "${GITHUB_OUTPUT}" + else + echo "has-failures=false" >> "${GITHUB_OUTPUT}" + fi + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: "Determine notification action" + id: notification + shell: bash + run: | + command -v gh >/dev/null 2>&1 || { echo "::error::gh CLI is not installed"; exit 1; } + gh auth status || gh auth login --with-token <<< "${GITHUB_TOKEN}" + python3 scripts/ci/slack_notification_state.py + env: + ARTIFACT_NAME: "slack-state-tests-${{ github.ref_name }}" + CURRENT_FAILURES: "${{ steps.get-failures.outputs.failed-jobs }}" + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: "Upload notification state" + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: "slack-state-tests-${{ github.ref_name }}" + path: ./slack-state/ + retention-days: 7 + overwrite: true + - name: "Notify Slack (new/changed failures)" + if: steps.notification.outputs.action == 'notify_new' + uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1 + with: + method: chat.postMessage + token: ${{ env.SLACK_BOT_TOKEN }} + # yamllint disable rule:line-length + payload: | + channel: "internal-airflow-ci-cd" + text: "🚨 Failure Alert: Scheduled CI (${{ needs.build-info.outputs.platform }}) on branch *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs }}\n\n*Details:* " + blocks: + - type: "section" + text: + type: "mrkdwn" + text: "🚨 Failure Alert: Scheduled CI (${{ needs.build-info.outputs.platform }}) on *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs }}\n\n*Details:* " + # yamllint enable rule:line-length + - name: "Notify Slack (still not fixed)" + if: steps.notification.outputs.action == 'notify_reminder' + uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1 + with: + method: chat.postMessage + token: ${{ env.SLACK_BOT_TOKEN }} + # yamllint disable rule:line-length + payload: | + channel: "internal-airflow-ci-cd" + text: "πŸš¨πŸ” Still not fixed: Scheduled CI (${{ needs.build-info.outputs.platform }}) on branch *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs }}\n\n*Details:* " + blocks: + - type: "section" + text: + type: "mrkdwn" + text: "πŸš¨πŸ” Still not fixed: Scheduled CI (${{ needs.build-info.outputs.platform }}) on *${{ github.ref_name }}*\n\nFailing jobs:\n${{ steps.get-failures.outputs.failed-jobs }}\n\n*Details:* " + # yamllint enable rule:line-length + - name: "Notify Slack (all tests passing)" + if: steps.notification.outputs.action == 'notify_recovery' + uses: slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 # v3.0.1 + with: + method: chat.postMessage + token: ${{ env.SLACK_BOT_TOKEN }} + # yamllint disable rule:line-length + payload: | + channel: "internal-airflow-ci-cd" + text: "βœ… All tests passing: Scheduled CI (${{ needs.build-info.outputs.platform }}) on branch *${{ github.ref_name }}*\n\n*Details:* " + blocks: + - type: "section" + text: + type: "mrkdwn" + text: "βœ… All tests passing: Scheduled CI (${{ needs.build-info.outputs.platform }}) on *${{ github.ref_name }}*\n\n*Details:* " + # yamllint enable rule:line-length + + summarize-warnings: + timeout-minutes: 15 + name: "Summarize warnings" + needs: + - build-info + - tests-mysql-core + - tests-mysql-providers + - tests-non-db-core + - tests-non-db-providers + - tests-postgres-core + - tests-postgres-providers + - tests-sqlite-core + - tests-sqlite-providers + - tests-task-sdk + - tests-airflow-ctl + - tests-special + - tests-with-lowest-direct-resolution-core + - tests-with-lowest-direct-resolution-providers + runs-on: ${{ fromJSON(needs.build-info.outputs.runner-type) }} + if: needs.build-info.outputs.run-unit-tests == 'true' + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: "Free up disk space" + shell: bash + run: ./scripts/tools/free_up_disk_space.sh + - name: "Download all test warning artifacts from the current build" + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: ./artifacts + pattern: test-warnings-* + - name: "Setup python" + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "${{ inputs.default-python-version }}" + - name: "Summarize all warnings" + run: | + ./scripts/ci/testing/summarize_captured_warnings.py ./artifacts \ + --pattern "**/warnings-*.txt" \ + --output ./files + - name: "Upload artifact for summarized warnings" + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: "test-summarized-warnings" + path: ./files/warn-summary-*.txt + retention-days: 7 + if-no-files-found: ignore + overwrite: true diff --git a/.github/workflows/ci-notification.yml b/.github/workflows/ci-notification.yml index f39702237df50..88938c819acaa 100644 --- a/.github/workflows/ci-notification.yml +++ b/.github/workflows/ci-notification.yml @@ -37,7 +37,9 @@ jobs: strategy: matrix: branch: ["v3-2-test"] - workflow-id: ["ci-amd-arm.yml"] + # Track AMD; ARM is the canary slot and is reported separately by + # `ci-arm.yml`'s notify-slack job on schedule events. + workflow-id: ["ci-amd.yml"] runs-on: ubuntu-latest steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" diff --git a/.github/workflows/e2e-flaky-tests-report.yml b/.github/workflows/e2e-flaky-tests-report.yml index 1f154dd236e16..6bf3b51d34c82 100644 --- a/.github/workflows/e2e-flaky-tests-report.yml +++ b/.github/workflows/e2e-flaky-tests-report.yml @@ -46,7 +46,10 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} MAX_RUNS: "10" - WORKFLOW_NAME: "ci-amd-arm.yml" + # E2E test runs come from the AMD workflow (every PR / push to main + # goes through it). The ARM canary is too sparse to drive flakiness + # analysis, so we keep the report tied to AMD. + WORKFLOW_NAME: "ci-amd.yml" BRANCH: "main" OUTPUT_FILE: "slack-message.json" diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index a1326e74d7642..dbff0e42201ce 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -92,7 +92,7 @@ jobs: - name: "Install Breeze" uses: ./.github/actions/breeze - name: "Save github context to file" - # See ci-amd-arm.yml for the full rationale: avoids ARG_MAX on big PRs by writing the + # See ci-amd.yml for the full rationale: avoids ARG_MAX on big PRs by writing the # github context to a file, and the single-quoted heredoc makes the zizmor # template-injection finding a false positive (no bash expansion happens inside it). shell: bash diff --git a/.github/workflows/update-constraints-on-push-stable.yml b/.github/workflows/update-constraints-on-push-stable.yml index 71a20ad357d00..bdd14da85ae7f 100644 --- a/.github/workflows/update-constraints-on-push-stable.yml +++ b/.github/workflows/update-constraints-on-push-stable.yml @@ -60,7 +60,7 @@ jobs: uses: ./.github/actions/breeze id: breeze - name: "Save github context to file" - # See ci-amd-arm.yml for the full rationale: avoids ARG_MAX on big PRs by writing the + # See ci-amd.yml for the full rationale: avoids ARG_MAX on big PRs by writing the # github context to a file, and the single-quoted heredoc makes the zizmor # template-injection finding a false positive (no bash expansion happens inside it). shell: bash diff --git a/.github/workflows/update-constraints-on-push.yml b/.github/workflows/update-constraints-on-push.yml index e910c4dd24a70..0b063b9ea55ba 100644 --- a/.github/workflows/update-constraints-on-push.yml +++ b/.github/workflows/update-constraints-on-push.yml @@ -63,7 +63,7 @@ jobs: uses: ./.github/actions/breeze id: breeze - name: "Save github context to file" - # See ci-amd-arm.yml for the full rationale: avoids ARG_MAX on big PRs by writing the + # See ci-amd.yml for the full rationale: avoids ARG_MAX on big PRs by writing the # github context to a file, and the single-quoted heredoc makes the zizmor # template-injection finding a false positive (no bash expansion happens inside it). shell: bash diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index e84a272c5ff35..896e67372b51b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -235,6 +235,13 @@ repos: ^pyproject\.toml$ pass_filenames: false require_serial: true + - id: check-ci-workflows-in-sync + name: Check ci-arm.yml and ci-amd.yml stay in sync + entry: ./scripts/ci/prek/check_ci_workflows_in_sync.py + language: python + files: ^\.github/workflows/ci-(arm|amd)\.yml$ + pass_filenames: false + require_serial: true - id: sync-uv-min-version-markers name: Sync `# sync-uv-min-version` markers with [tool.uv] required-version entry: ./scripts/ci/prek/sync_uv_min_version_markers.py diff --git a/README.md b/README.md index 74eb53e228ffa..a659243961629 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ | Version | Build Status | |---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Main | [![GitHub Build main](https://github.com/apache/airflow/actions/workflows/ci-amd-arm.yml/badge.svg)](https://github.com/apache/airflow/actions) | -| 3.x | [![GitHub Build 3.2](https://github.com/apache/airflow/actions/workflows/ci-amd-arm.yml/badge.svg?branch=v3-2-test)](https://github.com/apache/airflow/actions) | +| Main | [![GitHub Build main](https://github.com/apache/airflow/actions/workflows/ci-amd.yml/badge.svg)](https://github.com/apache/airflow/actions) | +| 3.x | [![GitHub Build 3.2](https://github.com/apache/airflow/actions/workflows/ci-amd.yml/badge.svg?branch=v3-2-test)](https://github.com/apache/airflow/actions) | | 2.x | [![GitHub Build 2.11](https://github.com/apache/airflow/actions/workflows/ci.yml/badge.svg?branch=v2-11-test)](https://github.com/apache/airflow/actions) | diff --git a/dev/README_AIRFLOW3_DEV.md b/dev/README_AIRFLOW3_DEV.md index ad48f143b95ea..a947d5d01cc09 100644 --- a/dev/README_AIRFLOW3_DEV.md +++ b/dev/README_AIRFLOW3_DEV.md @@ -80,7 +80,7 @@ PRs should **never** target `v3-2-stable` directly unless explicitly instructed > likely need additional changes before GA. However, when using this shortcut, the release manager **must** > verify that the `v3-X-test` push CI action ("Tests" workflow) has succeeded before cutting the RC. You can > check this at: -> https://github.com/apache/airflow/actions/workflows/ci-amd-arm.yml?query=event%3Apush+branch%3Av3-2-test +> https://github.com/apache/airflow/actions/workflows/ci-amd.yml?query=event%3Apush+branch%3Av3-2-test > (adjust the branch filter for the relevant `v3-X-test` branch). ## Developing for Airflow 3 diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 2f4c3020fb2c2..f7d8dc7961d55 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -535,7 +535,7 @@ still works but is no longer recommended. > likely need additional changes before GA. However, when using this shortcut, the release manager **must** > verify that the `v3-X-test` push CI action ("Tests" workflow) has succeeded before cutting the RC. You can > check this at: -> https://github.com/apache/airflow/actions/workflows/ci-amd-arm.yml?query=event%3Apush+branch%3Av3-2-test +> https://github.com/apache/airflow/actions/workflows/ci-amd.yml?query=event%3Apush+branch%3Av3-2-test > (adjust the branch filter for the relevant `v3-X-test` branch). - When the PR is approved (or when using the shortcut above), install `dev/breeze` in a virtualenv: diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 27660b3fea841..288598ba0f7a4 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -1594,6 +1594,12 @@ def selected_providers_list_as_string(self) -> str | None: def get_job_label(self, event_type: str, branch: str): import requests # type: ignore[import-untyped] + # The main CI is now split into ci-arm.yml and ci-amd.yml; the old + # ci-amd-arm.yml file no longer exists. This lookup is dormant for the + # main pipeline (which hardcodes runner-type per wrapper) and only + # remains here for the `is_disabled_integration` code path that still + # reads `runner_type`. The API call against a missing workflow returns + # nothing and the caller falls back to PUBLIC_AMD_RUNNERS. job_name = "Basic tests" workflow_name = "ci-amd-arm.yml" headers = {"Accept": "application/vnd.github.v3+json"} diff --git a/dev/update_github_branch_config.py b/dev/update_github_branch_config.py index 0cdb1691dd00f..6ae10426f1fc9 100755 --- a/dev/update_github_branch_config.py +++ b/dev/update_github_branch_config.py @@ -303,12 +303,12 @@ def update_readme_badge(new_branch: str, prev_branch: str, new_version: str, pre old_badge = ( f"[![GitHub Build {prev_version}" - f"](https://github.com/apache/airflow/actions/workflows/ci-amd-arm.yml/badge.svg" + f"](https://github.com/apache/airflow/actions/workflows/ci-amd.yml/badge.svg" f"?branch={prev_branch})](https://github.com/apache/airflow/actions)" ) new_badge = ( f"[![GitHub Build {new_version}" - f"](https://github.com/apache/airflow/actions/workflows/ci-amd-arm.yml/badge.svg" + f"](https://github.com/apache/airflow/actions/workflows/ci-amd.yml/badge.svg" f"?branch={new_branch})](https://github.com/apache/airflow/actions)" ) diff --git a/generated/PYPI_README.md b/generated/PYPI_README.md index 191997bd26046..2372610eeb432 100644 --- a/generated/PYPI_README.md +++ b/generated/PYPI_README.md @@ -32,8 +32,8 @@ PROJECT BY THE `generate-pypi-readme` PREK HOOK. YOUR CHANGES HERE WILL BE AUTOM | Version | Build Status | |---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Main | [![GitHub Build main](https://github.com/apache/airflow/actions/workflows/ci-amd-arm.yml/badge.svg)](https://github.com/apache/airflow/actions) | -| 3.x | [![GitHub Build 3.2](https://github.com/apache/airflow/actions/workflows/ci-amd-arm.yml/badge.svg?branch=v3-2-test)](https://github.com/apache/airflow/actions) | +| Main | [![GitHub Build main](https://github.com/apache/airflow/actions/workflows/ci-amd.yml/badge.svg)](https://github.com/apache/airflow/actions) | +| 3.x | [![GitHub Build 3.2](https://github.com/apache/airflow/actions/workflows/ci-amd.yml/badge.svg?branch=v3-2-test)](https://github.com/apache/airflow/actions) | | 2.x | [![GitHub Build 2.11](https://github.com/apache/airflow/actions/workflows/ci.yml/badge.svg?branch=v2-11-test)](https://github.com/apache/airflow/actions) | diff --git a/scripts/ci/analyze_e2e_flaky_tests.py b/scripts/ci/analyze_e2e_flaky_tests.py index c56fb7e48ebfc..a8942d7e261b2 100755 --- a/scripts/ci/analyze_e2e_flaky_tests.py +++ b/scripts/ci/analyze_e2e_flaky_tests.py @@ -34,7 +34,7 @@ Environment variables (optional): MAX_RUNS - Maximum number of workflow runs to analyze (default: 10) - WORKFLOW_NAME - Workflow file name to query (default: ci-amd-arm.yml) + WORKFLOW_NAME - Workflow file name to query (default: ci-amd.yml) BRANCH - Branch to filter runs (default: main) OUTPUT_FILE - Path for the Slack message output (default: slack-message.json) GITHUB_OUTPUT - Path to GitHub Actions output file @@ -525,7 +525,7 @@ def write_step_summary( def main() -> None: repo = os.environ.get("GITHUB_REPOSITORY", "apache/airflow") max_runs = int(os.environ.get("MAX_RUNS", "10")) - workflow = os.environ.get("WORKFLOW_NAME", "ci-amd-arm.yml") + workflow = os.environ.get("WORKFLOW_NAME", "ci-amd.yml") branch = os.environ.get("BRANCH", "main") output_file = Path(os.environ.get("OUTPUT_FILE", "slack-message.json")) diff --git a/scripts/ci/prek/check_ci_workflows_in_sync.py b/scripts/ci/prek/check_ci_workflows_in_sync.py new file mode 100755 index 0000000000000..3a219edd23279 --- /dev/null +++ b/scripts/ci/prek/check_ci_workflows_in_sync.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Verify ci-arm.yml and ci-amd.yml stay in sync. + +The two workflows are physical copies of each other (GitHub Actions has no +cross-file YAML include) and may differ ONLY in the small set of intentional +divergences listed below. This check normalizes both files against that +allowlist, then asserts the rest matches byte-for-byte. + +Documented divergences: + +1. Header intro comment (different prose explaining what each file is for) +2. Workflow ``name:`` β€” ``Tests (ARM)`` vs ``Tests (AMD)`` +3. Triggers β€” ARM = schedule + workflow_dispatch only; AMD = pull_request + + push (to release branches) + workflow_dispatch +4. ``concurrency.group`` prefix β€” ``ci-arm-`` vs ``ci-amd-`` +5. ``build-info`` outputs ``platform`` and ``runner-type`` β€” hardcoded per + architecture (and the surrounding comment naming the "ARM/AMD copy") +6. ``print-platform`` job β€” ``name:`` and the architecture echoed to + GITHUB_STEP_SUMMARY + +Anything else differing between the two files is a drift bug. To +intentionally introduce a new divergence, update the rules in this script +in the same PR. + +Run from the repo root: + + uv run --project scripts python scripts/ci/prek/check_ci_workflows_in_sync.py + +Exits 0 if the workflows are in sync, 1 (with a diff) otherwise. + +Local UX +-------- + +When run interactively (TTY attached, ``CI`` env var unset) the script +also tries to figure out which side changed since the branch diverged +from ``main`` and prints concrete next steps β€” either mirror the change +to the other file, or add it to the per-line / per-block allowlist below. +""" + +from __future__ import annotations + +import difflib +import os +import re +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[3] +ARM = REPO_ROOT / ".github" / "workflows" / "ci-arm.yml" +AMD = REPO_ROOT / ".github" / "workflows" / "ci-amd.yml" +THIS_SCRIPT = Path(__file__).relative_to(REPO_ROOT) + +# Per-line regex normalizations applied to BOTH files. Each rule replaces +# the architecture-specific token with a placeholder so the two normalized +# files become identical. If you add a per-line difference between ARM and +# AMD, append a rule here. +LINE_RULES: list[tuple[str, str]] = [ + (r"^name: Tests \((?:ARM|AMD)\)$", "name: Tests (PLACEHOLDER)"), + (r"^ group: ci-(?:arm|amd)-", " group: ci-PLACEHOLDER-"), + (r'^ platform: "linux/(?:arm64|amd64)"$', ' platform: "linux/PLACEHOLDER"'), + (r"^ runner-type: '\[\"ubuntu-22\.04(?:-arm)?\"\]'$", " runner-type: 'PLACEHOLDER'"), + ( + r"^ # (?:ARM|AMD) copy\)\. The matching (?:AMD|ARM) copy lives in ci-(?:amd|arm)\.yml\.$", + " # PLACEHOLDER copy). The matching PLACEHOLDER copy lives in ci-PLACEHOLDER.yml.", + ), + (r'^ name: "Platform: (?:ARM|AMD)"$', ' name: "Platform: PLACEHOLDER"'), + ( + r"^ run: \"echo '## Architecture: (?:ARM|AMD)' >> \$GITHUB_STEP_SUMMARY\"$", + " run: \"echo '## Architecture: PLACEHOLDER' >> $GITHUB_STEP_SUMMARY\"", + ), +] + +# Whole sections that legitimately exist in only one file. Each entry is the +# verbatim block (including trailing newline) that must appear in the named +# file and must NOT appear in the other. The blocks are stripped before +# diffing so they don't show up as drift. +ARM_ONLY_BLOCK = """ schedule: + - cron: '28 1,3,7,9,13,15,19,21 * * *' +""" + +AMD_ONLY_BLOCK = """ push: + branches: + - v[0-9]+-[0-9]+-test + - providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+ + pull_request: + branches: + - main + - v[0-9]+-[0-9]+-test + - v[0-9]+-[0-9]+-stable + - providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+ + types: [opened, reopened, synchronize, ready_for_review] +""" + +# The header comment block (between the `---` document marker and the +# `name:` line) is intentionally different between files β€” each describes +# what its own file is for. Collapse to a placeholder before diffing. +HEADER_PATTERN = re.compile(r"(?<=\n---\n)(.*?)(?=^name: )", re.MULTILINE | re.DOTALL) +HEADER_PLACEHOLDER = "# PLACEHOLDER: per-file intro comment\n" + + +# --- ANSI color helpers ------------------------------------------------------- +# Local UX: colorize the report when stdout is a TTY and we're not in CI. + +_INTERACTIVE = os.environ.get("FORCE_INTERACTIVE") == "1" or ( + sys.stdout.isatty() and not os.environ.get("CI") and not os.environ.get("PREK_HEADLESS") +) + + +def _ansi(code: str, text: str) -> str: + if not _INTERACTIVE: + return text + return f"\033[{code}m{text}\033[0m" + + +def red(t: str) -> str: + return _ansi("1;31", t) + + +def green(t: str) -> str: + return _ansi("1;32", t) + + +def yellow(t: str) -> str: + return _ansi("1;33", t) + + +def cyan(t: str) -> str: + return _ansi("1;36", t) + + +def bold(t: str) -> str: + return _ansi("1", t) + + +# --- normalization ------------------------------------------------------------ + + +def normalize(content: str, *, side: str) -> str: + """Strip side-specific blocks and apply line rules so both files compare equal.""" + if side == "arm": + if ARM_ONLY_BLOCK not in content: + raise SystemExit(f"ci-arm.yml is missing its expected ARM-only trigger block:\n{ARM_ONLY_BLOCK}") + if AMD_ONLY_BLOCK in content: + raise SystemExit("ci-arm.yml contains the AMD-only trigger block β€” should not.") + content = content.replace(ARM_ONLY_BLOCK, "") + elif side == "amd": + if AMD_ONLY_BLOCK not in content: + raise SystemExit(f"ci-amd.yml is missing its expected AMD-only trigger block:\n{AMD_ONLY_BLOCK}") + if ARM_ONLY_BLOCK in content: + raise SystemExit("ci-amd.yml contains the ARM-only trigger block β€” should not.") + content = content.replace(AMD_ONLY_BLOCK, "") + else: + raise ValueError(f"Unknown side: {side}") + + for pattern, replacement in LINE_RULES: + content = re.sub(pattern, replacement, content, flags=re.MULTILINE) + content = HEADER_PATTERN.sub(HEADER_PLACEHOLDER, content) + return content + + +# --- main-branch comparison (local UX) ---------------------------------------- + + +def _git(args: list[str]) -> str | None: + """Run git from the repo root; return stdout text on success, None on failure.""" + try: + result = subprocess.run( + ["git", "-C", str(REPO_ROOT), *args], + capture_output=True, + text=True, + check=False, + ) + except FileNotFoundError: + return None + if result.returncode != 0: + return None + return result.stdout + + +def _resolve_baseline_ref() -> str | None: + """Pick a reasonable baseline ref to diff against β€” the closest of upstream/main, origin/main, or main.""" + for candidate in ("upstream/main", "origin/main", "main"): + if _git(["rev-parse", "--verify", "--quiet", candidate]) is not None: + return candidate + return None + + +def _diff_since_baseline(file_path: Path, baseline: str) -> str | None: + """Return the unified diff of ``file_path`` from baseline to the working tree. + + Uses ``git diff -- `` (NOT the three-dot form) so the + comparison covers committed AND uncommitted edits. The check needs to + catch a developer who edits one file and forgets to mirror it before + even running ``git add``. + """ + rel = file_path.relative_to(REPO_ROOT).as_posix() + output = _git(["diff", "--no-color", baseline, "--", rel]) + if output is None: + return None + return output.strip() or None + + +def _is_new_file_diff(diff: str | None) -> bool: + """True if the diff describes the file as added (no baseline content to compare).""" + return ( + bool(diff) and "new file mode" in diff.splitlines()[1] + if diff and len(diff.splitlines()) > 1 + else False + ) + + +def _trim_diff(diff: str, max_lines: int = 60) -> str: + """Cap diff to max_lines so the report stays readable; mention how many lines were dropped.""" + lines = diff.splitlines() + if len(lines) <= max_lines: + return diff + dropped = len(lines) - max_lines + return "\n".join( + lines[:max_lines] + [f"... ({dropped} more lines suppressed; run `git diff` to see full)"] + ) + + +def _suggest_actions(arm_diff: str | None, amd_diff: str | None) -> None: + """Print concrete suggestions about which file to edit, based on which side changed.""" + print() + print(bold("Suggested next steps:")) + print() + arm_new = _is_new_file_diff(arm_diff) + amd_new = _is_new_file_diff(amd_diff) + if arm_new and amd_new: + print( + f" β€’ Both {cyan('ci-arm.yml')} and {cyan('ci-amd.yml')} were added in this branch β€” " + f"there is no baseline to attribute the drift to. Compare the two files directly " + f"({cyan('diff .github/workflows/ci-arm.yml .github/workflows/ci-amd.yml')}) and " + f"either pick one as the source of truth or extend the allowlist in " + f"{cyan(str(THIS_SCRIPT))}." + ) + return + if arm_diff and not amd_diff: + print( + f" β€’ Only {cyan('ci-arm.yml')} has changed since the baseline. To bring AMD in line, " + f"mirror the change into {cyan('ci-amd.yml')}:" + ) + print() + print(_indent(_trim_diff(arm_diff))) + print() + print( + f" β€’ If the change is intentionally ARM-only, add the new lines to " + f"{cyan('LINE_RULES')} (or {cyan('ARM_ONLY_BLOCK')}) in {cyan(str(THIS_SCRIPT))}." + ) + elif amd_diff and not arm_diff: + print( + f" β€’ Only {cyan('ci-amd.yml')} has changed since the baseline. To bring ARM in line, " + f"mirror the change into {cyan('ci-arm.yml')}:" + ) + print() + print(_indent(_trim_diff(amd_diff))) + print() + print( + f" β€’ If the change is intentionally AMD-only, add the new lines to " + f"{cyan('LINE_RULES')} (or {cyan('AMD_ONLY_BLOCK')}) in {cyan(str(THIS_SCRIPT))}." + ) + elif arm_diff and amd_diff: + print( + " β€’ Both files have changes since the baseline. Compare the two diffs below " + "and decide which side is the source of truth, then mirror manually." + ) + print() + print(yellow("ci-arm.yml diff vs baseline:")) + print(_indent(_trim_diff(arm_diff))) + print() + print(yellow("ci-amd.yml diff vs baseline:")) + print(_indent(_trim_diff(amd_diff))) + print() + print(f" β€’ If the divergence is intentional, document it in {cyan(str(THIS_SCRIPT))}.") + else: + # Neither file changed against the baseline, but the normalized diff is non-empty. + # That means the drift was already present before the branch β€” likely a bug in this + # script's allowlist or a pre-existing inconsistency that nobody caught. + print( + f" β€’ Neither file changed since the baseline, yet the normalized diff is non-empty. " + f"This usually means {cyan(str(THIS_SCRIPT))}'s allowlist is missing a rule for a " + f"pre-existing divergence. Inspect the diff above and update LINE_RULES / " + f"ARM_ONLY_BLOCK / AMD_ONLY_BLOCK accordingly." + ) + + +def _indent(text: str, prefix: str = " ") -> str: + return "\n".join(prefix + line for line in text.splitlines()) + + +# --- entry point -------------------------------------------------------------- + + +def main() -> int: + if not ARM.exists() or not AMD.exists(): + print(red(f"ERROR: expected both {ARM.name} and {AMD.name} to exist under .github/workflows/.")) + return 1 + + arm_normalized = normalize(ARM.read_text(), side="arm") + amd_normalized = normalize(AMD.read_text(), side="amd") + + if arm_normalized == amd_normalized: + print(green(f"OK: {ARM.name} and {AMD.name} are in sync (apart from documented divergences).")) + return 0 + + print(red(f"ERROR: {ARM.name} and {AMD.name} have diverged outside the allowed set.")) + print() + print(bold("Normalized diff:")) + print() + diff = difflib.unified_diff( + arm_normalized.splitlines(), + amd_normalized.splitlines(), + fromfile=f"{ARM.name} (normalized)", + tofile=f"{AMD.name} (normalized)", + lineterm="", + ) + for line in diff: + if line.startswith("+") and not line.startswith("+++"): + print(green(line)) + elif line.startswith("-") and not line.startswith("---"): + print(red(line)) + elif line.startswith("@@"): + print(cyan(line)) + else: + print(line) + + if _INTERACTIVE: + baseline = _resolve_baseline_ref() + if baseline is None: + print() + print( + yellow( + "Could not resolve a baseline ref (tried upstream/main, origin/main, main). " + "Skipping the per-side change attribution; fix manually using the diff above." + ) + ) + return 1 + arm_diff = _diff_since_baseline(ARM, baseline) + amd_diff = _diff_since_baseline(AMD, baseline) + print() + print(bold(f"Comparing each file against {cyan(baseline)} to identify the changed side…")) + _suggest_actions(arm_diff, amd_diff) + else: + print() + print( + yellow( + "Run this script locally outside CI to see which side changed and get a " + "concrete suggestion (mirror the change, or add it to the allowlist in this script)." + ) + ) + + return 1 + + +if __name__ == "__main__": + sys.exit(main())