From 0c4f14e61151a704835f6f57cdefcbb50f01c9dc Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 10 Sep 2025 08:01:35 -0600 Subject: [PATCH 1/6] Add CI job to test airflow standalone cli command --- .github/workflows/basic-tests.yml | 38 +++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 548b9ce5b0fae..916ade1aaf323 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -394,3 +394,41 @@ jobs: run: | breeze release-management generate-issue-content-core \ --limit-pr-count 2 --previous-release 3.0.1 --current-release 3.0.2 --verbose + + test-airflow-standalone: + timeout-minutes: 30 + name: "Test Airflow standalone command" + runs-on: ${{ fromJSON(inputs.runners) }} + env: + PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + steps: + - name: "Cleanup repo" + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + persist-credentials: false + - name: "Install Breeze" + uses: ./.github/actions/breeze + - name: "Test airflow standalone command" + run: | + # Start airflow standalone and wait for "Airflow is ready" message + timeout 600s breeze run bash -c " + airflow standalone 2>&1 | tee /tmp/airflow.log & + AIRFLOW_PID=\$! + + for i in {1..600}; do + if grep -q 'Airflow is ready' /tmp/airflow.log; then + echo 'Airflow standalone is ready!' + kill \$AIRFLOW_PID + exit 0 + fi + sleep 1 + done + + echo 'FAILED: Airflow standalone did not start up in time' + kill \$AIRFLOW_PID + exit 1 + " + From ae6a6525c568584e91819376303b6c90cf319731 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 10 Sep 2025 10:07:12 -0600 Subject: [PATCH 2/6] Ci for standalone --- .github/workflows/basic-tests.yml | 56 ++++++++++++++++++------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 916ade1aaf323..33ee5e0692284 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -397,38 +397,48 @@ jobs: test-airflow-standalone: timeout-minutes: 30 - name: "Test Airflow standalone command" + name: "Test Airflow standalone commands" runs-on: ${{ fromJSON(inputs.runners) }} env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" + UV_VERSION: ${{inputs.uv-version}} + AIRFLOW_HOME: ~/airflow steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: persist-credentials: false - - name: "Install Breeze" - uses: ./.github/actions/breeze + - name: "Install uv" + run: curl -LsSf https://astral.sh/uv/${UV_VERSION}/install.sh | sh + - name: "Set up Airflow home directory" + run: | + echo "Setting AIRFLOW_HOME to $AIRFLOW_HOME" + mkdir -p $AIRFLOW_HOME + - name: "Install Airflow from current repo (simulating user installation)" + run: | + uv pip install -e ./airflow-core - name: "Test airflow standalone command" run: | - # Start airflow standalone and wait for "Airflow is ready" message - timeout 600s breeze run bash -c " - airflow standalone 2>&1 | tee /tmp/airflow.log & - AIRFLOW_PID=\$! + airflow standalone 2>&1 | tee airflow_startup.log & + AIRFLOW_PID=$! + + # Wait for ready message till timeout (10 minutes) + for i in {1..600}; do + if ! kill -0 $AIRFLOW_PID 2>/dev/null; then + wait $AIRFLOW_PID + EXIT_CODE=$? + echo "FAILED: Airflow standalone exited with code $EXIT_CODE" + exit $EXIT_CODE + fi - for i in {1..600}; do - if grep -q 'Airflow is ready' /tmp/airflow.log; then - echo 'Airflow standalone is ready!' - kill \$AIRFLOW_PID - exit 0 - fi - sleep 1 - done + if grep -q "Airflow is ready" airflow_startup.log; then + echo "SUCCESS: Airflow standalone is ready!" + kill $AIRFLOW_PID + exit 0 + fi - echo 'FAILED: Airflow standalone did not start up in time' - kill \$AIRFLOW_PID - exit 1 - " + sleep 1 + done + echo "FAILED: Airflow standalone did not become ready in time" + kill $AIRFLOW_PID 2>/dev/null || true + exit 1 From ece25b64bf52a8be58098f82a9966bb0f4dd4dae Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 10 Sep 2025 10:13:30 -0600 Subject: [PATCH 3/6] Ci for standalone --- .github/workflows/basic-tests.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 33ee5e0692284..91eeba198b618 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -415,6 +415,8 @@ jobs: mkdir -p $AIRFLOW_HOME - name: "Install Airflow from current repo (simulating user installation)" run: | + uv venv + source .venv/bin/activate uv pip install -e ./airflow-core - name: "Test airflow standalone command" run: | From 25b96491cff51274b895a273d5da440cbfcf756e Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 10 Sep 2025 10:21:34 -0600 Subject: [PATCH 4/6] Ci for standalone --- .github/workflows/basic-tests.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 91eeba198b618..b35c2a79533d9 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -420,9 +420,10 @@ jobs: uv pip install -e ./airflow-core - name: "Test airflow standalone command" run: | + source .venv/bin/activate airflow standalone 2>&1 | tee airflow_startup.log & AIRFLOW_PID=$! - + # Wait for ready message till timeout (10 minutes) for i in {1..600}; do if ! kill -0 $AIRFLOW_PID 2>/dev/null; then @@ -431,16 +432,16 @@ jobs: echo "FAILED: Airflow standalone exited with code $EXIT_CODE" exit $EXIT_CODE fi - + if grep -q "Airflow is ready" airflow_startup.log; then echo "SUCCESS: Airflow standalone is ready!" kill $AIRFLOW_PID exit 0 fi - + sleep 1 done - + echo "FAILED: Airflow standalone did not become ready in time" kill $AIRFLOW_PID 2>/dev/null || true exit 1 From 0082f6c973ff28221ec4899ec48edcabd11cd98b Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Wed, 10 Sep 2025 12:17:10 -0600 Subject: [PATCH 5/6] Ci for standalone --- .github/workflows/basic-tests.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index b35c2a79533d9..cbe68805ad9d6 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -423,7 +423,7 @@ jobs: source .venv/bin/activate airflow standalone 2>&1 | tee airflow_startup.log & AIRFLOW_PID=$! - + # Wait for ready message till timeout (10 minutes) for i in {1..600}; do if ! kill -0 $AIRFLOW_PID 2>/dev/null; then @@ -432,16 +432,19 @@ jobs: echo "FAILED: Airflow standalone exited with code $EXIT_CODE" exit $EXIT_CODE fi - + if grep -q "Airflow is ready" airflow_startup.log; then echo "SUCCESS: Airflow standalone is ready!" kill $AIRFLOW_PID exit 0 fi - + sleep 1 done - + echo "FAILED: Airflow standalone did not become ready in time" + echo "=== AIRFLOW STARTUP LOGS ===" + cat airflow_startup.log + echo "=== END AIRFLOW STARTUP LOGS ===" kill $AIRFLOW_PID 2>/dev/null || true exit 1 From 523ed31b585ae6d0a48df236aaccdd83af40af48 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Thu, 11 Sep 2025 07:41:21 -0600 Subject: [PATCH 6/6] Ci for standalone --- .github/actions/install-pre-commit/action.yml | 88 +++++++++++++++++++ .github/workflows/basic-tests.yml | 9 +- 2 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 .github/actions/install-pre-commit/action.yml diff --git a/.github/actions/install-pre-commit/action.yml b/.github/actions/install-pre-commit/action.yml new file mode 100644 index 0000000000000..c0806bf83ea50 --- /dev/null +++ b/.github/actions/install-pre-commit/action.yml @@ -0,0 +1,88 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: 'Install pre-commit' +description: 'Installs pre-commit and related packages' +inputs: + python-version: + description: 'Python version to use' + default: "3.9" + uv-version: + description: 'uv version to use' + default: "0.7.3" # Keep this comment to allow automatic replacement of uv version + pre-commit-version: + description: 'pre-commit version to use' + default: "4.2.0" # Keep this comment to allow automatic replacement of pre-commit version + pre-commit-uv-version: + description: 'pre-commit-uv version to use' + default: "4.1.4" # Keep this comment to allow automatic replacement of pre-commit-uv version +runs: + using: "composite" + steps: + - name: Install pre-commit, uv, and pre-commit-uv + shell: bash + env: + UV_VERSION: ${{inputs.uv-version}} + PRE_COMMIT_VERSION: ${{inputs.pre-commit-version}} + PRE_COMMIT_UV_VERSION: ${{inputs.pre-commit-uv-version}} + run: | + pip install uv==${UV_VERSION} || true + uv tool install pre-commit==${PRE_COMMIT_VERSION} --with uv==${UV_VERSION} \ + --with pre-commit-uv==${PRE_COMMIT_UV_VERSION} + working-directory: ${{ github.workspace }} + # We need to use tar file with archive to restore all the permissions and symlinks + - name: "Delete ~.cache" + run: | + du ~/ --max-depth=2 + echo + echo Deleting ~/.cache + echo + rm -rf ~/.cache + echo + shell: bash + - name: "Restore pre-commit cache" + uses: apache/infrastructure-actions/stash/restore@1c35b5ccf8fba5d4c3fdf25a045ca91aa0cbc468 + with: + key: cache-pre-commit-v4-${{ inputs.python-version }}-${{ hashFiles('.pre-commit-config.yaml') }} + path: /tmp/ + id: restore-pre-commit-cache + - name: "Check if pre-commit cache tarball exists" + shell: bash + run: | + if [ -f /tmp/cache-pre-commit.tar.gz ]; then + echo "✅ Cache tarball found: /tmp/cache-pre-commit.tar.gz" + else + echo "❌ Cache tarball missing. Expected /tmp/cache-pre-commit.tar.gz" + exit 1 + fi + if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + - name: "Restore .cache from the tar file" + run: tar -C ~ -xzf /tmp/cache-pre-commit.tar.gz + shell: bash + if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + - name: "Show restored files" + run: | + echo "Restored files" + du ~/ --max-depth=2 + echo + shell: bash + if: steps.restore-pre-commit-cache.outputs.stash-hit == 'true' + - name: Install pre-commit hooks + shell: bash + run: pre-commit install-hooks || (cat ~/.cache/pre-commit/pre-commit.log && exit 1) + working-directory: ${{ github.workspace }} diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index cbe68805ad9d6..42acd83418685 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -402,6 +402,7 @@ jobs: env: UV_VERSION: ${{inputs.uv-version}} AIRFLOW_HOME: ~/airflow + FORCE_COLOR: 1 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 @@ -416,12 +417,11 @@ jobs: - name: "Install Airflow from current repo (simulating user installation)" run: | uv venv - source .venv/bin/activate + set -x uv pip install -e ./airflow-core - name: "Test airflow standalone command" run: | - source .venv/bin/activate - airflow standalone 2>&1 | tee airflow_startup.log & + uv run --no-sync airflow standalone 2>&1 | tee airflow_startup.log & AIRFLOW_PID=$! # Wait for ready message till timeout (10 minutes) @@ -443,8 +443,5 @@ jobs: done echo "FAILED: Airflow standalone did not become ready in time" - echo "=== AIRFLOW STARTUP LOGS ===" - cat airflow_startup.log - echo "=== END AIRFLOW STARTUP LOGS ===" kill $AIRFLOW_PID 2>/dev/null || true exit 1