From 78af575e17de2bff67922637df982f26dbf5f79a Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 20:35:35 -0800 Subject: [PATCH 1/9] ci: revamp test workflows --- .github/workflows/pytest_fast.yml | 71 +++++++++++++++++++ .../{python_pytest.yml => pytest_matrix.yml} | 67 +++-------------- ...-tests.yml => test-connectors-command.yml} | 30 +++----- 3 files changed, 89 insertions(+), 79 deletions(-) create mode 100644 .github/workflows/pytest_fast.yml rename .github/workflows/{python_pytest.yml => pytest_matrix.yml} (60%) rename .github/workflows/{connector-tests.yml => test-connectors-command.yml} (81%) diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml new file mode 100644 index 000000000..55d14bea1 --- /dev/null +++ b/.github/workflows/pytest_fast.yml @@ -0,0 +1,71 @@ +name: PyTest (Fast) + +on: + push: + branches: + - main + paths: + - 'airbyte_cdk/**' + - 'poetry.lock' + - 'pyproject.toml' + pull_request: + paths: + - 'airbyte_cdk/**' + - 'poetry.lock' + - 'pyproject.toml' + +jobs: + pytest-fast: + name: Pytest (Fast) + runs-on: ubuntu-latest + steps: + # Common steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v9 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.10" + cache: "poetry" + - name: Install dependencies + run: poetry install --all-extras + + - name: Run Pytest with Coverage (Fast Tests Only) + timeout-minutes: 60 + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + run: > + poetry run coverage run -m pytest + --durations=5 --exitfirst + -m "not slow and not requires_creds and not linting and not flaky" + + - name: Run Pytest with Coverage (Flaky Tests Only) + timeout-minutes: 60 + continue-on-error: true + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + run: > + poetry run coverage run -m pytest + --durations=5 --exitfirst + -m "flaky and not slow and not requires_creds" + + - name: Print Coverage Report + if: always() + run: poetry run coverage report + + - name: Create Coverage Artifacts + if: always() + run: | + poetry run coverage html -d htmlcov + poetry run coverage xml -o htmlcov/coverage.xml + + - name: Upload coverage to GitHub Artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: fasttest-coverage + path: htmlcov/ diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/pytest_matrix.yml similarity index 60% rename from .github/workflows/python_pytest.yml rename to .github/workflows/pytest_matrix.yml index ce5edbdcf..8417a08e6 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/pytest_matrix.yml @@ -5,70 +5,23 @@ # 2. `pytest`: Run all tests, across multiple Python versions. # # Note that `pytest-fast` also skips tests that require credentials, allowing it to run on forks. -name: PyTest +name: PyTest Matrix on: push: branches: - main - pull_request: {} + paths: + - 'airbyte_cdk/**' + - 'poetry.lock' + - 'pyproject.toml' + pull_request: + paths: + - 'airbyte_cdk/**' + - 'poetry.lock' + - 'pyproject.toml' jobs: - pytest-fast: - name: Pytest (Fast) - runs-on: ubuntu-latest - steps: - # Common steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Set up Poetry - uses: Gr1N/setup-poetry@v9 - with: - poetry-version: "1.7.1" - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: "3.10" - cache: "poetry" - - name: Install dependencies - run: poetry install --all-extras - - - name: Run Pytest with Coverage (Fast Tests Only) - timeout-minutes: 60 - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - run: > - poetry run coverage run -m pytest - --durations=5 --exitfirst - -m "not slow and not requires_creds and not linting and not flaky" - - - name: Run Pytest with Coverage (Flaky Tests Only) - timeout-minutes: 60 - continue-on-error: true - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - run: > - poetry run coverage run -m pytest - --durations=5 --exitfirst - -m "flaky and not slow and not requires_creds" - - - name: Print Coverage Report - if: always() - run: poetry run coverage report - - - name: Create Coverage Artifacts - if: always() - run: | - poetry run coverage html -d htmlcov - poetry run coverage xml -o htmlcov/coverage.xml - - - name: Upload coverage to GitHub Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: fasttest-coverage - path: htmlcov/ - pytest: name: Pytest (All, Python ${{ matrix.python-version }}, ${{ matrix.os }}) # Don't run on forks. Run on pushes to main, and on PRs that are not from forks. diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/test-connectors-command.yml similarity index 81% rename from .github/workflows/connector-tests.yml rename to .github/workflows/test-connectors-command.yml index d0bc02e39..97ffee239 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/test-connectors-command.yml @@ -1,4 +1,10 @@ -name: Connectors Tests +name: Test Connectors +on: + workflow_dispatch: + workflow_run: + workflows: [PyTest (Fast)] + types: + - completed concurrency: # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. @@ -11,15 +17,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true -on: - workflow_dispatch: - pull_request: - types: - - opened - - synchronize jobs: cdk_changes: name: Get Changes + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest permissions: statuses: write @@ -51,21 +52,6 @@ jobs: vector-db-based: ${{ steps.changes.outputs.vector-db-based }} sql: ${{ steps.changes.outputs.sql }} - # # The Connector CI Tests is a status check emitted by airbyte-ci - # # We make it pass once we have determined that there are no changes to the connectors - # - name: "Skip Connectors CI tests" - # if: steps.changes.outputs.src != 'true' && github.event_name == 'pull_request' - # run: | - # curl --request POST \ - # --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \ - # --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ - # --header 'content-type: application/json' \ - # --data '{ - # "state": "success", - # "context": "CDK Changes - Connectors Tests", - # "target_url": "${{ github.event.workflow_run.html_url }}" - # }' \ - connectors_ci: needs: cdk_changes # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR From 7bb07fc3116a98e3d997687e8be9f5e814e1b8f5 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 20:36:54 -0800 Subject: [PATCH 2/9] rename workflow --- .../{test-connectors-command.yml => test-connectors.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{test-connectors-command.yml => test-connectors.yml} (100%) diff --git a/.github/workflows/test-connectors-command.yml b/.github/workflows/test-connectors.yml similarity index 100% rename from .github/workflows/test-connectors-command.yml rename to .github/workflows/test-connectors.yml From c1462705cf94dc910aea061a540c84eaf411f27e Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 20:37:53 -0800 Subject: [PATCH 3/9] dummy change --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a0eb63253..104d5c32b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,6 +2,7 @@ requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"] build-backend = "poetry_dynamic_versioning.backend" + [tool.poetry] name = "airbyte-cdk" description = "A framework for writing Airbyte Connectors." From ae4c90a6c8402d92183423c6a9a775c41a375150 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 20:57:42 -0800 Subject: [PATCH 4/9] try without spec. characters in name --- .github/workflows/pytest_fast.yml | 2 +- .github/workflows/test-connectors.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml index 55d14bea1..91d8e2005 100644 --- a/.github/workflows/pytest_fast.yml +++ b/.github/workflows/pytest_fast.yml @@ -1,4 +1,4 @@ -name: PyTest (Fast) +name: PyTest Fast on: push: diff --git a/.github/workflows/test-connectors.yml b/.github/workflows/test-connectors.yml index 97ffee239..2dfe7d775 100644 --- a/.github/workflows/test-connectors.yml +++ b/.github/workflows/test-connectors.yml @@ -2,7 +2,7 @@ name: Test Connectors on: workflow_dispatch: workflow_run: - workflows: [PyTest (Fast)] + workflows: [PyTest Fast] types: - completed From 05ac1a478cd89b506414fe6cfff5bb58e56e2deb Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 21:07:49 -0800 Subject: [PATCH 5/9] update test workflows --- .github/workflows/pytest_fast.yml | 18 +++--------------- .github/workflows/test-connectors.yml | 9 ++++++--- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml index 91d8e2005..e577d55dc 100644 --- a/.github/workflows/pytest_fast.yml +++ b/.github/workflows/pytest_fast.yml @@ -16,7 +16,7 @@ on: jobs: pytest-fast: - name: Pytest (Fast) + name: Test runs-on: ubuntu-latest steps: # Common steps: @@ -35,23 +35,11 @@ jobs: run: poetry install --all-extras - name: Run Pytest with Coverage (Fast Tests Only) - timeout-minutes: 60 - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} + timeout-minutes: 20 run: > poetry run coverage run -m pytest --durations=5 --exitfirst - -m "not slow and not requires_creds and not linting and not flaky" - - - name: Run Pytest with Coverage (Flaky Tests Only) - timeout-minutes: 60 - continue-on-error: true - env: - GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - run: > - poetry run coverage run -m pytest - --durations=5 --exitfirst - -m "flaky and not slow and not requires_creds" + -m "not slow" - name: Print Coverage Report if: always() diff --git a/.github/workflows/test-connectors.yml b/.github/workflows/test-connectors.yml index 2dfe7d775..0ccc809df 100644 --- a/.github/workflows/test-connectors.yml +++ b/.github/workflows/test-connectors.yml @@ -75,9 +75,6 @@ jobs: cdk_extra: vector-db-based - connector: destination-motherduck cdk_extra: sql - if: > - ( github.event_name == 'pull_request' && needs.cdk_changes.outputs.src == 'true' && github.event.pull_request.head.repo.fork != true - ) || github.event_name == 'workflow_dispatch' name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" steps: - name: Abort if extra not changed (${{matrix.cdk_extra}}) @@ -89,12 +86,18 @@ jobs: exit 1 continue-on-error: true # Get the monorepo so we can test the connectors + - name: Checkout CDK + if: steps.no_changes.outcome != 'failure' + uses: actions/checkout@v4 + with: + path: airbyte-python-cdk - name: Checkout Airbyte Monorepo uses: actions/checkout@v4 if: steps.no_changes.outcome != 'failure' with: repository: airbytehq/airbyte ref: master + path: airbyte - name: Test Connector if: steps.no_changes.outcome != 'failure' timeout-minutes: 90 From 7198065d0e639146944a8125882877b343961b72 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 21:08:51 -0800 Subject: [PATCH 6/9] revert name change --- .github/workflows/{test-connectors.yml => connector-tests.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{test-connectors.yml => connector-tests.yml} (100%) diff --git a/.github/workflows/test-connectors.yml b/.github/workflows/connector-tests.yml similarity index 100% rename from .github/workflows/test-connectors.yml rename to .github/workflows/connector-tests.yml From 728e8694e0571d3ba7260965da340e688fb4f980 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 21:11:23 -0800 Subject: [PATCH 7/9] update if condition --- .github/workflows/connector-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 0ccc809df..ec8e95f73 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -20,7 +20,7 @@ concurrency: jobs: cdk_changes: name: Get Changes - if: ${{ github.event.workflow_run.conclusion == 'success' }} + # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest permissions: statuses: write From 4802920b38f5d2148cc45925babe545ec6dd0652 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 21:18:50 -0800 Subject: [PATCH 8/9] add 'cd airbyte' --- .github/workflows/connector-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index ec8e95f73..79a98459d 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -104,6 +104,7 @@ jobs: env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} run: | + cd airbyte make tools.airbyte-ci-binary.install airbyte-ci connectors \ --name ${{matrix.connector}} \ From 6cfaa08ea4854696cddef2aff3b3949415b83252 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Mon, 11 Nov 2024 23:29:23 -0800 Subject: [PATCH 9/9] update tests --- .github/workflows/connector-tests.yml | 13 +++++++++---- .github/workflows/pytest_fast.yml | 4 ++-- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.github/workflows/connector-tests.yml b/.github/workflows/connector-tests.yml index 79a98459d..570523323 100644 --- a/.github/workflows/connector-tests.yml +++ b/.github/workflows/connector-tests.yml @@ -1,10 +1,15 @@ name: Test Connectors on: workflow_dispatch: - workflow_run: - workflows: [PyTest Fast] + pull_request: types: - - completed + - opened + - synchronize + # TODO: Consider moving to run these only after the "PyTest (Fast)" workflow is successful. + # workflow_run: + # workflows: [PyTest (Fast)] + # types: + # - completed concurrency: # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. @@ -20,7 +25,6 @@ concurrency: jobs: cdk_changes: name: Get Changes - # if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest permissions: statuses: write @@ -75,6 +79,7 @@ jobs: cdk_extra: vector-db-based - connector: destination-motherduck cdk_extra: sql + name: "Check: '${{matrix.connector}}' (skip=${{needs.cdk_changes.outputs[matrix.cdk_extra] == 'false'}})" steps: - name: Abort if extra not changed (${{matrix.cdk_extra}}) diff --git a/.github/workflows/pytest_fast.yml b/.github/workflows/pytest_fast.yml index e577d55dc..c25116440 100644 --- a/.github/workflows/pytest_fast.yml +++ b/.github/workflows/pytest_fast.yml @@ -1,4 +1,4 @@ -name: PyTest Fast +name: Pytest (Fast) on: push: @@ -16,7 +16,7 @@ on: jobs: pytest-fast: - name: Test + name: Pytest (Fast) runs-on: ubuntu-latest steps: # Common steps: