From 3661f4f1d3745114a5bb41b486174a995327602d Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 12 Sep 2023 08:32:42 +0200 Subject: [PATCH] Update min-sqlalchemy version to account for latest features used Some of the recent sqlalchemy changes are not working with minimum version of sqlalchemy of ours - for example `where` syntax does not allow moe than one clause and we are already passing more in _do_delete_old_records (added in #33527). This syntax however was added in SQL Alchemy 1.4.28 and our minimum version was 1.4.27. This change bumps the minimum SQLAlchemy version to 1.4.28 but it also adds a special test job that only runs on Postgres that downgrades the SQLAlchemy to the minimum supported version (retrieved from setup.cfg). This way, we will be able to detect such incompatible changes at the PR time. This is a new flag `--downgrade-sqlalchemy` on test command that works similar to earlier `--upgrade-boto`. We also enable the `--upgrade-boto` and `--downgrade-sqlalchemy` flags to be used for `breeze shell` command - thanks to that we can easily test both flags with `breeze shell` command. --- .github/workflows/ci.yml | 49 +++++++++++++++++++ Dockerfile.ci | 23 ++++++--- .../commands/developer_commands.py | 8 +++ .../commands/developer_commands_config.py | 7 +++ .../commands/testing_commands.py | 12 ++--- .../commands/testing_commands_config.py | 1 + .../src/airflow_breeze/params/shell_params.py | 1 + .../airflow_breeze/utils/common_options.py | 12 +++++ .../utils/docker_command_utils.py | 2 + images/breeze/output-commands-hash.txt | 6 +-- images/breeze/output_shell.svg | 24 ++++++--- images/breeze/output_testing_tests.svg | 26 +++++----- scripts/ci/docker-compose/_docker.env | 1 + scripts/ci/docker-compose/base.yml | 1 + scripts/ci/docker-compose/devcontainer.env | 1 + scripts/docker/entrypoint_ci.sh | 23 ++++++--- setup.cfg | 2 +- 17 files changed, 156 insertions(+), 43 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index df31ab2d38bde..7a2d6998f0f36 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1024,6 +1024,55 @@ jobs: uses: ./.github/actions/post_tests_failure if: failure() + tests-postgres-min-sqlalchemy: + timeout-minutes: 130 + name: > + MinSQLAlchemy${{needs.build-info.outputs.default-postgres-version}}, + Py${{needs.build-info.outputs.default-python-version}}: + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} + runs-on: "${{needs.build-info.outputs.runs-on}}" + needs: [build-info, wait-for-ci-images] + env: + RUNS_ON: "${{needs.build-info.outputs.runs-on}}" + PARALLEL_TEST_TYPES: "${{needs.build-info.outputs.parallel-test-types-list-as-string}}" + PR_LABELS: "${{needs.build-info.outputs.pull-request-labels}}" + FULL_TESTS_NEEDED: "${{needs.build-info.outputs.full-tests-needed}}" + DEBUG_RESOURCES: "${{needs.build-info.outputs.debug-resources}}" + BACKEND: "postgres" + PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" + PYTHON_VERSION: "${needs.build-info.outputs.default-python-version}}" + POSTGRES_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" + BACKEND_VERSION: "${{needs.build-info.outputs.default-postgres-version}}" + DOWNGRADE_SQLALCHEMY: "true" + JOB_ID: > + postgres-min-sqlalchemy-${{needs.build-info.outputs.default-python-version}}- + ${{needs.build-info.outputs.default-postgres-version}} + COVERAGE: "${{needs.build-info.outputs.run-coverage}}" + if: needs.build-info.outputs.run-tests == 'true' + steps: + - name: Cleanup repo + shell: bash + run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v3 + with: + persist-credentials: false + - name: > + Prepare breeze & CI image: ${{needs.build-info.outputs.default-python-version}}:${{env.IMAGE_TAG}} + uses: ./.github/actions/prepare_breeze_and_image + - name: > + Tests: ${{needs.build-info.outputs.default-python-version}}: + ${{needs.build-info.outputs.parallel-test-types-list-as-string}} + run: breeze testing tests --run-in-parallel + - name: > + Post Tests success: ${{needs.build-info.outputs.default-python-version}}:Boto" + uses: ./.github/actions/post_tests_success + if: success() + - name: > + Post Tests failure: ${{needs.build-info.outputs.default-python-version}}:Boto" + uses: ./.github/actions/post_tests_failure + if: failure() + tests-postgres-in-progress-features-disabled: timeout-minutes: 130 name: > diff --git a/Dockerfile.ci b/Dockerfile.ci index 48e626e570523..55fc05cc0e5ec 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -949,6 +949,22 @@ fi rm -f "${AIRFLOW_SOURCES}/pytest.ini" rm -f "${AIRFLOW_SOURCES}/.coveragerc" +if [[ ${UPGRADE_BOTO=} == "true" ]]; then + echo + echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" + echo + pip uninstall --root-user-action ignore aiobotocore -y || true + pip install --root-user-action ignore --upgrade boto3 botocore + pip check +fi +if [[ ${DOWNGRADE_SQLALCHEMY=} == "true" ]]; then + min_sqlalchemy_version=$(grep "sqlalchemy>=" setup.cfg | sed "s/.*>=\([0-9\.]*\).*/\1/") + echo + echo "${COLOR_BLUE}Downgrading sqlalchemy to minimum supported version: ${min_sqlalchemy_version}${COLOR_RESET}" + echo + pip install --root-user-action ignore "sqlalchemy==${min_sqlalchemy_version}" + pip check +fi set +u if [[ "${RUN_TESTS}" != "true" ]]; then @@ -1179,13 +1195,6 @@ else exit 1 fi fi -if [[ ${UPGRADE_BOTO=} == "true" ]]; then - echo - echo "${COLOR_BLUE}Upgrading boto3, botocore to latest version to run Amazon tests with them${COLOR_RESET}" - echo - pip uninstall aiobotocore -y || true - pip install --upgrade boto3 botocore -fi readonly SELECTED_TESTS CLI_TESTS API_TESTS PROVIDERS_TESTS CORE_TESTS WWW_TESTS \ ALL_TESTS ALL_PRESELECTED_TESTS diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands.py b/dev/breeze/src/airflow_breeze/commands/developer_commands.py index 21f285295187f..60ac45d4cf62d 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands.py @@ -51,6 +51,7 @@ option_celery_broker, option_celery_flower, option_db_reset, + option_downgrade_sqlalchemy, option_dry_run, option_executor, option_force_build, @@ -70,6 +71,7 @@ option_platform_single, option_postgres_version, option_python, + option_upgrade_boto, option_use_airflow_version, option_use_packages_from_dist, option_verbose, @@ -161,6 +163,8 @@ def run(self): @option_image_tag_for_running @option_max_time @option_include_mypy_volume +@option_upgrade_boto +@option_downgrade_sqlalchemy @option_verbose @option_dry_run @option_github_repository @@ -196,6 +200,8 @@ def shell( celery_broker: str, celery_flower: bool, extra_args: tuple, + upgrade_boto: bool, + downgrade_sqlalchemy: bool, ): """Enter breeze environment. this is the default command use when no other is selected.""" if get_verbose() or get_dry_run(): @@ -233,6 +239,8 @@ def shell( executor=executor, celery_broker=celery_broker, celery_flower=celery_flower, + upgrade_boto=upgrade_boto, + downgrade_sqlalchemy=downgrade_sqlalchemy, ) sys.exit(result.returncode) diff --git a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py index 81c99e793f954..9ef4f8ca96d63 100644 --- a/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/developer_commands_config.py @@ -99,6 +99,13 @@ "--package-format", ], }, + { + "name": "Upgrading/downgrading selected packages", + "options": [ + "--upgrade-boto", + "--downgrade-sqlalchemy", + ], + }, ], "breeze compile-www-assets": [ { diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands.py b/dev/breeze/src/airflow_breeze/commands/testing_commands.py index ee3b445cb3c7c..b5d09c944bc69 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands.py @@ -38,6 +38,7 @@ option_backend, option_db_reset, option_debug_resources, + option_downgrade_sqlalchemy, option_dry_run, option_github_repository, option_image_name, @@ -52,6 +53,7 @@ option_python, option_run_in_parallel, option_skip_cleanup, + option_upgrade_boto, option_use_airflow_version, option_verbose, ) @@ -359,12 +361,8 @@ def run_tests_in_parallel( show_default=True, envvar="PARALLEL_TEST_TYPES", ) -@click.option( - "--upgrade-boto", - help="Remove aiobotocore and upgrade botocore and boto to the latest version.", - is_flag=True, - envvar="UPGRADE_BOTO", -) +@option_upgrade_boto +@option_downgrade_sqlalchemy @click.option( "--collect-only", help="Collect tests only, do not run them.", @@ -408,6 +406,7 @@ def command_for_tests( mount_sources: str, extra_pytest_args: tuple, upgrade_boto: bool, + downgrade_sqlalchemy: bool, collect_only: bool, remove_arm_packages: bool, github_repository: str, @@ -428,6 +427,7 @@ def command_for_tests( forward_ports=False, test_type=test_type, upgrade_boto=upgrade_boto, + downgrade_sqlalchemy=downgrade_sqlalchemy, collect_only=collect_only, remove_arm_packages=remove_arm_packages, github_repository=github_repository, diff --git a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py index 0c91145b5eec4..b1913d9f058df 100644 --- a/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/testing_commands_config.py @@ -56,6 +56,7 @@ "--use-airflow-version", "--mount-sources", "--upgrade-boto", + "--downgrade-sqlalchemy", "--remove-arm-packages", "--skip-docker-compose-down", ], diff --git a/dev/breeze/src/airflow_breeze/params/shell_params.py b/dev/breeze/src/airflow_breeze/params/shell_params.py index 99bc8c4d3cfd5..b2240a42c991c 100644 --- a/dev/breeze/src/airflow_breeze/params/shell_params.py +++ b/dev/breeze/src/airflow_breeze/params/shell_params.py @@ -120,6 +120,7 @@ class ShellParams: dry_run: bool = False verbose: bool = False upgrade_boto: bool = False + downgrade_sqlalchemy: bool = False executor: str = START_AIRFLOW_DEFAULT_ALLOWED_EXECUTORS celery_broker: str = DEFAULT_CELERY_BROKER celery_flower: bool = False diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 95a0b46ebe0dd..f443587913527 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -623,3 +623,15 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, help="Local directory path of cloned airflow-site repo.", required=True, ) +option_upgrade_boto = click.option( + "--upgrade-boto", + help="Remove aiobotocore and upgrade botocore and boto to the latest version.", + is_flag=True, + envvar="UPGRADE_BOTO", +) +option_downgrade_sqlalchemy = click.option( + "--downgrade-sqlalchemy", + help="Downgrade SQLAlchemy to minimum supported version.", + is_flag=True, + envvar="DOWNGRADE_SQLALCHEMY", +) diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 82527e6710a6a..3dd9b4ffb639d 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -596,6 +596,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: set_value_to_default_if_not_set(env, "TEST_TYPE", "") set_value_to_default_if_not_set(env, "TEST_TIMEOUT", "60") set_value_to_default_if_not_set(env, "UPGRADE_BOTO", "false") + set_value_to_default_if_not_set(env, "DOWNGRADE_SQLALCHEMY", "false") set_value_to_default_if_not_set(env, "UPGRADE_TO_NEWER_DEPENDENCIES", "false") set_value_to_default_if_not_set(env, "USE_PACKAGES_FROM_DIST", "false") set_value_to_default_if_not_set(env, "VERBOSE", "false") @@ -643,6 +644,7 @@ def update_expected_environment_variables(env: dict[str, str]) -> None: "SQLITE_URL": "sqlite_url", "START_AIRFLOW": "start_airflow", "UPGRADE_BOTO": "upgrade_boto", + "DOWNGRADE_SQLALCHEMY": "downgrade_sqlalchemy", "USE_AIRFLOW_VERSION": "use_airflow_version", "USE_PACKAGES_FROM_DIST": "use_packages_from_dist", "VERSION_SUFFIX_FOR_PYPI": "version_suffix_for_pypi", diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 3d22259a4bd8c..9331ce8e379c5 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -62,11 +62,11 @@ setup:regenerate-command-images:d5e29ec6acb1a6af7d83772c2962f89d setup:self-upgrade:4af905a147fcd6670a0e33d3d369a94b setup:version:be116d90a21c2afe01087f7609774e1e setup:fd391bab5277ad3aca65987a84144d51 -shell:1462cde6f7e11a73cb42d4eec93c598e +shell:1d7a1d16c4c9da86a3d8dc19ee698278 start-airflow:d4815dea2cfc0af7038697c1d9a13996 static-checks:19926b8fcea5784b28d4a0d99865363c testing:docker-compose-tests:fd154a058082fcfda12eb877a9a89338 testing:helm-tests:0669be17b744ba057adbf38681bd8e68 testing:integration-tests:e745af9dd595adaa6f17ef02fbaae3b5 -testing:tests:29da9579379b1d013c02dd6c4471b958 -testing:355a33f1e38298dba301d638d956fb55 +testing:tests:f5b196d717b2099172595063a7a3d0aa +testing:a9bdfb94a0d08deee7aaf4037cc55a34 diff --git a/images/breeze/output_shell.svg b/images/breeze/output_shell.svg index e9414eda1e193..2fbccea4816db 100644 --- a/images/breeze/output_shell.svg +++ b/images/breeze/output_shell.svg @@ -1,4 +1,4 @@ - +