diff --git a/Dockerfile.ci b/Dockerfile.ci index 96468ceb36cc6..0761021304197 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1393,11 +1393,10 @@ ENV AIRFLOW_REPO=${AIRFLOW_REPO}\ RUN echo "Airflow version: ${AIRFLOW_VERSION}" # Those are additional constraints that are needed for some extras but we do not want to -# force them on the main Airflow package. -# Recent aiobotocore (2.6.0+) requires this specific botocore version, while -# boto3 1.28.17 is the last version supporting that, so we pin them to help pip -# avoid extra work. Revisit this when a new aiobotocore version is released. -ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="botocore==1.31.17 boto3<=1.28.17" +# force them on the main Airflow package. Currently we need no extra limits as PIP 23.1+ has much better +# dependency resolution and we do not need to limit the versions of the dependencies +# aiobotocore is limited temporarily until it stops backtracking pip +ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="" ARG UPGRADE_TO_NEWER_DEPENDENCIES="false" ARG VERSION_SUFFIX_FOR_PYPI="" diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index 6aa32825105bf..2959764e00237 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -67,7 +67,19 @@ dependencies: - apache-airflow>=2.4.0 - apache-airflow-providers-common-sql>=1.3.1 - apache-airflow-providers-http - - boto3>=1.24.0 + # We should update minimum version of boto3 and here regularly to avoid `pip` backtracking with the number + # of candidates to consider. We should also make sure that all the below related packages have also the + # same minimum version specified. Boto3 1.28.0 has been released on July 6 2023. We should also make sure we + # set it to the version that `aiobotocore` supports (see `aiobotocore` optional dependency at the end + # of this file). Currently we set aiobotocore as minimum 2.5.3 - as this is was the first version + # that supported boto3 1.28. NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS + - boto3>=1.28.0 + - mypy-boto3-rds>=1.28.0 + - mypy-boto3-redshift-data>=1.28.0 + - mypy-boto3-s3>=1.28.0 + - mypy-boto3-appflow>=1.28.0 + # NOTE!!! BOTOCORE VERSIONS ARE SHIFTED BY 3 MINOR VERSIONS + - botocore>=1.31.0 - asgiref # watchtower 3 has been released end Jan and introduced breaking change across the board that might # change logging behaviour: @@ -77,11 +89,7 @@ dependencies: - jsonpath_ng>=1.5.3 - redshift_connector>=2.0.888 - sqlalchemy_redshift>=0.8.6 - - mypy-boto3-rds>=1.24.0 - - mypy-boto3-redshift-data>=1.24.0 - - mypy-boto3-appflow>=1.28.16 - asgiref - - mypy-boto3-s3>=1.24.0 integrations: - integration-name: Amazon Athena @@ -699,7 +707,7 @@ additional-extras: # boto3 have native async support and we move away from aio aiobotocore - name: aiobotocore dependencies: - - aiobotocore[boto3]>=2.2.0 + - aiobotocore[boto3]>=2.5.3 - name: cncf.kubernetes dependencies: - apache-airflow-providers-cncf-kubernetes>=7.2.0 diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 0d5a7408ea355..23de3da647656 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -26,12 +26,13 @@ "apache-airflow>=2.4.0", "asgiref", "asgiref", - "boto3>=1.24.0", + "boto3>=1.28.0", + "botocore>=1.31.0", "jsonpath_ng>=1.5.3", - "mypy-boto3-appflow>=1.28.16", - "mypy-boto3-rds>=1.24.0", - "mypy-boto3-redshift-data>=1.24.0", - "mypy-boto3-s3>=1.24.0", + "mypy-boto3-appflow>=1.28.0", + "mypy-boto3-rds>=1.28.0", + "mypy-boto3-redshift-data>=1.28.0", + "mypy-boto3-s3>=1.28.0", "redshift_connector>=2.0.888", "sqlalchemy_redshift>=0.8.6", "watchtower~=2.0.1" diff --git a/scripts/in_container/_in_container_utils.sh b/scripts/in_container/_in_container_utils.sh index 16663f47389c0..288248b4eca49 100644 --- a/scripts/in_container/_in_container_utils.sh +++ b/scripts/in_container/_in_container_utils.sh @@ -297,6 +297,12 @@ function install_all_providers_from_pypi_with_eager_upgrade() { local res for provider_package in ${ALL_PROVIDERS_PACKAGES} do + # Until we release "yandex" provider with protobuf support we need to remove it from the list of providers + # to install, because it is impossible to find common requirements for already released yandex provider + # and current airflow + if [[ ${provider_package} == "apache-airflow-providers-yandex" ]]; then + continue + fi echo -n "Checking if ${provider_package} is available in PyPI: " res=$(curl --head -s -o /dev/null -w "%{http_code}" "https://pypi.org/project/${provider_package}/") if [[ ${res} == "200" ]]; then @@ -306,7 +312,11 @@ function install_all_providers_from_pypi_with_eager_upgrade() { echo "${COLOR_YELLOW}Skipped${COLOR_RESET}" fi done + + echo "Installing provider packages: ${packages_to_install[*]}" + + # we add eager requirements to make sure to take into account limitations that will allow us to # install all providers. We install only those packages that are available in PyPI - we might # Have some new providers in the works and they might not yet be simply available in PyPI