diff --git a/airflow-core/src/airflow/config_templates/unit_tests.cfg b/airflow-core/src/airflow/config_templates/unit_tests.cfg index acd2aa771332b..c4493bc828b90 100644 --- a/airflow-core/src/airflow/config_templates/unit_tests.cfg +++ b/airflow-core/src/airflow/config_templates/unit_tests.cfg @@ -126,6 +126,10 @@ ssl_assert_hostname = False ssl_show_warn = False ca_certs = +[elasticsearch_configs] +verify_certs = True +http_compress = False + [example_section] # This section is used to test coercions of configuration values retrieval diff --git a/providers/elasticsearch/pyproject.toml b/providers/elasticsearch/pyproject.toml index f69df62f69a87..713d306a4a697 100644 --- a/providers/elasticsearch/pyproject.toml +++ b/providers/elasticsearch/pyproject.toml @@ -59,7 +59,7 @@ requires-python = ">=3.10" # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` dependencies = [ "apache-airflow>=2.11.0", - "apache-airflow-providers-common-compat>=1.11.0", + "apache-airflow-providers-common-compat>=1.11.0", # use next version "apache-airflow-providers-common-sql>=1.27.0", "elasticsearch>=8.10,<9", ] diff --git a/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py b/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py index d01bfa619e777..e35296c08285a 100644 --- a/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py +++ b/providers/elasticsearch/src/airflow/providers/elasticsearch/log/es_task_handler.py @@ -38,10 +38,9 @@ from elasticsearch import helpers from elasticsearch.exceptions import NotFoundError -from airflow.configuration import conf from airflow.models.dagrun import DagRun from airflow.providers.common.compat.module_loading import import_string -from airflow.providers.common.compat.sdk import AirflowException, timezone +from airflow.providers.common.compat.sdk import AirflowException, conf, timezone from airflow.providers.elasticsearch.log.es_json_formatter import ElasticsearchJSONFormatter from airflow.providers.elasticsearch.log.es_response import ElasticSearchResponse, Hit from airflow.providers.elasticsearch.version_compat import AIRFLOW_V_3_0_PLUS @@ -158,7 +157,7 @@ def __init__( offset_field: str = "offset", host: str = "http://localhost:9200", frontend: str = "localhost:5601", - index_patterns: str = conf.get("elasticsearch", "index_patterns"), + index_patterns: str = conf.get("elasticsearch", "index_patterns", fallback="_all"), index_patterns_callable: str = conf.get("elasticsearch", "index_patterns_callable", fallback=""), es_kwargs: dict | None | Literal["default_es_kwargs"] = "default_es_kwargs", max_bytes: int = 0, diff --git a/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py b/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py index ee47d8b6d4798..d3b27b19874f3 100644 --- a/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py +++ b/providers/elasticsearch/tests/unit/elasticsearch/log/test_es_task_handler.py @@ -32,7 +32,7 @@ import pendulum import pytest -from airflow.configuration import conf +from airflow.providers.common.compat.sdk import conf from airflow.providers.elasticsearch.log.es_response import ElasticSearchResponse from airflow.providers.elasticsearch.log.es_task_handler import ( VALID_ES_CONFIG_KEYS, diff --git a/providers/microsoft/azure/pyproject.toml b/providers/microsoft/azure/pyproject.toml index fe581ec7e0d35..d48156199f91e 100644 --- a/providers/microsoft/azure/pyproject.toml +++ b/providers/microsoft/azure/pyproject.toml @@ -59,7 +59,7 @@ requires-python = ">=3.10" # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` dependencies = [ "apache-airflow>=2.11.0", - "apache-airflow-providers-common-compat>=1.11.0", + "apache-airflow-providers-common-compat>=1.11.0", #use next version "adlfs>=2023.10.0", "azure-batch>=8.0.0", "azure-cosmos>=4.6.0", diff --git a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/log/wasb_task_handler.py b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/log/wasb_task_handler.py index a51625eb0396b..c32d5410b371d 100644 --- a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/log/wasb_task_handler.py +++ b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/log/wasb_task_handler.py @@ -26,7 +26,7 @@ import attrs from azure.core.exceptions import HttpResponseError -from airflow.configuration import conf +from airflow.providers.common.compat.sdk import conf from airflow.providers.microsoft.azure.version_compat import AIRFLOW_V_3_0_PLUS from airflow.utils.log.file_task_handler import FileTaskHandler from airflow.utils.log.logging_mixin import LoggingMixin diff --git a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/adx.py b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/adx.py index e12544e9db611..88c1b9688ec71 100644 --- a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/adx.py +++ b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/adx.py @@ -23,8 +23,7 @@ from functools import cached_property from typing import TYPE_CHECKING -from airflow.configuration import conf -from airflow.providers.common.compat.sdk import BaseOperator +from airflow.providers.common.compat.sdk import BaseOperator, conf from airflow.providers.microsoft.azure.hooks.adx import AzureDataExplorerHook if TYPE_CHECKING: diff --git a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/data_factory.py b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/data_factory.py index 1c58d198dc7b3..dba2bb126caaa 100644 --- a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/data_factory.py +++ b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/operators/data_factory.py @@ -22,13 +22,13 @@ from functools import cached_property from typing import TYPE_CHECKING, Any -from airflow.configuration import conf from airflow.providers.common.compat.sdk import ( AirflowException, BaseHook, BaseOperator, BaseOperatorLink, XCom, + conf, ) from airflow.providers.microsoft.azure.hooks.data_factory import ( AzureDataFactoryHook, diff --git a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/sensors/data_factory.py b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/sensors/data_factory.py index b5a8f62a50d11..5ff56cb2ce4c5 100644 --- a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/sensors/data_factory.py +++ b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/sensors/data_factory.py @@ -21,8 +21,7 @@ from functools import cached_property from typing import TYPE_CHECKING -from airflow.configuration import conf -from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator +from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator, conf from airflow.providers.microsoft.azure.hooks.data_factory import ( AzureDataFactoryHook, AzureDataFactoryPipelineRunException, diff --git a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/sensors/wasb.py b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/sensors/wasb.py index 2d62342c5c2f7..25129101d6c3c 100644 --- a/providers/microsoft/azure/src/airflow/providers/microsoft/azure/sensors/wasb.py +++ b/providers/microsoft/azure/src/airflow/providers/microsoft/azure/sensors/wasb.py @@ -21,8 +21,7 @@ from datetime import timedelta from typing import TYPE_CHECKING -from airflow.configuration import conf -from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator +from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator, conf from airflow.providers.microsoft.azure.hooks.wasb import WasbHook from airflow.providers.microsoft.azure.triggers.wasb import WasbBlobSensorTrigger, WasbPrefixSensorTrigger diff --git a/providers/microsoft/mssql/tests/unit/microsoft/mssql/hooks/test_mssql.py b/providers/microsoft/mssql/tests/unit/microsoft/mssql/hooks/test_mssql.py index 2af157160835b..24972cd60db07 100644 --- a/providers/microsoft/mssql/tests/unit/microsoft/mssql/hooks/test_mssql.py +++ b/providers/microsoft/mssql/tests/unit/microsoft/mssql/hooks/test_mssql.py @@ -23,8 +23,8 @@ import pytest import sqlalchemy -from airflow.configuration import conf from airflow.models import Connection +from airflow.providers.common.compat.sdk import conf from airflow.providers.microsoft.mssql.dialects.mssql import MsSqlDialect from tests_common.test_utils.file_loading import load_file_from_resources diff --git a/providers/microsoft/winrm/pyproject.toml b/providers/microsoft/winrm/pyproject.toml index c8ef1ce0f2e10..2a27a29fcdd41 100644 --- a/providers/microsoft/winrm/pyproject.toml +++ b/providers/microsoft/winrm/pyproject.toml @@ -59,7 +59,7 @@ requires-python = ">=3.10" # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` dependencies = [ "apache-airflow>=2.11.0", - "apache-airflow-providers-common-compat>=1.10.1", + "apache-airflow-providers-common-compat>=1.10.1", # use next version "pywinrm>=0.5.0", ] diff --git a/providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/operators/winrm.py b/providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/operators/winrm.py index 9eb40d6284671..a360df4e1febc 100644 --- a/providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/operators/winrm.py +++ b/providers/microsoft/winrm/src/airflow/providers/microsoft/winrm/operators/winrm.py @@ -22,8 +22,7 @@ from collections.abc import Sequence from typing import TYPE_CHECKING -from airflow.configuration import conf -from airflow.providers.common.compat.sdk import AirflowException, BaseOperator +from airflow.providers.common.compat.sdk import AirflowException, BaseOperator, conf from airflow.providers.microsoft.winrm.hooks.winrm import WinRMHook if TYPE_CHECKING: diff --git a/providers/opensearch/pyproject.toml b/providers/opensearch/pyproject.toml index 067e3305c9598..704557e688a13 100644 --- a/providers/opensearch/pyproject.toml +++ b/providers/opensearch/pyproject.toml @@ -59,7 +59,7 @@ requires-python = ">=3.10" # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` dependencies = [ "apache-airflow>=2.11.0", - "apache-airflow-providers-common-compat>=1.11.0", + "apache-airflow-providers-common-compat>=1.11.0", # use next version "opensearch-py>=2.2.0", ] diff --git a/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py b/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py index 61a5db72f834a..fd13a1a8b4c42 100644 --- a/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py +++ b/providers/opensearch/src/airflow/providers/opensearch/log/os_task_handler.py @@ -32,10 +32,9 @@ from opensearchpy import OpenSearch from opensearchpy.exceptions import NotFoundError -from airflow.configuration import conf from airflow.models import DagRun from airflow.providers.common.compat.module_loading import import_string -from airflow.providers.common.compat.sdk import AirflowException +from airflow.providers.common.compat.sdk import AirflowException, conf from airflow.providers.opensearch.log.os_json_formatter import OpensearchJSONFormatter from airflow.providers.opensearch.log.os_response import Hit, OpensearchResponse from airflow.providers.opensearch.version_compat import AIRFLOW_V_3_0_PLUS @@ -105,7 +104,6 @@ def _ensure_ti(ti: TaskInstanceKey | TaskInstance, session) -> TaskInstance: def get_os_kwargs_from_config() -> dict[str, Any]: open_search_config = conf.getsection("opensearch_configs") kwargs_dict = {key: value for key, value in open_search_config.items()} if open_search_config else {} - return kwargs_dict diff --git a/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py b/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py index c976182f2c6e9..ab86d4ce5c5b8 100644 --- a/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py +++ b/providers/opensearch/tests/unit/opensearch/log/test_os_task_handler.py @@ -30,7 +30,7 @@ import pytest from opensearchpy.exceptions import NotFoundError -from airflow.configuration import conf +from airflow.providers.common.compat.sdk import conf from airflow.providers.opensearch.log.os_response import OpensearchResponse from airflow.providers.opensearch.log.os_task_handler import ( OpensearchTaskHandler, diff --git a/providers/standard/pyproject.toml b/providers/standard/pyproject.toml index 912a7bea45852..eca439dcce4da 100644 --- a/providers/standard/pyproject.toml +++ b/providers/standard/pyproject.toml @@ -59,7 +59,7 @@ requires-python = ">=3.10" # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` dependencies = [ "apache-airflow>=2.11.0", - "apache-airflow-providers-common-compat>=1.10.1", + "apache-airflow-providers-common-compat>=1.10.1", # use next version ] # The optional dependencies should be modified in place in the generated file diff --git a/providers/standard/src/airflow/providers/standard/operators/hitl.py b/providers/standard/src/airflow/providers/standard/operators/hitl.py index de208174827af..37fcadfe014ae 100644 --- a/providers/standard/src/airflow/providers/standard/operators/hitl.py +++ b/providers/standard/src/airflow/providers/standard/operators/hitl.py @@ -28,7 +28,7 @@ from typing import TYPE_CHECKING, Any from urllib.parse import ParseResult, urlencode, urlparse, urlunparse -from airflow.configuration import conf +from airflow.providers.common.compat.sdk import conf from airflow.providers.standard.exceptions import HITLRejectException, HITLTimeoutError, HITLTriggerEventError from airflow.providers.standard.operators.branch import BranchMixIn from airflow.providers.standard.triggers.hitl import HITLTrigger, HITLTriggerEventSuccessPayload diff --git a/providers/standard/src/airflow/providers/standard/operators/trigger_dagrun.py b/providers/standard/src/airflow/providers/standard/operators/trigger_dagrun.py index 3060c2dafe0d3..4ed76b057af5e 100644 --- a/providers/standard/src/airflow/providers/standard/operators/trigger_dagrun.py +++ b/providers/standard/src/airflow/providers/standard/operators/trigger_dagrun.py @@ -28,7 +28,6 @@ from sqlalchemy.orm.exc import NoResultFound from airflow.api.common.trigger_dag import trigger_dag -from airflow.configuration import conf from airflow.exceptions import DagNotFound, DagRunAlreadyExists from airflow.models.dag import DagModel from airflow.models.dagrun import DagRun @@ -38,6 +37,7 @@ AirflowSkipException, BaseOperatorLink, XCom, + conf, timezone, ) from airflow.providers.standard.triggers.external_task import DagStateTrigger diff --git a/providers/standard/src/airflow/providers/standard/sensors/external_task.py b/providers/standard/src/airflow/providers/standard/sensors/external_task.py index 5c44503372c5b..18e6578830661 100644 --- a/providers/standard/src/airflow/providers/standard/sensors/external_task.py +++ b/providers/standard/src/airflow/providers/standard/sensors/external_task.py @@ -23,9 +23,13 @@ from collections.abc import Callable, Collection, Iterable, Sequence from typing import TYPE_CHECKING, ClassVar -from airflow.configuration import conf from airflow.models.dag import DagModel -from airflow.providers.common.compat.sdk import AirflowSkipException, BaseOperatorLink, BaseSensorOperator +from airflow.providers.common.compat.sdk import ( + AirflowSkipException, + BaseOperatorLink, + BaseSensorOperator, + conf, +) from airflow.providers.standard.exceptions import ( DuplicateStateError, ExternalDagDeletedError, diff --git a/providers/standard/src/airflow/providers/standard/sensors/filesystem.py b/providers/standard/src/airflow/providers/standard/sensors/filesystem.py index a83a25776cfa9..50606c381ae89 100644 --- a/providers/standard/src/airflow/providers/standard/sensors/filesystem.py +++ b/providers/standard/src/airflow/providers/standard/sensors/filesystem.py @@ -25,8 +25,7 @@ from glob import glob from typing import TYPE_CHECKING, Any -from airflow.configuration import conf -from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator +from airflow.providers.common.compat.sdk import AirflowException, BaseSensorOperator, conf from airflow.providers.standard.hooks.filesystem import FSHook from airflow.providers.standard.triggers.file import FileTrigger diff --git a/providers/standard/src/airflow/providers/standard/sensors/time.py b/providers/standard/src/airflow/providers/standard/sensors/time.py index 03b0dd2333fef..940a72c5eb953 100644 --- a/providers/standard/src/airflow/providers/standard/sensors/time.py +++ b/providers/standard/src/airflow/providers/standard/sensors/time.py @@ -22,9 +22,8 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any -from airflow.configuration import conf from airflow.exceptions import AirflowProviderDeprecationWarning -from airflow.providers.common.compat.sdk import BaseSensorOperator, timezone +from airflow.providers.common.compat.sdk import BaseSensorOperator, conf, timezone from airflow.providers.standard.triggers.temporal import DateTimeTrigger try: diff --git a/providers/standard/src/airflow/providers/standard/sensors/time_delta.py b/providers/standard/src/airflow/providers/standard/sensors/time_delta.py index a1c1d21e7b884..e139499fec6e0 100644 --- a/providers/standard/src/airflow/providers/standard/sensors/time_delta.py +++ b/providers/standard/src/airflow/providers/standard/sensors/time_delta.py @@ -25,9 +25,8 @@ from deprecated.classic import deprecated from packaging.version import Version -from airflow.configuration import conf from airflow.exceptions import AirflowProviderDeprecationWarning -from airflow.providers.common.compat.sdk import AirflowSkipException, BaseSensorOperator, timezone +from airflow.providers.common.compat.sdk import AirflowSkipException, BaseSensorOperator, conf, timezone from airflow.providers.standard.triggers.temporal import DateTimeTrigger, TimeDeltaTrigger from airflow.providers.standard.version_compat import AIRFLOW_V_3_0_PLUS @@ -194,9 +193,11 @@ def __init__( def execute(self, context: Context) -> None: if self.deferrable: self.defer( - trigger=TimeDeltaTrigger(self.time_to_wait, end_from_trigger=True) - if AIRFLOW_V_3_0_PLUS - else TimeDeltaTrigger(self.time_to_wait), + trigger=( + TimeDeltaTrigger(self.time_to_wait, end_from_trigger=True) + if AIRFLOW_V_3_0_PLUS + else TimeDeltaTrigger(self.time_to_wait) + ), method_name="execute_complete", ) else: diff --git a/providers/standard/src/airflow/providers/standard/utils/python_virtualenv.py b/providers/standard/src/airflow/providers/standard/utils/python_virtualenv.py index 33b4a0fb7e0fc..fd0e144725935 100644 --- a/providers/standard/src/airflow/providers/standard/utils/python_virtualenv.py +++ b/providers/standard/src/airflow/providers/standard/utils/python_virtualenv.py @@ -30,7 +30,7 @@ import jinja2 from jinja2 import select_autoescape -from airflow.configuration import conf +from airflow.providers.common.compat.sdk import conf def _is_uv_installed() -> bool: diff --git a/providers/standard/tests/unit/standard/operators/test_trigger_dagrun.py b/providers/standard/tests/unit/standard/operators/test_trigger_dagrun.py index be35c058f2f31..0f8d171658138 100644 --- a/providers/standard/tests/unit/standard/operators/test_trigger_dagrun.py +++ b/providers/standard/tests/unit/standard/operators/test_trigger_dagrun.py @@ -25,13 +25,12 @@ import time_machine from sqlalchemy import delete, select, update -from airflow.configuration import conf from airflow.exceptions import DagRunAlreadyExists from airflow.models.dag import DagModel from airflow.models.dagrun import DagRun from airflow.models.log import Log from airflow.models.taskinstance import TaskInstance -from airflow.providers.common.compat.sdk import AirflowException, TaskDeferred +from airflow.providers.common.compat.sdk import AirflowException, TaskDeferred, conf from airflow.providers.standard.operators.trigger_dagrun import TriggerDagRunOperator from airflow.providers.standard.triggers.external_task import DagStateTrigger from airflow.utils.session import create_session