From 3e09bbf5e98026d0ff6e351deefaea40616204e7 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 28 Nov 2024 02:50:55 +0100 Subject: [PATCH 1/2] Get rid of database isolation option As part of removing AIP-44 we remove database isolation mode and remove pytest markers that skipped some tests. Part of #44436 --- .github/workflows/ci.yml | 2 +- Dockerfile.ci | 40 --- airflow/utils/session.py | 18 -- contributing-docs/testing/unit_tests.rst | 34 -- dev/breeze/doc/images/output-commands.svg | 148 +++++---- dev/breeze/doc/images/output_shell.svg | 292 +++++++++--------- dev/breeze/doc/images/output_shell.txt | 2 +- .../doc/images/output_start-airflow.svg | 206 ++++++------ .../doc/images/output_start-airflow.txt | 2 +- .../images/output_testing_providers-tests.svg | 118 ++++--- .../images/output_testing_providers-tests.txt | 2 +- .../airflow_breeze/commands/common_options.py | 6 - .../commands/developer_commands.py | 7 - .../commands/developer_commands_config.py | 3 - .../airflow_breeze/commands/main_command.py | 2 - .../commands/testing_commands.py | 6 - .../commands/testing_commands_config.py | 8 - .../src/airflow_breeze/params/shell_params.py | 2 - providers/tests/airbyte/hooks/test_airbyte.py | 2 +- .../aws/auth_manager/cli/test_avp_commands.py | 2 - .../test_aws_security_manager_override.py | 2 - .../aws/auth_manager/test_aws_auth_manager.py | 5 +- .../aws/auth_manager/views/test_auth.py | 5 +- .../aws/transfers/test_google_api_to_s3.py | 2 +- .../tests/apache/druid/hooks/test_druid.py | 3 - .../tests/apache/livy/hooks/test_livy.py | 2 - .../tests/arangodb/sensors/test_arangodb.py | 2 +- .../tests/asana/operators/test_asana_tasks.py | 2 +- .../celery/log_handlers/test_log_handlers.py | 2 +- .../executors/test_kubernetes_executor.py | 2 - .../log_handlers/test_log_handlers.py | 2 +- .../cncf/kubernetes/operators/test_pod.py | 2 +- .../kubernetes/test_template_rendering.py | 2 +- .../tests/common/io/xcom/test_backend.py | 121 ++++---- .../tests/common/sql/operators/test_sql.py | 5 +- .../tests/common/sql/sensors/test_sql.py | 5 +- .../edge/worker_api/routes/test_rpc_api.py | 2 +- .../api_endpoints/test_asset_endpoint.py | 1 - .../auth_manager/api_endpoints/test_auth.py | 1 - .../auth_manager/api_endpoints/test_cors.py | 1 - .../api_endpoints/test_dag_endpoint.py | 1 - .../api_endpoints/test_dag_run_endpoint.py | 1 - .../api_endpoints/test_dag_source_endpoint.py | 1 - .../test_dag_warning_endpoint.py | 1 - .../api_endpoints/test_event_log_endpoint.py | 1 - .../test_import_error_endpoint.py | 1 - .../test_task_instance_endpoint.py | 1 - .../api_endpoints/test_variable_endpoint.py | 1 - .../api_endpoints/test_xcom_endpoint.py | 1 - .../test_role_and_permission_schema.py | 2 +- .../common/auth_backend/test_google_openid.py | 1 - .../operators/test_campaign_manager.py | 1 - .../operators/test_display_video.py | 1 - .../openlineage/plugins/test_execution.py | 1 - .../tests/standard/operators/test_python.py | 12 - .../tests/standard/sensors/test_time_delta.py | 1 - .../tests/standard/sensors/test_weekday.py | 3 - .../standard/triggers/test_external_task.py | 1 - scripts/docker/entrypoint_ci.sh | 40 --- scripts/in_container/bin/run_tmux | 6 - task_sdk/tests/defintions/test_asset.py | 1 - tests/always/test_example_dags.py | 1 - .../endpoints/test_asset_endpoint.py | 2 +- .../endpoints/test_config_endpoint.py | 2 +- .../endpoints/test_connection_endpoint.py | 2 +- .../endpoints/test_dag_endpoint.py | 2 +- .../endpoints/test_dag_parsing.py | 2 +- .../endpoints/test_dag_run_endpoint.py | 2 +- .../endpoints/test_dag_source_endpoint.py | 2 +- .../endpoints/test_dag_stats_endpoint.py | 2 +- .../endpoints/test_dag_warning_endpoint.py | 2 +- .../endpoints/test_event_log_endpoint.py | 2 +- .../endpoints/test_extra_link_endpoint.py | 2 +- .../endpoints/test_health_endpoint.py | 2 +- .../endpoints/test_import_error_endpoint.py | 2 +- .../endpoints/test_log_endpoint.py | 2 +- .../test_mapped_task_instance_endpoint.py | 2 +- .../endpoints/test_plugin_endpoint.py | 2 +- .../endpoints/test_pool_endpoint.py | 2 +- .../endpoints/test_provider_endpoint.py | 2 +- .../endpoints/test_task_endpoint.py | 2 +- .../endpoints/test_task_instance_endpoint.py | 2 +- .../endpoints/test_variable_endpoint.py | 2 +- .../endpoints/test_version_endpoint.py | 2 +- .../endpoints/test_xcom_endpoint.py | 2 +- .../schemas/test_asset_schema.py | 2 +- .../schemas/test_connection_schema.py | 2 +- .../schemas/test_dag_run_schema.py | 2 +- .../api_connexion/schemas/test_dag_schema.py | 2 - .../schemas/test_error_schema.py | 2 +- .../schemas/test_event_log_schema.py | 2 +- .../schemas/test_pool_schemas.py | 2 +- .../schemas/test_task_instance_schema.py | 1 - .../api_connexion/schemas/test_xcom_schema.py | 2 +- tests/api_connexion/test_auth.py | 2 +- tests/api_connexion/test_error_handling.py | 2 +- tests/api_connexion/test_security.py | 2 +- .../core_api/routes/public/test_assets.py | 2 +- .../routes/public/test_extra_links.py | 2 +- .../core_api/routes/public/test_monitor.py | 2 +- .../core_api/routes/public/test_pools.py | 2 +- .../core_api/routes/public/test_providers.py | 2 +- .../core_api/routes/public/test_version.py | 2 +- .../endpoints/test_rpc_api_endpoint.py | 2 +- tests/assets/test_manager.py | 2 - tests/cli/commands/test_backfill_command.py | 2 +- tests/cli/commands/test_celery_command.py | 2 +- tests/cli/commands/test_connection_command.py | 2 +- tests/cli/commands/test_dag_command.py | 2 +- .../commands/test_dag_processor_command.py | 2 +- tests/cli/commands/test_db_command.py | 2 +- tests/cli/commands/test_jobs_command.py | 1 - tests/cli/commands/test_kerberos_command.py | 2 +- tests/cli/commands/test_kubernetes_command.py | 2 +- tests/cli/commands/test_plugins_command.py | 2 +- tests/cli/commands/test_pool_command.py | 2 +- .../test_rotate_fernet_key_command.py | 2 +- tests/cli/commands/test_scheduler_command.py | 2 +- tests/cli/commands/test_task_command.py | 2 +- tests/cli/commands/test_triggerer_command.py | 2 +- tests/cli/commands/test_variable_command.py | 2 +- tests/core/test_core.py | 2 +- tests/core/test_sentry.py | 2 - tests/core/test_settings.py | 54 +--- tests/core/test_sqlalchemy_config.py | 2 +- tests/dag_processing/test_job_runner.py | 9 - tests/dag_processing/test_processor.py | 20 -- tests/decorators/test_bash.py | 15 - .../decorators/test_branch_external_python.py | 1 - tests/decorators/test_branch_python.py | 1 - tests/decorators/test_branch_virtualenv.py | 1 - tests/decorators/test_condition.py | 4 - tests/decorators/test_python.py | 15 - tests/decorators/test_sensor.py | 6 - tests/decorators/test_short_circuit.py | 1 - tests/executors/test_base_executor.py | 2 - tests/executors/test_executor_loader.py | 2 - tests/executors/test_local_executor.py | 2 +- tests/jobs/test_base_job.py | 2 - tests/jobs/test_local_task_job.py | 17 - tests/jobs/test_scheduler_job.py | 2 +- tests/jobs/test_triggerer_job.py | 12 - tests/lineage/test_lineage.py | 2 - tests/listeners/test_asset_listener.py | 1 - .../test_dag_import_error_listener.py | 2 - tests/listeners/test_listeners.py | 5 - tests/models/test_base.py | 2 +- tests/models/test_baseoperator.py | 2 - tests/models/test_baseoperatormeta.py | 7 - tests/models/test_cleartasks.py | 2 +- tests/models/test_dag.py | 2 +- tests/models/test_dag_version.py | 2 +- tests/models/test_dagbag.py | 11 - tests/models/test_dagcode.py | 2 +- tests/models/test_dagrun.py | 2 +- tests/models/test_dagwarning.py | 2 +- tests/models/test_mappedoperator.py | 26 -- tests/models/test_param.py | 3 - tests/models/test_pool.py | 2 +- tests/models/test_renderedtifields.py | 7 - tests/models/test_serialized_dag.py | 7 - tests/models/test_skipmixin.py | 2 - tests/models/test_taskinstance.py | 91 ------ tests/models/test_trigger.py | 5 - tests/models/test_variable.py | 10 - tests/models/test_xcom.py | 9 +- tests/models/test_xcom_arg.py | 1 - tests/models/test_xcom_arg_map.py | 8 - tests/operators/test_trigger_dagrun.py | 11 - tests/plugins/test_plugins_manager.py | 2 - tests/sensors/test_base.py | 1 - tests/sensors/test_external_task_sensor.py | 43 --- tests/sensors/test_filesystem.py | 1 - tests/serialization/test_pydantic_models.py | 2 - tests/task/test_standard_task_runner.py | 5 - .../deps/test_dag_ti_slots_available_dep.py | 2 +- tests/ti_deps/deps/test_dag_unpaused_dep.py | 2 +- tests/ti_deps/deps/test_dagrun_exists_dep.py | 2 +- .../deps/test_mapped_task_upstream_dep.py | 2 +- .../deps/test_not_in_retry_period_dep.py | 2 +- .../deps/test_not_previously_skipped_dep.py | 2 +- .../deps/test_pool_slots_available_dep.py | 2 +- tests/ti_deps/deps/test_prev_dagrun_dep.py | 2 +- .../deps/test_ready_to_reschedule_dep.py | 2 +- .../deps/test_runnable_exec_date_dep.py | 2 +- tests/ti_deps/deps/test_task_concurrency.py | 2 +- .../ti_deps/deps/test_task_not_running_dep.py | 2 +- tests/ti_deps/deps/test_trigger_rule_dep.py | 2 +- tests/ti_deps/deps/test_valid_state_dep.py | 2 +- tests/utils/log/test_colored_log.py | 2 +- tests/utils/log/test_log_reader.py | 2 +- tests/utils/test_cli_util.py | 2 +- tests/utils/test_db.py | 2 +- tests/utils/test_db_cleanup.py | 2 +- tests/utils/test_dot_renderer.py | 2 +- tests/utils/test_email.py | 2 - tests/utils/test_helpers.py | 2 - tests/utils/test_log_handlers.py | 2 +- tests/utils/test_process_utils.py | 4 - tests/utils/test_retries.py | 2 - tests/utils/test_session.py | 2 +- tests/utils/test_sqlalchemy.py | 2 +- tests/utils/test_state.py | 2 +- ...test_task_handler_with_custom_formatter.py | 1 - tests/utils/test_types.py | 2 +- tests/utils/test_usage_data_collection.py | 1 - tests/www/test_app.py | 2 +- tests/www/test_auth.py | 2 - tests/www/test_security_manager.py | 2 - tests/www/test_utils.py | 6 - tests_common/pytest_plugin.py | 28 -- tests_common/test_utils/db.py | 6 - 212 files changed, 540 insertions(+), 1287 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf9960189220a..4d023c77e0da9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -289,7 +289,7 @@ jobs: runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} # generate no providers constraints only in canary builds - they take quite some time to generate - # they are not needed for regular builds, they are only needed to update constraints in canaries + # they are not needed for regular builds, they are only needed to upe constraints in canaries generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} image-tag: ${{ needs.build-info.outputs.image-tag }} chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} diff --git a/Dockerfile.ci b/Dockerfile.ci index 68d415975f189..c0d1e3b06e309 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -892,19 +892,6 @@ function environment_initialization() { fi RUN_TESTS=${RUN_TESTS:="false"} - if [[ ${DATABASE_ISOLATION=} == "true" ]]; then - echo "${COLOR_BLUE}Force database isolation configuration:${COLOR_RESET}" - export AIRFLOW__CORE__DATABASE_ACCESS_ISOLATION=True - export AIRFLOW__CORE__INTERNAL_API_URL=http://localhost:9080 - # some random secret keys. Setting them as environment variables will make them used in tests and in - # the internal API server - export AIRFLOW__CORE__INTERNAL_API_SECRET_KEY="Z27xjUwQTz4txlWZyJzLqg==" - export AIRFLOW__CORE__FERNET_KEY="l7KBR9aaH2YumhL1InlNf24gTNna8aW2WiwF2s-n_PE=" - if [[ ${START_AIRFLOW=} != "true" ]]; then - export RUN_TESTS_WITH_DATABASE_ISOLATION="true" - fi - fi - CI=${CI:="false"} # Added to have run-tests on path @@ -1086,33 +1073,6 @@ function check_run_tests() { python "${IN_CONTAINER_DIR}/remove_arm_packages.py" fi - if [[ ${DATABASE_ISOLATION=} == "true" ]]; then - echo "${COLOR_BLUE}Starting internal API server:${COLOR_RESET}" - # We need to start the internal API server before running tests - airflow db migrate - # We set a very large clock grace allowing to have tests running in other time/years - AIRFLOW__CORE__INTERNAL_API_CLOCK_GRACE=999999999 airflow internal-api >"${AIRFLOW_HOME}/logs/internal-api.log" 2>&1 & - echo - echo -n "${COLOR_YELLOW}Waiting for internal API server to listen on 9080. ${COLOR_RESET}" - echo - for _ in $(seq 1 40) - do - sleep 0.5 - nc -z localhost 9080 && echo && echo "${COLOR_GREEN}Internal API server started!!${COLOR_RESET}" && break - echo -n "." - done - if ! nc -z localhost 9080; then - echo - echo "${COLOR_RED}Internal API server did not start in 20 seconds!!${COLOR_RESET}" - echo - echo "${COLOR_BLUE}Logs:${COLOR_RESET}" - echo - cat "${AIRFLOW_HOME}/logs/internal-api.log" - echo - exit 1 - fi - fi - if [[ ${TEST_GROUP:=""} == "system" ]]; then exec "${IN_CONTAINER_DIR}/run_system_tests.sh" "${@}" else diff --git a/airflow/utils/session.py b/airflow/utils/session.py index fde56354b4782..1ee899338059a 100644 --- a/airflow/utils/session.py +++ b/airflow/utils/session.py @@ -17,7 +17,6 @@ from __future__ import annotations import contextlib -import os from collections.abc import Generator from functools import wraps from inspect import signature @@ -26,29 +25,12 @@ from sqlalchemy.orm import Session as SASession from airflow import settings -from airflow.api_internal.internal_api_call import InternalApiConfig -from airflow.settings import TracebackSession, TracebackSessionForTests from airflow.typing_compat import ParamSpec @contextlib.contextmanager def create_session(scoped: bool = True) -> Generator[SASession, None, None]: """Contextmanager that will create and teardown a session.""" - if InternalApiConfig.get_use_internal_api(): - if os.environ.get("RUN_TESTS_WITH_DATABASE_ISOLATION", "false").lower() == "true": - traceback_session_for_tests = TracebackSessionForTests() - try: - yield traceback_session_for_tests - if traceback_session_for_tests.current_db_session: - traceback_session_for_tests.current_db_session.commit() - except Exception: - traceback_session_for_tests.current_db_session.rollback() - raise - finally: - traceback_session_for_tests.current_db_session.close() - else: - yield TracebackSession() - return if scoped: Session = getattr(settings, "Session", None) else: diff --git a/contributing-docs/testing/unit_tests.rst b/contributing-docs/testing/unit_tests.rst index c85389fb919f2..6c39b0780edac 100644 --- a/contributing-docs/testing/unit_tests.rst +++ b/contributing-docs/testing/unit_tests.rst @@ -691,40 +691,6 @@ You can also use fixture to create object that needs database just like this. conn = request.getfixturevalue(conn) ... -Running tests with Database isolation -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Running tests with DB isolation is a special case of tests that require ``internal-api`` component to be -started in order to execute the tests. Only selected tests can be run with the database isolation -(TODO: add the list) - they are simulating running untrusted components (dag file processor, triggerer, -worker) running in an environment where there is no DB configuration and certain "internal_api" endpoints -are used to communicate with the internal-api component (that can access the DB directly). - -In the ``database isolation mode`` the test code can access the DB and perform setup/teardown but the code -directly from airflow package will fail if the database is accessed directly - all the DB calls should go -through the internal API component. - -When you run ``breeze testing providers-tests --database-isolation`` - the internal API server will be started for -you automatically: - -.. code-block:: shell - - breeze testing providers-tests --database-isolation tests/dag_processing/ - -However, when you want to run the tests interactively, you need to use ``breeze shell --database-isolation`` -command and either use ``tmux`` to split your terminal and run the internal API component in the second -pane or run it after re-entering the shell with separate ``breeze exec`` command. - -.. code-block:: shell - - breeze shell --database-isolation tests/dag_processing/ - > tmux - > Ctrl-B " - > Panel 1: airflow internal-api - > Panel 2: pytest tests/dag_processing - - - Running Unit tests ------------------ diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index cb38b91013a91..12e276fe01a49 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -1,4 +1,4 @@ - +