From 19488d3fef49bea7d80065c45d33817837020b61 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Thu, 22 Feb 2024 15:35:24 -0800 Subject: [PATCH 1/5] Add executor field to the DB This field will be used by AIP-61 code to store and retrieve which executor a task is meant to run on. --- ...0137_2_9_0_add_new_executor_field_to_db.py | 48 + airflow/models/abstractoperator.py | 2 + airflow/models/baseoperator.py | 13 + airflow/models/mappedoperator.py | 5 + airflow/models/taskinstance.py | 8 + .../serialization/pydantic/taskinstance.py | 1 + airflow/serialization/schema.json | 1 + docs/apache-airflow/img/airflow_erd.sha256 | 14 +- docs/apache-airflow/img/airflow_erd.svg | 1058 +++++++++++++++++ docs/apache-airflow/migrations-ref.rst | 12 + tests/models/test_taskinstance.py | 1 + tests/serialization/test_dag_serialization.py | 1 + tests/www/views/test_views_tasks.py | 7 + 13 files changed, 1170 insertions(+), 1 deletion(-) create mode 100644 airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py diff --git a/airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py b/airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py new file mode 100644 index 0000000000000..08d29b4cd5be8 --- /dev/null +++ b/airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py @@ -0,0 +1,48 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""add new executor field to db + +Revision ID: 677fdbb7fc54 +Revises: ab34f260b71c +Create Date: 2024-03-11 15:26:59.186579 + +""" + +import sqlalchemy as sa +from alembic import op + + +# revision identifiers, used by Alembic. +revision = '677fdbb7fc54' +down_revision = 'ab34f260b71c' +branch_labels = None +depends_on = None +airflow_version = '2.9.0' + + +def upgrade(): + """Apply add executor field to task instance""" + # NOTE: I cannot find a straightforward sdk doc for SA types. I don't know + # if I need default=None. No idea how to tell what are the defaults without docs + op.add_column('task_instance', sa.Column('executor', sa.String(length=1000), default=None)) + + +def downgrade(): + """Unapply add executor field to task instance""" + op.drop_column('task_instance', 'executor') diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index 380c6fcf00f3d..7a47c02382d86 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -60,6 +60,8 @@ DEFAULT_OWNER: str = conf.get_mandatory_value("operators", "default_owner") DEFAULT_POOL_SLOTS: int = 1 DEFAULT_PRIORITY_WEIGHT: int = 1 +# TODO: Is None the right thing here? We want NULL essentially +DEFAULT_EXECUTOR: str | None = None DEFAULT_QUEUE: str = conf.get_mandatory_value("operators", "default_queue") DEFAULT_IGNORE_FIRST_DEPENDS_ON_PAST: bool = conf.getboolean( "scheduler", "ignore_first_depends_on_past_by_default" diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index 5334fc90205aa..110601beb22b2 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -63,6 +63,7 @@ ) from airflow.lineage import apply_lineage, prepare_lineage from airflow.models.abstractoperator import ( + DEFAULT_EXECUTOR, DEFAULT_IGNORE_FIRST_DEPENDS_ON_PAST, DEFAULT_OWNER, DEFAULT_POOL_SLOTS, @@ -208,6 +209,7 @@ def partial(**kwargs): "wait_for_past_depends_before_skipping": DEFAULT_WAIT_FOR_PAST_DEPENDS_BEFORE_SKIPPING, "wait_for_downstream": False, "retries": DEFAULT_RETRIES, + "executor": DEFAULT_EXECUTOR, "queue": DEFAULT_QUEUE, "pool_slots": DEFAULT_POOL_SLOTS, "execution_timeout": DEFAULT_TASK_EXECUTION_TIMEOUT, @@ -259,6 +261,7 @@ def partial( on_retry_callback: None | TaskStateChangeCallback | list[TaskStateChangeCallback] | ArgNotSet = NOTSET, on_skipped_callback: None | TaskStateChangeCallback | list[TaskStateChangeCallback] | ArgNotSet = NOTSET, run_as_user: str | None | ArgNotSet = NOTSET, + executor: str | None = None, executor_config: dict | None | ArgNotSet = NOTSET, inlets: Any | None | ArgNotSet = NOTSET, outlets: Any | None | ArgNotSet = NOTSET, @@ -326,6 +329,7 @@ def partial( "on_success_callback": on_success_callback, "on_skipped_callback": on_skipped_callback, "run_as_user": run_as_user, + "executor": executor, "executor_config": executor_config, "inlets": inlets, "outlets": outlets, @@ -682,6 +686,7 @@ class derived from this one results in the creation of a task object, runs across execution_dates. :param max_active_tis_per_dagrun: When set, a task will be able to limit the concurrent task instances per DAG run. + :param executor: which executor to target when running this task. NOT YET SUPPORTED :param executor_config: Additional task-level configuration parameters that are interpreted by a specific executor. Parameters are namespaced by the name of executor. @@ -783,6 +788,7 @@ def say_hello_world(**context): "do_xcom_push", "multiple_outputs", "allow_nested_operators", + "executor", } # Defines if the operator supports lineage without manual definitions @@ -851,6 +857,7 @@ def __init__( map_index_template: str | None = None, max_active_tis_per_dag: int | None = None, max_active_tis_per_dagrun: int | None = None, + executor: str | None = None, executor_config: dict | None = None, do_xcom_push: bool = True, multiple_outputs: bool = False, @@ -924,6 +931,12 @@ def __init__( if end_date: self.end_date = timezone.convert_to_utc(end_date) + if executor: + warnings.warn( + "Specifying executors for operators is not yet" + f"supported, the value {executor} will have no effect" + ) + self.executor = executor self.executor_config = executor_config or {} self.run_as_user = run_as_user self.retries = parse_retries(retries) diff --git a/airflow/models/mappedoperator.py b/airflow/models/mappedoperator.py index 994e041d9fa5c..9e1200c0494c8 100644 --- a/airflow/models/mappedoperator.py +++ b/airflow/models/mappedoperator.py @@ -28,6 +28,7 @@ from airflow.compat.functools import cache from airflow.exceptions import AirflowException, UnmappableOperator from airflow.models.abstractoperator import ( + DEFAULT_EXECUTOR, DEFAULT_IGNORE_FIRST_DEPENDS_ON_PAST, DEFAULT_OWNER, DEFAULT_POOL_SLOTS, @@ -620,6 +621,10 @@ def on_skipped_callback(self, value: TaskStateChangeCallbackAttrType) -> None: def run_as_user(self) -> str | None: return self.partial_kwargs.get("run_as_user") + @property + def executor(self) -> str | None: + return self.partial_kwargs.get("executor", DEFAULT_EXECUTOR) + @property def executor_config(self) -> dict: return self.partial_kwargs.get("executor_config", {}) diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index c9bd2ce617154..c21c2ae279be6 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -547,6 +547,7 @@ def _refresh_from_db( task_instance.queued_dttm = ti.queued_dttm task_instance.queued_by_job_id = ti.queued_by_job_id task_instance.pid = ti.pid + task_instance.executor = ti.executor task_instance.executor_config = ti.executor_config task_instance.external_executor_id = ti.external_executor_id task_instance.trigger_id = ti.trigger_id @@ -952,6 +953,7 @@ def _refresh_from_task( task_instance.run_as_user = task.run_as_user # Do not set max_tries to task.retries here because max_tries is a cumulative # value that needs to be stored in the db. + task_instance.executor = task.executor task_instance.executor_config = task.executor_config task_instance.operator = task.task_type task_instance.custom_operator_name = getattr(task, "custom_operator_name", None) @@ -1298,6 +1300,8 @@ class TaskInstance(Base, LoggingMixin): queued_dttm = Column(UtcDateTime) queued_by_job_id = Column(Integer) pid = Column(Integer) + # TODO: Should this be nullable? + executor = Column(String(1000)) executor_config = Column(ExecutorConfigType(pickler=dill)) updated_at = Column(UtcDateTime, default=timezone.utcnow, onupdate=timezone.utcnow) rendered_map_index = Column(String(250)) @@ -1483,6 +1487,7 @@ def insert_mapping(run_id: str, task: Operator, map_index: int) -> dict[str, Any "priority_weight": priority_weight, "run_as_user": task.run_as_user, "max_tries": task.retries, + "executor": task.executor, "executor_config": task.executor_config, "operator": task.task_type, "custom_operator_name": getattr(task, "custom_operator_name", None), @@ -3676,6 +3681,7 @@ def __init__( try_number: int, map_index: int, state: str, + executor: str | None, executor_config: Any, pool: str, queue: str, @@ -3691,6 +3697,7 @@ def __init__( self.end_date = end_date self.try_number = try_number self.state = state + self.executor = executor self.executor_config = executor_config self.run_as_user = run_as_user self.pool = pool @@ -3729,6 +3736,7 @@ def from_ti(cls, ti: TaskInstance) -> SimpleTaskInstance: end_date=ti.end_date, try_number=ti.try_number, state=ti.state, + executor=ti.executor, executor_config=ti.executor_config, pool=ti.pool, queue=ti.queue, diff --git a/airflow/serialization/pydantic/taskinstance.py b/airflow/serialization/pydantic/taskinstance.py index 16c44860998fa..cf27d755b5ed9 100644 --- a/airflow/serialization/pydantic/taskinstance.py +++ b/airflow/serialization/pydantic/taskinstance.py @@ -99,6 +99,7 @@ class TaskInstancePydantic(BaseModelPydantic, LoggingMixin): queued_dttm: Optional[datetime] queued_by_job_id: Optional[int] pid: Optional[int] + executor: Optional[str] executor_config: Any updated_at: Optional[datetime] rendered_map_index: Optional[str] diff --git a/airflow/serialization/schema.json b/airflow/serialization/schema.json index a2a6732763641..85631e0965078 100644 --- a/airflow/serialization/schema.json +++ b/airflow/serialization/schema.json @@ -261,6 +261,7 @@ "params": { "$ref": "#/definitions/params_dict" }, "priority_weight": { "type": "number" }, "weight_rule": { "type": "string" }, + "executor": { "type": "string" }, "executor_config": { "$ref": "#/definitions/dict" }, "do_xcom_push": { "type": "boolean" }, "ui_color": { "$ref": "#/definitions/color" }, diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 09f84daea2acb..88439baf8540e 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1 +1,13 @@ -2a24225537326f38be5df14e0b7a8dca867122093e0fa932f1a11ac12d1fb11c \ No newline at end of file +<<<<<<< HEAD +2a24225537326f38be5df14e0b7a8dca867122093e0fa932f1a11ac12d1fb11c +======= +<<<<<<< HEAD +1f9b9955efc927a319bb8c79df50f0f23a59e19b4f8379f95af346c19428c444 +======= +<<<<<<< HEAD +ce71fa2d8f6daea541c46d3528961bdee87421b3612ed2f2451e88630a52fd70 +======= +10cd471ad3066f967bfaf1c6c2eea016f136fdab479629276a0b1a964a9e6984 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> d5f63cc309... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index dc32fe0566902..8d95408eacbee 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -243,6 +243,7 @@ dag_run_note +<<<<<<< HEAD dag_run_note @@ -263,17 +264,74 @@ user_id [INTEGER] +======= +<<<<<<< HEAD + +dag_run_note + +dag_run_id + [INTEGER] + NOT NULL + +content + [VARCHAR(1000)] + +created_at + [TIMESTAMP] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL + +user_id + [INTEGER] +======= + +dag_run_note + +dag_run_id + [INTEGER] + NOT NULL + +content + [VARCHAR(1000)] + +created_at + [TIMESTAMP] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL + +user_id + [INTEGER] +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB ab_user--dag_run_note +<<<<<<< HEAD 0..N +======= +<<<<<<< HEAD + +0..N +>>>>>>> 5b2489abf4... Add executor field to the DB {0,1} +======= + +0..N +{0,1} +>>>>>>> 3ce982cef8... Add executor field to the DB task_instance_note +<<<<<<< HEAD task_instance_note @@ -306,13 +364,94 @@ user_id [INTEGER] +======= +<<<<<<< HEAD + +task_instance_note + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +content + [VARCHAR(1000)] + +created_at + [TIMESTAMP] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL + +user_id + [INTEGER] +======= + +task_instance_note + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +content + [VARCHAR(1000)] + +created_at + [TIMESTAMP] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL + +user_id + [INTEGER] +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB ab_user--task_instance_note +<<<<<<< HEAD 0..N {0,1} +======= +<<<<<<< HEAD + +0..N +{0,1} +======= + +0..N +{0,1} +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB @@ -940,9 +1079,21 @@ dag_run--dag_run_note +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB @@ -961,13 +1112,26 @@ dag_run--dagrun_dataset_event +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance +<<<<<<< HEAD task_instance @@ -1069,24 +1233,253 @@ updated_at [TIMESTAMP] +======= +<<<<<<< HEAD + +task_instance + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +custom_operator_name + [VARCHAR(1000)] + +duration + [DOUBLE_PRECISION] + +end_date + [TIMESTAMP] + +executor_config + [BYTEA] + +external_executor_id + [VARCHAR(250)] + +hostname + [VARCHAR(1000)] + +job_id + [INTEGER] + +max_tries + [INTEGER] + +next_kwargs + [JSON] + +next_method + [VARCHAR(1000)] + +operator + [VARCHAR(1000)] + +pid + [INTEGER] + +pool + [VARCHAR(256)] + NOT NULL + +pool_slots + [INTEGER] + NOT NULL + +priority_weight + [INTEGER] + +queue + [VARCHAR(256)] + +queued_by_job_id + [INTEGER] + +queued_dttm + [TIMESTAMP] + +rendered_map_index + [VARCHAR(250)] + +start_date + [TIMESTAMP] + +state + [VARCHAR(20)] + +trigger_id + [INTEGER] + +trigger_timeout + [TIMESTAMP] + +try_number + [INTEGER] + +unixname + [VARCHAR(1000)] + +updated_at + [TIMESTAMP] +======= + +task_instance + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +custom_operator_name + [VARCHAR(1000)] + +duration + [DOUBLE_PRECISION] + +end_date + [TIMESTAMP] + +executor + [VARCHAR(1000)] + +executor_config + [BYTEA] + +external_executor_id + [VARCHAR(250)] + +hostname + [VARCHAR(1000)] + +job_id + [INTEGER] + +max_tries + [INTEGER] + +next_kwargs + [JSON] + +next_method + [VARCHAR(1000)] + +operator + [VARCHAR(1000)] + +pid + [INTEGER] + +pool + [VARCHAR(256)] + NOT NULL + +pool_slots + [INTEGER] + NOT NULL + +priority_weight + [INTEGER] + +queue + [VARCHAR(256)] + +queued_by_job_id + [INTEGER] + +queued_dttm + [TIMESTAMP] + +rendered_map_index + [VARCHAR(250)] + +start_date + [TIMESTAMP] + +state + [VARCHAR(20)] + +trigger_id + [INTEGER] + +trigger_timeout + [TIMESTAMP] + +try_number + [INTEGER] + +unixname + [VARCHAR(1000)] + +updated_at + [TIMESTAMP] +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB dag_run--task_instance +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB dag_run--task_instance +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_reschedule +<<<<<<< HEAD task_reschedule @@ -1129,80 +1522,290 @@ try_number [INTEGER] NOT NULL +======= +<<<<<<< HEAD + +task_reschedule + +id + [INTEGER] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +duration + [INTEGER] + NOT NULL + +end_date + [TIMESTAMP] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +reschedule_date + [TIMESTAMP] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +start_date + [TIMESTAMP] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +try_number + [INTEGER] + NOT NULL +======= + +task_reschedule + +id + [INTEGER] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +duration + [INTEGER] + NOT NULL + +end_date + [TIMESTAMP] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +reschedule_date + [TIMESTAMP] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +start_date + [TIMESTAMP] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +try_number + [INTEGER] + NOT NULL +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB dag_run--task_reschedule +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB dag_run--task_reschedule +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_instance_note +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_instance_note +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_instance_note +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_instance_note +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_reschedule +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_reschedule +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_reschedule +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_reschedule +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_fail +<<<<<<< HEAD task_fail @@ -1234,38 +1837,154 @@ task_id [VARCHAR(250)] NOT NULL +======= +<<<<<<< HEAD + +task_fail + +id + [INTEGER] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +duration + [INTEGER] + +end_date + [TIMESTAMP] + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +start_date + [TIMESTAMP] + +task_id + [VARCHAR(250)] + NOT NULL +======= + +task_fail + +id + [INTEGER] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +duration + [INTEGER] + +end_date + [TIMESTAMP] + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +start_date + [TIMESTAMP] + +task_id + [VARCHAR(250)] + NOT NULL +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_fail +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_fail +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_fail +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_fail +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_map +<<<<<<< HEAD task_map @@ -1291,38 +2010,142 @@ length [INTEGER] NOT NULL +======= +<<<<<<< HEAD + +task_map + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +keys + [JSON] + +length + [INTEGER] + NOT NULL +======= + +task_map + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +keys + [JSON] + +length + [INTEGER] + NOT NULL +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_map +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_map +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_map +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--task_map +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB xcom +<<<<<<< HEAD xcom @@ -1356,38 +2179,158 @@ value [BYTEA] +======= +<<<<<<< HEAD + +xcom + +dag_run_id + [INTEGER] + NOT NULL + +key + [VARCHAR(512)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +timestamp + [TIMESTAMP] + NOT NULL + +value + [BYTEA] +======= + +xcom + +dag_run_id + [INTEGER] + NOT NULL + +key + [VARCHAR(512)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +timestamp + [TIMESTAMP] + NOT NULL + +value + [BYTEA] +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--xcom +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--xcom +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--xcom +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +0..N +1 +======= + +0..N +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--xcom +<<<<<<< HEAD 0..N 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB rendered_task_instance_fields +<<<<<<< HEAD rendered_task_instance_fields @@ -1413,34 +2356,137 @@ rendered_fields [JSON] NOT NULL +======= +<<<<<<< HEAD + +rendered_task_instance_fields + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +k8s_pod_yaml + [JSON] + +rendered_fields + [JSON] + NOT NULL +======= + +rendered_task_instance_fields + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +k8s_pod_yaml + [JSON] + +rendered_fields + [JSON] + NOT NULL +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--rendered_task_instance_fields +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--rendered_task_instance_fields +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--rendered_task_instance_fields +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB task_instance--rendered_task_instance_fields +<<<<<<< HEAD 1 1 +======= +<<<<<<< HEAD + +1 +1 +======= + +1 +1 +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB @@ -1620,9 +2666,21 @@ trigger--task_instance +<<<<<<< HEAD 0..N {0,1} +======= +<<<<<<< HEAD + +0..N +{0,1} +======= + +0..N +{0,1} +>>>>>>> 3ce982cef8... Add executor field to the DB +>>>>>>> 5b2489abf4... Add executor field to the DB diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 13c70abe9d64f..be5ddc36b568d 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,14 +39,26 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=================================+===================+===================+==============================================================+ +<<<<<<< HEAD | ``1949afb29106`` (head) | ``ee1467d4aa35`` | ``2.9.0`` | update trigger kwargs type | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``ee1467d4aa35`` | ``b4078ac230a1`` | ``2.9.0`` | add display name for dag and task instance | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``b4078ac230a1`` | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | +======= +<<<<<<< HEAD +| ``b4078ac230a1`` (head) | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | +>>>>>>> 5b2489abf4... Add executor field to the DB +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``8e1c784a4fc7`` | ``ab34f260b71c`` | ``2.9.0`` | Adding max_consecutive_failed_dag_runs column to dag_model | +======= +<<<<<<< HEAD +| ``8e1c784a4fc7`` (head) | ``ab34f260b71c`` | ``2.9.0`` | Adding max_consecutive_failed_dag_runs column to dag_model | +>>>>>>> d5f63cc309... Add executor field to the DB | | | | table | +======= +| ``677fdbb7fc54`` (head) | ``ab34f260b71c`` | ``2.9.0`` | add new executor field to db | +>>>>>>> 3ce982cef8... Add executor field to the DB +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``ab34f260b71c`` | ``d75389605139`` | ``2.9.0`` | add dataset_expression in DagModel | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ diff --git a/tests/models/test_taskinstance.py b/tests/models/test_taskinstance.py index e6187311429c9..8dacc839cb8a3 100644 --- a/tests/models/test_taskinstance.py +++ b/tests/models/test_taskinstance.py @@ -3315,6 +3315,7 @@ def test_refresh_from_db(self, create_task_instance): "rendered_map_index": None, "queued_by_job_id": 321, "pid": 123, + "executor": "some_executor", "executor_config": {"Some": {"extra": "information"}}, "external_executor_id": "some_executor_id", "trigger_timeout": None, diff --git a/tests/serialization/test_dag_serialization.py b/tests/serialization/test_dag_serialization.py index 4b5632cd20f8a..b02cda5e3be37 100644 --- a/tests/serialization/test_dag_serialization.py +++ b/tests/serialization/test_dag_serialization.py @@ -1266,6 +1266,7 @@ def test_no_new_fields_added_to_base_operator(self): "email_on_failure": True, "email_on_retry": True, "execution_timeout": None, + "executor": None, "executor_config": {}, "ignore_first_depends_on_past": True, "inlets": [], diff --git a/tests/www/views/test_views_tasks.py b/tests/www/views/test_views_tasks.py index 90cc5bbc1c67a..bc7ce29cec73f 100644 --- a/tests/www/views/test_views_tasks.py +++ b/tests/www/views/test_views_tasks.py @@ -1043,6 +1043,7 @@ def test_task_instances(admin_client): "duration": None, "end_date": None, "execution_date": DEFAULT_DATE.isoformat(), + "executor": None, "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1077,6 +1078,7 @@ def test_task_instances(admin_client): "duration": None, "end_date": None, "execution_date": DEFAULT_DATE.isoformat(), + "executor": None, "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1111,6 +1113,7 @@ def test_task_instances(admin_client): "duration": None, "end_date": None, "execution_date": DEFAULT_DATE.isoformat(), + "executor": None, "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1145,6 +1148,7 @@ def test_task_instances(admin_client): "duration": None, "end_date": None, "execution_date": DEFAULT_DATE.isoformat(), + "executor": None, "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1179,6 +1183,7 @@ def test_task_instances(admin_client): "duration": None, "end_date": None, "execution_date": DEFAULT_DATE.isoformat(), + "executor": None, "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1213,6 +1218,7 @@ def test_task_instances(admin_client): "duration": None, "end_date": None, "execution_date": DEFAULT_DATE.isoformat(), + "executor": None, "executor_config": {}, "external_executor_id": None, "hostname": "", @@ -1247,6 +1253,7 @@ def test_task_instances(admin_client): "duration": None, "end_date": None, "execution_date": DEFAULT_DATE.isoformat(), + "executor": None, "executor_config": {}, "external_executor_id": None, "hostname": "", From 2b0faf7e0f61c11420a5499c89868e1a5cbba6b1 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Wed, 20 Mar 2024 14:49:31 -0700 Subject: [PATCH 2/5] Fixes and updates from PR --- .../0137_2_9_0_add_new_executor_field_to_db.py | 2 -- airflow/models/abstractoperator.py | 1 - airflow/models/baseoperator.py | 6 +++--- airflow/models/taskinstance.py | 1 - docs/apache-airflow/img/airflow_erd.sha256 | 14 ++++++++++++++ docs/apache-airflow/img/airflow_erd.svg | 4 ++-- 6 files changed, 19 insertions(+), 9 deletions(-) diff --git a/airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py b/airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py index 08d29b4cd5be8..5caf4f2103c53 100644 --- a/airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py +++ b/airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py @@ -38,8 +38,6 @@ def upgrade(): """Apply add executor field to task instance""" - # NOTE: I cannot find a straightforward sdk doc for SA types. I don't know - # if I need default=None. No idea how to tell what are the defaults without docs op.add_column('task_instance', sa.Column('executor', sa.String(length=1000), default=None)) diff --git a/airflow/models/abstractoperator.py b/airflow/models/abstractoperator.py index 7a47c02382d86..c78eeb9ee7d6f 100644 --- a/airflow/models/abstractoperator.py +++ b/airflow/models/abstractoperator.py @@ -60,7 +60,6 @@ DEFAULT_OWNER: str = conf.get_mandatory_value("operators", "default_owner") DEFAULT_POOL_SLOTS: int = 1 DEFAULT_PRIORITY_WEIGHT: int = 1 -# TODO: Is None the right thing here? We want NULL essentially DEFAULT_EXECUTOR: str | None = None DEFAULT_QUEUE: str = conf.get_mandatory_value("operators", "default_queue") DEFAULT_IGNORE_FIRST_DEPENDS_ON_PAST: bool = conf.getboolean( diff --git a/airflow/models/baseoperator.py b/airflow/models/baseoperator.py index 110601beb22b2..05807db47e1b7 100644 --- a/airflow/models/baseoperator.py +++ b/airflow/models/baseoperator.py @@ -261,7 +261,7 @@ def partial( on_retry_callback: None | TaskStateChangeCallback | list[TaskStateChangeCallback] | ArgNotSet = NOTSET, on_skipped_callback: None | TaskStateChangeCallback | list[TaskStateChangeCallback] | ArgNotSet = NOTSET, run_as_user: str | None | ArgNotSet = NOTSET, - executor: str | None = None, + executor: str | None | ArgNotSet = NOTSET, executor_config: dict | None | ArgNotSet = NOTSET, inlets: Any | None | ArgNotSet = NOTSET, outlets: Any | None | ArgNotSet = NOTSET, @@ -686,7 +686,7 @@ class derived from this one results in the creation of a task object, runs across execution_dates. :param max_active_tis_per_dagrun: When set, a task will be able to limit the concurrent task instances per DAG run. - :param executor: which executor to target when running this task. NOT YET SUPPORTED + :param executor: Which executor to target when running this task. NOT YET SUPPORTED :param executor_config: Additional task-level configuration parameters that are interpreted by a specific executor. Parameters are namespaced by the name of executor. @@ -934,7 +934,7 @@ def __init__( if executor: warnings.warn( "Specifying executors for operators is not yet" - f"supported, the value {executor} will have no effect" + f"supported, the value {executor!r} will have no effect" ) self.executor = executor self.executor_config = executor_config or {} diff --git a/airflow/models/taskinstance.py b/airflow/models/taskinstance.py index c21c2ae279be6..c0db3d6c101f7 100644 --- a/airflow/models/taskinstance.py +++ b/airflow/models/taskinstance.py @@ -1300,7 +1300,6 @@ class TaskInstance(Base, LoggingMixin): queued_dttm = Column(UtcDateTime) queued_by_job_id = Column(Integer) pid = Column(Integer) - # TODO: Should this be nullable? executor = Column(String(1000)) executor_config = Column(ExecutorConfigType(pickler=dill)) updated_at = Column(UtcDateTime, default=timezone.utcnow, onupdate=timezone.utcnow) diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 88439baf8540e..0d874d9b6703a 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1,13 +1,27 @@ <<<<<<< HEAD +<<<<<<< HEAD 2a24225537326f38be5df14e0b7a8dca867122093e0fa932f1a11ac12d1fb11c ======= +======= +>>>>>>> 4640e9d64b... Fixes and updates from PR <<<<<<< HEAD 1f9b9955efc927a319bb8c79df50f0f23a59e19b4f8379f95af346c19428c444 ======= +======= +>>>>>>> 358ea638f5... Fixes and updates from PR <<<<<<< HEAD ce71fa2d8f6daea541c46d3528961bdee87421b3612ed2f2451e88630a52fd70 ======= 10cd471ad3066f967bfaf1c6c2eea016f136fdab479629276a0b1a964a9e6984 >>>>>>> 3ce982cef8... Add executor field to the DB +<<<<<<< HEAD >>>>>>> d5f63cc309... Add executor field to the DB +<<<<<<< HEAD >>>>>>> 5b2489abf4... Add executor field to the DB +======= +======= +======= +b6d18f2ba16f96bbc4da69d0d51c90d71a892144f8603f2821dfa1f3294629de +>>>>>>> d3d1df810e... Fixes and updates from PR +>>>>>>> 358ea638f5... Fixes and updates from PR +>>>>>>> 4640e9d64b... Fixes and updates from PR diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 8d95408eacbee..95f4571559a05 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -2284,7 +2284,7 @@ 1 ======= -1 +0..N 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB @@ -2303,7 +2303,7 @@ 1 ======= -0..N +1 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB From 256b01f214f27056f46e81e6c2e5a9e35cd57922 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Wed, 20 Mar 2024 14:56:30 -0700 Subject: [PATCH 3/5] Repair migration after rebase --- ...138_2_9_0_add_new_executor_field_to_db.py} | 6 +- docs/apache-airflow/img/airflow_erd.sha256 | 9 + docs/apache-airflow/img/airflow_erd.svg | 321 +++++++++++++++++- docs/apache-airflow/migrations-ref.rst | 11 +- 4 files changed, 336 insertions(+), 11 deletions(-) rename airflow/migrations/versions/{0137_2_9_0_add_new_executor_field_to_db.py => 0138_2_9_0_add_new_executor_field_to_db.py} (93%) diff --git a/airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py b/airflow/migrations/versions/0138_2_9_0_add_new_executor_field_to_db.py similarity index 93% rename from airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py rename to airflow/migrations/versions/0138_2_9_0_add_new_executor_field_to_db.py index 5caf4f2103c53..f65a8f33c0fa8 100644 --- a/airflow/migrations/versions/0137_2_9_0_add_new_executor_field_to_db.py +++ b/airflow/migrations/versions/0138_2_9_0_add_new_executor_field_to_db.py @@ -19,8 +19,8 @@ """add new executor field to db Revision ID: 677fdbb7fc54 -Revises: ab34f260b71c -Create Date: 2024-03-11 15:26:59.186579 +Revises: 8e1c784a4fc7 +Create Date: 2024-03-20 15:26:59.186579 """ @@ -30,7 +30,7 @@ # revision identifiers, used by Alembic. revision = '677fdbb7fc54' -down_revision = 'ab34f260b71c' +down_revision = '8e1c784a4fc7' branch_labels = None depends_on = None airflow_version = '2.9.0' diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 0d874d9b6703a..967297246ea4e 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1,9 +1,12 @@ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD 2a24225537326f38be5df14e0b7a8dca867122093e0fa932f1a11ac12d1fb11c ======= ======= >>>>>>> 4640e9d64b... Fixes and updates from PR +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase <<<<<<< HEAD 1f9b9955efc927a319bb8c79df50f0f23a59e19b4f8379f95af346c19428c444 ======= @@ -24,4 +27,10 @@ ce71fa2d8f6daea541c46d3528961bdee87421b3612ed2f2451e88630a52fd70 b6d18f2ba16f96bbc4da69d0d51c90d71a892144f8603f2821dfa1f3294629de >>>>>>> d3d1df810e... Fixes and updates from PR >>>>>>> 358ea638f5... Fixes and updates from PR +<<<<<<< HEAD >>>>>>> 4640e9d64b... Fixes and updates from PR +======= +======= +c912e1f161da724c66690c717a7c2d483e74c268c1cbc1decfe70a709ab9b4db +>>>>>>> a25a99d7cc... Repair migration after rebase +>>>>>>> 7d40bcf99e... Repair migration after rebase diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 95f4571559a05..a9a89d4b3c091 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -244,6 +244,9 @@ dag_run_note <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase dag_run_note @@ -264,6 +267,7 @@ user_id [INTEGER] +<<<<<<< HEAD ======= <<<<<<< HEAD @@ -309,11 +313,14 @@ [INTEGER] >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase ab_user--dag_run_note <<<<<<< HEAD +<<<<<<< HEAD 0..N ======= @@ -321,17 +328,19 @@ 0..N >>>>>>> 5b2489abf4... Add executor field to the DB -{0,1} ======= - -0..N -{0,1} ->>>>>>> 3ce982cef8... Add executor field to the DB + +0..N +>>>>>>> 7d40bcf99e... Repair migration after rebase +{0,1} task_instance_note <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance_note @@ -364,6 +373,7 @@ user_id [INTEGER] +<<<<<<< HEAD ======= <<<<<<< HEAD @@ -433,11 +443,14 @@ [INTEGER] >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase ab_user--task_instance_note <<<<<<< HEAD +<<<<<<< HEAD 0..N {0,1} @@ -452,6 +465,11 @@ {0,1} >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +{0,1} +>>>>>>> 7d40bcf99e... Repair migration after rebase @@ -1080,6 +1098,7 @@ dag_run--dag_run_note <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -1094,6 +1113,11 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase @@ -1113,6 +1137,7 @@ dag_run--dagrun_dataset_event <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -1127,11 +1152,19 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance @@ -1160,6 +1193,7 @@ end_date [TIMESTAMP] +<<<<<<< HEAD executor_config [BYTEA] @@ -1218,6 +1252,66 @@ task_display_name [VARCHAR(2000)] +======= +executor + [VARCHAR(1000)] + +executor_config + [BYTEA] + +external_executor_id + [VARCHAR(250)] + +hostname + [VARCHAR(1000)] + +job_id + [INTEGER] + +max_tries + [INTEGER] + +next_kwargs + [JSON] + +next_method + [VARCHAR(1000)] + +operator + [VARCHAR(1000)] + +pid + [INTEGER] + +pool + [VARCHAR(256)] + NOT NULL + +pool_slots + [INTEGER] + NOT NULL + +priority_weight + [INTEGER] + +queue + [VARCHAR(256)] + +queued_by_job_id + [INTEGER] + +queued_dttm + [TIMESTAMP] + +rendered_map_index + [VARCHAR(250)] + +start_date + [TIMESTAMP] + +state + [VARCHAR(20)] +>>>>>>> 7d40bcf99e... Repair migration after rebase trigger_id [INTEGER] @@ -1233,6 +1327,7 @@ updated_at [TIMESTAMP] +<<<<<<< HEAD ======= <<<<<<< HEAD @@ -1437,11 +1532,14 @@ [TIMESTAMP] >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase dag_run--task_instance <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -1456,11 +1554,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase dag_run--task_instance <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -1475,11 +1579,19 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_reschedule <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase task_reschedule @@ -1522,6 +1634,7 @@ try_number [INTEGER] NOT NULL +<<<<<<< HEAD ======= <<<<<<< HEAD @@ -1611,11 +1724,14 @@ NOT NULL >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase dag_run--task_reschedule <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -1630,11 +1746,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase dag_run--task_reschedule <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -1649,11 +1771,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_instance_note <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -1668,11 +1796,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_instance_note <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -1687,11 +1821,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_instance_note <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -1706,11 +1846,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_instance_note <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -1725,11 +1871,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_reschedule <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -1744,11 +1896,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_reschedule <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -1763,11 +1921,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_reschedule <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -1782,11 +1946,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_reschedule <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -1801,11 +1971,19 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_fail <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase task_fail @@ -1837,6 +2015,7 @@ task_id [VARCHAR(250)] NOT NULL +<<<<<<< HEAD ======= <<<<<<< HEAD @@ -1904,11 +2083,14 @@ NOT NULL >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_fail <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -1923,11 +2105,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_fail <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -1942,11 +2130,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_fail <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -1961,11 +2155,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_fail <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -1980,11 +2180,19 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_map <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase task_map @@ -2010,6 +2218,7 @@ length [INTEGER] NOT NULL +<<<<<<< HEAD ======= <<<<<<< HEAD @@ -2065,11 +2274,14 @@ NOT NULL >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_map <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -2084,11 +2296,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_map <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -2103,11 +2321,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_map <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -2122,11 +2346,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--task_map <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -2141,11 +2371,19 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase xcom <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase xcom @@ -2179,6 +2417,7 @@ value [BYTEA] +<<<<<<< HEAD ======= <<<<<<< HEAD @@ -2250,15 +2489,20 @@ [BYTEA] >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--xcom <<<<<<< HEAD +<<<<<<< HEAD 1 1 ======= +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase <<<<<<< HEAD 0..N @@ -2268,16 +2512,27 @@ 0..N 1 >>>>>>> 3ce982cef8... Add executor field to the DB +<<<<<<< HEAD >>>>>>> 5b2489abf4... Add executor field to the DB +======= +======= + +0..N +1 +>>>>>>> a25a99d7cc... Repair migration after rebase +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--xcom <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 ======= +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase <<<<<<< HEAD 1 @@ -2287,12 +2542,21 @@ 0..N 1 >>>>>>> 3ce982cef8... Add executor field to the DB +<<<<<<< HEAD >>>>>>> 5b2489abf4... Add executor field to the DB +======= +======= + +1 +1 +>>>>>>> a25a99d7cc... Repair migration after rebase +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--xcom <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -2307,11 +2571,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--xcom <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -2326,11 +2596,19 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase rendered_task_instance_fields <<<<<<< HEAD +<<<<<<< HEAD +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase rendered_task_instance_fields @@ -2356,6 +2634,7 @@ rendered_fields [JSON] NOT NULL +<<<<<<< HEAD ======= <<<<<<< HEAD @@ -2411,11 +2690,14 @@ NOT NULL >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--rendered_task_instance_fields <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -2430,11 +2712,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--rendered_task_instance_fields <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -2449,11 +2737,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--rendered_task_instance_fields <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -2468,11 +2762,17 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase task_instance--rendered_task_instance_fields <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -2487,6 +2787,11 @@ 1 >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +1 +1 +>>>>>>> 7d40bcf99e... Repair migration after rebase @@ -2667,6 +2972,7 @@ trigger--task_instance <<<<<<< HEAD +<<<<<<< HEAD 0..N {0,1} @@ -2681,6 +2987,11 @@ {0,1} >>>>>>> 3ce982cef8... Add executor field to the DB >>>>>>> 5b2489abf4... Add executor field to the DB +======= + +0..N +{0,1} +>>>>>>> 7d40bcf99e... Repair migration after rebase diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index be5ddc36b568d..34a1d7fd540a5 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -40,12 +40,15 @@ Here's the list of all the Database Migrations that are executed via when you ru | Revision ID | Revises ID | Airflow Version | Description | +=================================+===================+===================+==============================================================+ <<<<<<< HEAD +<<<<<<< HEAD | ``1949afb29106`` (head) | ``ee1467d4aa35`` | ``2.9.0`` | update trigger kwargs type | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``ee1467d4aa35`` | ``b4078ac230a1`` | ``2.9.0`` | add display name for dag and task instance | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``b4078ac230a1`` | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | ======= +======= +>>>>>>> 7d40bcf99e... Repair migration after rebase <<<<<<< HEAD | ``b4078ac230a1`` (head) | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | >>>>>>> 5b2489abf4... Add executor field to the DB @@ -55,10 +58,12 @@ Here's the list of all the Database Migrations that are executed via when you ru <<<<<<< HEAD | ``8e1c784a4fc7`` (head) | ``ab34f260b71c`` | ``2.9.0`` | Adding max_consecutive_failed_dag_runs column to dag_model | >>>>>>> d5f63cc309... Add executor field to the DB -| | | | table | ======= -| ``677fdbb7fc54`` (head) | ``ab34f260b71c`` | ``2.9.0`` | add new executor field to db | ->>>>>>> 3ce982cef8... Add executor field to the DB +| ``677fdbb7fc54`` (head) | ``8e1c784a4fc7`` | ``2.9.0`` | add new executor field to db | ++---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ +| ``8e1c784a4fc7`` | ``ab34f260b71c`` | ``2.9.0`` | Adding max_consecutive_failed_dag_runs column to dag_model | +>>>>>>> a25a99d7cc... Repair migration after rebase +| | | | table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``ab34f260b71c`` | ``d75389605139`` | ``2.9.0`` | add dataset_expression in DagModel | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ From 5634c4312a4bd16bb6b68f49072ecc246b11ba59 Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Mon, 25 Mar 2024 12:18:31 -0700 Subject: [PATCH 4/5] Rebase yet another migration overtop --- ... => 0139_2_10_0_add_new_executor_field_to_db.py} | 8 ++++---- docs/apache-airflow/img/airflow_erd.sha256 | 4 ++++ docs/apache-airflow/img/airflow_erd.svg | 12 ++++++++++++ docs/apache-airflow/migrations-ref.rst | 13 +++++-------- 4 files changed, 25 insertions(+), 12 deletions(-) rename airflow/migrations/versions/{0138_2_9_0_add_new_executor_field_to_db.py => 0139_2_10_0_add_new_executor_field_to_db.py} (91%) diff --git a/airflow/migrations/versions/0138_2_9_0_add_new_executor_field_to_db.py b/airflow/migrations/versions/0139_2_10_0_add_new_executor_field_to_db.py similarity index 91% rename from airflow/migrations/versions/0138_2_9_0_add_new_executor_field_to_db.py rename to airflow/migrations/versions/0139_2_10_0_add_new_executor_field_to_db.py index f65a8f33c0fa8..9e3d615f5093a 100644 --- a/airflow/migrations/versions/0138_2_9_0_add_new_executor_field_to_db.py +++ b/airflow/migrations/versions/0139_2_10_0_add_new_executor_field_to_db.py @@ -19,8 +19,8 @@ """add new executor field to db Revision ID: 677fdbb7fc54 -Revises: 8e1c784a4fc7 -Create Date: 2024-03-20 15:26:59.186579 +Revises: b4078ac230a1 +Create Date: 2024-03-25 15:26:59.186579 """ @@ -30,10 +30,10 @@ # revision identifiers, used by Alembic. revision = '677fdbb7fc54' -down_revision = '8e1c784a4fc7' +down_revision = 'b4078ac230a1' branch_labels = None depends_on = None -airflow_version = '2.9.0' +airflow_version = '2.10.0' def upgrade(): diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index 967297246ea4e..e327a3153dd48 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1,6 +1,7 @@ <<<<<<< HEAD <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD 2a24225537326f38be5df14e0b7a8dca867122093e0fa932f1a11ac12d1fb11c ======= ======= @@ -34,3 +35,6 @@ b6d18f2ba16f96bbc4da69d0d51c90d71a892144f8603f2821dfa1f3294629de c912e1f161da724c66690c717a7c2d483e74c268c1cbc1decfe70a709ab9b4db >>>>>>> a25a99d7cc... Repair migration after rebase >>>>>>> 7d40bcf99e... Repair migration after rebase +======= +23474e1d31011898efa811648171531f77fcb4224406507265f996cd8a72adcd +>>>>>>> 946158562e... Rebase yet another migration overtop diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index a9a89d4b3c091..420e5d0dc9135 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -2497,6 +2497,7 @@ task_instance--xcom <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD 1 1 @@ -2521,12 +2522,18 @@ 1 >>>>>>> a25a99d7cc... Repair migration after rebase >>>>>>> 7d40bcf99e... Repair migration after rebase +======= + +0..N +1 +>>>>>>> 946158562e... Rebase yet another migration overtop task_instance--xcom <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD 0..N 1 @@ -2551,6 +2558,11 @@ 1 >>>>>>> a25a99d7cc... Repair migration after rebase >>>>>>> 7d40bcf99e... Repair migration after rebase +======= + +1 +1 +>>>>>>> 946158562e... Rebase yet another migration overtop diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 34a1d7fd540a5..7dc8da994b90d 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -41,6 +41,7 @@ Here's the list of all the Database Migrations that are executed via when you ru +=================================+===================+===================+==============================================================+ <<<<<<< HEAD <<<<<<< HEAD +<<<<<<< HEAD | ``1949afb29106`` (head) | ``ee1467d4aa35`` | ``2.9.0`` | update trigger kwargs type | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``ee1467d4aa35`` | ``b4078ac230a1`` | ``2.9.0`` | add display name for dag and task instance | @@ -52,17 +53,13 @@ Here's the list of all the Database Migrations that are executed via when you ru <<<<<<< HEAD | ``b4078ac230a1`` (head) | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | >>>>>>> 5b2489abf4... Add executor field to the DB -+---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``8e1c784a4fc7`` | ``ab34f260b71c`` | ``2.9.0`` | Adding max_consecutive_failed_dag_runs column to dag_model | ======= -<<<<<<< HEAD -| ``8e1c784a4fc7`` (head) | ``ab34f260b71c`` | ``2.9.0`` | Adding max_consecutive_failed_dag_runs column to dag_model | ->>>>>>> d5f63cc309... Add executor field to the DB -======= -| ``677fdbb7fc54`` (head) | ``8e1c784a4fc7`` | ``2.9.0`` | add new executor field to db | +| ``677fdbb7fc54`` (head) | ``b4078ac230a1`` | ``2.10.0`` | add new executor field to db | +>>>>>>> 946158562e... Rebase yet another migration overtop ++---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ +| ``b4078ac230a1`` | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``8e1c784a4fc7`` | ``ab34f260b71c`` | ``2.9.0`` | Adding max_consecutive_failed_dag_runs column to dag_model | ->>>>>>> a25a99d7cc... Repair migration after rebase | | | | table | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``ab34f260b71c`` | ``d75389605139`` | ``2.9.0`` | add dataset_expression in DagModel | From 66be9cc5b882833bdef775e3ad368c12dacee40b Mon Sep 17 00:00:00 2001 From: Niko Oliveira Date: Mon, 1 Apr 2024 14:11:47 -0700 Subject: [PATCH 5/5] Yet another rebase with conflict resolutions --- ...41_2_10_0_add_new_executor_field_to_db.py} | 6 +- docs/apache-airflow/img/airflow_erd.sha256 | 41 +- docs/apache-airflow/img/airflow_erd.svg | 2298 ++++------------- docs/apache-airflow/migrations-ref.rst | 18 +- 4 files changed, 467 insertions(+), 1896 deletions(-) rename airflow/migrations/versions/{0139_2_10_0_add_new_executor_field_to_db.py => 0141_2_10_0_add_new_executor_field_to_db.py} (93%) diff --git a/airflow/migrations/versions/0139_2_10_0_add_new_executor_field_to_db.py b/airflow/migrations/versions/0141_2_10_0_add_new_executor_field_to_db.py similarity index 93% rename from airflow/migrations/versions/0139_2_10_0_add_new_executor_field_to_db.py rename to airflow/migrations/versions/0141_2_10_0_add_new_executor_field_to_db.py index 9e3d615f5093a..67e463f1cc1d5 100644 --- a/airflow/migrations/versions/0139_2_10_0_add_new_executor_field_to_db.py +++ b/airflow/migrations/versions/0141_2_10_0_add_new_executor_field_to_db.py @@ -19,8 +19,8 @@ """add new executor field to db Revision ID: 677fdbb7fc54 -Revises: b4078ac230a1 -Create Date: 2024-03-25 15:26:59.186579 +Revises: 1949afb29106 +Create Date: 2024-04-01 15:26:59.186579 """ @@ -30,7 +30,7 @@ # revision identifiers, used by Alembic. revision = '677fdbb7fc54' -down_revision = 'b4078ac230a1' +down_revision = '1949afb29106' branch_labels = None depends_on = None airflow_version = '2.10.0' diff --git a/docs/apache-airflow/img/airflow_erd.sha256 b/docs/apache-airflow/img/airflow_erd.sha256 index e327a3153dd48..d1514428254b4 100644 --- a/docs/apache-airflow/img/airflow_erd.sha256 +++ b/docs/apache-airflow/img/airflow_erd.sha256 @@ -1,40 +1 @@ -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -2a24225537326f38be5df14e0b7a8dca867122093e0fa932f1a11ac12d1fb11c -======= -======= ->>>>>>> 4640e9d64b... Fixes and updates from PR -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase -<<<<<<< HEAD -1f9b9955efc927a319bb8c79df50f0f23a59e19b4f8379f95af346c19428c444 -======= -======= ->>>>>>> 358ea638f5... Fixes and updates from PR -<<<<<<< HEAD -ce71fa2d8f6daea541c46d3528961bdee87421b3612ed2f2451e88630a52fd70 -======= -10cd471ad3066f967bfaf1c6c2eea016f136fdab479629276a0b1a964a9e6984 ->>>>>>> 3ce982cef8... Add executor field to the DB -<<<<<<< HEAD ->>>>>>> d5f63cc309... Add executor field to the DB -<<<<<<< HEAD ->>>>>>> 5b2489abf4... Add executor field to the DB -======= -======= -======= -b6d18f2ba16f96bbc4da69d0d51c90d71a892144f8603f2821dfa1f3294629de ->>>>>>> d3d1df810e... Fixes and updates from PR ->>>>>>> 358ea638f5... Fixes and updates from PR -<<<<<<< HEAD ->>>>>>> 4640e9d64b... Fixes and updates from PR -======= -======= -c912e1f161da724c66690c717a7c2d483e74c268c1cbc1decfe70a709ab9b4db ->>>>>>> a25a99d7cc... Repair migration after rebase ->>>>>>> 7d40bcf99e... Repair migration after rebase -======= -23474e1d31011898efa811648171531f77fcb4224406507265f996cd8a72adcd ->>>>>>> 946158562e... Rebase yet another migration overtop +cccb1a4a3f22027e354cea27bb34996fd45146494cbe6893d938c02c2ddb1a61 \ No newline at end of file diff --git a/docs/apache-airflow/img/airflow_erd.svg b/docs/apache-airflow/img/airflow_erd.svg index 420e5d0dc9135..58ea95b0f403f 100644 --- a/docs/apache-airflow/img/airflow_erd.svg +++ b/docs/apache-airflow/img/airflow_erd.svg @@ -220,256 +220,99 @@ ab_user_role - -ab_user_role - -id - [INTEGER] - NOT NULL - -role_id - [INTEGER] - -user_id - [INTEGER] + +ab_user_role + +id + [INTEGER] + NOT NULL + +role_id + [INTEGER] + +user_id + [INTEGER] ab_user--ab_user_role - -0..N -{0,1} + +0..N +{0,1} dag_run_note -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase - -dag_run_note - -dag_run_id - [INTEGER] - NOT NULL - -content - [VARCHAR(1000)] - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -user_id - [INTEGER] -<<<<<<< HEAD -======= -<<<<<<< HEAD - -dag_run_note - -dag_run_id - [INTEGER] - NOT NULL - -content - [VARCHAR(1000)] - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -user_id - [INTEGER] -======= - -dag_run_note - -dag_run_id - [INTEGER] - NOT NULL - -content - [VARCHAR(1000)] - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -user_id - [INTEGER] ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase + +dag_run_note + +dag_run_id + [INTEGER] + NOT NULL + +content + [VARCHAR(1000)] + +created_at + [TIMESTAMP] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL + +user_id + [INTEGER] ab_user--dag_run_note -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -======= -<<<<<<< HEAD - -0..N ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N ->>>>>>> 7d40bcf99e... Repair migration after rebase -{0,1} + +0..N +{0,1} task_instance_note -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase - -task_instance_note - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -content - [VARCHAR(1000)] - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -user_id - [INTEGER] -<<<<<<< HEAD -======= -<<<<<<< HEAD - -task_instance_note - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -content - [VARCHAR(1000)] - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -user_id - [INTEGER] -======= - -task_instance_note - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -content - [VARCHAR(1000)] - -created_at - [TIMESTAMP] - NOT NULL - -updated_at - [TIMESTAMP] - NOT NULL - -user_id - [INTEGER] ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase + +task_instance_note + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +content + [VARCHAR(1000)] + +created_at + [TIMESTAMP] + NOT NULL + +updated_at + [TIMESTAMP] + NOT NULL + +user_id + [INTEGER] ab_user--task_instance_note -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -{0,1} -======= -<<<<<<< HEAD - -0..N -{0,1} -======= - -0..N -{0,1} ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -{0,1} ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +{0,1} @@ -1097,1713 +940,510 @@ dag_run--dag_run_note -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 dagrun_dataset_event - -dagrun_dataset_event - -dag_run_id - [INTEGER] - NOT NULL - -event_id - [INTEGER] - NOT NULL + +dagrun_dataset_event + +dag_run_id + [INTEGER] + NOT NULL + +event_id + [INTEGER] + NOT NULL dag_run--dagrun_dataset_event -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase - -task_instance - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -custom_operator_name - [VARCHAR(1000)] - -duration - [DOUBLE_PRECISION] - -end_date - [TIMESTAMP] - -<<<<<<< HEAD -executor_config - [BYTEA] - -external_executor_id - [VARCHAR(250)] - -hostname - [VARCHAR(1000)] - -job_id - [INTEGER] - -max_tries - [INTEGER] - -next_kwargs - [JSON] - -next_method - [VARCHAR(1000)] - -operator - [VARCHAR(1000)] - -pid - [INTEGER] - -pool - [VARCHAR(256)] - NOT NULL - -pool_slots - [INTEGER] - NOT NULL - -priority_weight - [INTEGER] - -queue - [VARCHAR(256)] - -queued_by_job_id - [INTEGER] - -queued_dttm - [TIMESTAMP] - -rendered_map_index - [VARCHAR(250)] - -start_date - [TIMESTAMP] - -state - [VARCHAR(20)] - -task_display_name - [VARCHAR(2000)] -======= -executor - [VARCHAR(1000)] - -executor_config - [BYTEA] - -external_executor_id - [VARCHAR(250)] - -hostname - [VARCHAR(1000)] - -job_id - [INTEGER] - -max_tries - [INTEGER] - -next_kwargs - [JSON] - -next_method - [VARCHAR(1000)] - -operator - [VARCHAR(1000)] - -pid - [INTEGER] - -pool - [VARCHAR(256)] - NOT NULL - -pool_slots - [INTEGER] - NOT NULL - -priority_weight - [INTEGER] - -queue - [VARCHAR(256)] - -queued_by_job_id - [INTEGER] - -queued_dttm - [TIMESTAMP] - -rendered_map_index - [VARCHAR(250)] - -start_date - [TIMESTAMP] - -state - [VARCHAR(20)] ->>>>>>> 7d40bcf99e... Repair migration after rebase - -trigger_id - [INTEGER] - -trigger_timeout - [TIMESTAMP] - -try_number - [INTEGER] - -unixname - [VARCHAR(1000)] - -updated_at - [TIMESTAMP] -<<<<<<< HEAD -======= -<<<<<<< HEAD - -task_instance - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -custom_operator_name - [VARCHAR(1000)] - -duration - [DOUBLE_PRECISION] - -end_date - [TIMESTAMP] - -executor_config - [BYTEA] - -external_executor_id - [VARCHAR(250)] - -hostname - [VARCHAR(1000)] - -job_id - [INTEGER] - -max_tries - [INTEGER] - -next_kwargs - [JSON] - -next_method - [VARCHAR(1000)] - -operator - [VARCHAR(1000)] - -pid - [INTEGER] - -pool - [VARCHAR(256)] - NOT NULL - -pool_slots - [INTEGER] - NOT NULL - -priority_weight - [INTEGER] - -queue - [VARCHAR(256)] - -queued_by_job_id - [INTEGER] - -queued_dttm - [TIMESTAMP] - -rendered_map_index - [VARCHAR(250)] - -start_date - [TIMESTAMP] - -state - [VARCHAR(20)] - -trigger_id - [INTEGER] - -trigger_timeout - [TIMESTAMP] - -try_number - [INTEGER] - -unixname - [VARCHAR(1000)] - -updated_at - [TIMESTAMP] -======= - -task_instance - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -custom_operator_name - [VARCHAR(1000)] - -duration - [DOUBLE_PRECISION] - -end_date - [TIMESTAMP] - -executor - [VARCHAR(1000)] - -executor_config - [BYTEA] - -external_executor_id - [VARCHAR(250)] - -hostname - [VARCHAR(1000)] - -job_id - [INTEGER] - -max_tries - [INTEGER] - -next_kwargs - [JSON] - -next_method - [VARCHAR(1000)] - -operator - [VARCHAR(1000)] - -pid - [INTEGER] - -pool - [VARCHAR(256)] - NOT NULL - -pool_slots - [INTEGER] - NOT NULL - -priority_weight - [INTEGER] - -queue - [VARCHAR(256)] - -queued_by_job_id - [INTEGER] - -queued_dttm - [TIMESTAMP] - -rendered_map_index - [VARCHAR(250)] - -start_date - [TIMESTAMP] - -state - [VARCHAR(20)] - -trigger_id - [INTEGER] - -trigger_timeout - [TIMESTAMP] - -try_number - [INTEGER] - -unixname - [VARCHAR(1000)] - -updated_at - [TIMESTAMP] ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase + +task_instance + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +custom_operator_name + [VARCHAR(1000)] + +duration + [DOUBLE_PRECISION] + +end_date + [TIMESTAMP] + +executor + [VARCHAR(1000)] + +executor_config + [BYTEA] + +external_executor_id + [VARCHAR(250)] + +hostname + [VARCHAR(1000)] + +job_id + [INTEGER] + +max_tries + [INTEGER] + +next_kwargs + [JSON] + +next_method + [VARCHAR(1000)] + +operator + [VARCHAR(1000)] + +pid + [INTEGER] + +pool + [VARCHAR(256)] + NOT NULL + +pool_slots + [INTEGER] + NOT NULL + +priority_weight + [INTEGER] + +queue + [VARCHAR(256)] + +queued_by_job_id + [INTEGER] + +queued_dttm + [TIMESTAMP] + +rendered_map_index + [VARCHAR(250)] + +start_date + [TIMESTAMP] + +state + [VARCHAR(20)] + +task_display_name + [VARCHAR(2000)] + +trigger_id + [INTEGER] + +trigger_timeout + [TIMESTAMP] + +try_number + [INTEGER] + +unixname + [VARCHAR(1000)] + +updated_at + [TIMESTAMP] dag_run--task_instance -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 dag_run--task_instance -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_reschedule -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase - -task_reschedule - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -duration - [INTEGER] - NOT NULL - -end_date - [TIMESTAMP] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -reschedule_date - [TIMESTAMP] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -start_date - [TIMESTAMP] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -try_number - [INTEGER] - NOT NULL -<<<<<<< HEAD -======= -<<<<<<< HEAD - -task_reschedule - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -duration - [INTEGER] - NOT NULL - -end_date - [TIMESTAMP] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -reschedule_date - [TIMESTAMP] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -start_date - [TIMESTAMP] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -try_number - [INTEGER] - NOT NULL -======= - -task_reschedule - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -duration - [INTEGER] - NOT NULL - -end_date - [TIMESTAMP] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -reschedule_date - [TIMESTAMP] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -start_date - [TIMESTAMP] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -try_number - [INTEGER] - NOT NULL ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase + +task_reschedule + +id + [INTEGER] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +duration + [INTEGER] + NOT NULL + +end_date + [TIMESTAMP] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +reschedule_date + [TIMESTAMP] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +start_date + [TIMESTAMP] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +try_number + [INTEGER] + NOT NULL dag_run--task_reschedule -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 dag_run--task_reschedule -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 task_instance--task_instance_note -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance--task_instance_note -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance--task_instance_note -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance--task_instance_note -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance--task_reschedule -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 task_instance--task_reschedule -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 task_instance--task_reschedule -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 task_instance--task_reschedule -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 task_fail -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase - -task_fail - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -duration - [INTEGER] - -end_date - [TIMESTAMP] - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -start_date - [TIMESTAMP] - -task_id - [VARCHAR(250)] - NOT NULL -<<<<<<< HEAD -======= -<<<<<<< HEAD - -task_fail - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -duration - [INTEGER] - -end_date - [TIMESTAMP] - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -start_date - [TIMESTAMP] - -task_id - [VARCHAR(250)] - NOT NULL -======= - -task_fail - -id - [INTEGER] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -duration - [INTEGER] - -end_date - [TIMESTAMP] - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -start_date - [TIMESTAMP] - -task_id - [VARCHAR(250)] - NOT NULL ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase + +task_fail + +id + [INTEGER] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +duration + [INTEGER] + +end_date + [TIMESTAMP] + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +start_date + [TIMESTAMP] + +task_id + [VARCHAR(250)] + NOT NULL task_instance--task_fail -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 task_instance--task_fail -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 task_instance--task_fail -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 task_instance--task_fail -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 task_map -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase - -task_map - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -keys - [JSON] - -length - [INTEGER] - NOT NULL -<<<<<<< HEAD -======= -<<<<<<< HEAD - -task_map - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -keys - [JSON] - -length - [INTEGER] - NOT NULL -======= - -task_map - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -keys - [JSON] - -length - [INTEGER] - NOT NULL ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase + +task_map + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +keys + [JSON] + +length + [INTEGER] + NOT NULL task_instance--task_map -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance--task_map -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance--task_map -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance--task_map -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 xcom -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase - -xcom - -dag_run_id - [INTEGER] - NOT NULL - -key - [VARCHAR(512)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -timestamp - [TIMESTAMP] - NOT NULL - -value - [BYTEA] -<<<<<<< HEAD -======= -<<<<<<< HEAD - -xcom - -dag_run_id - [INTEGER] - NOT NULL - -key - [VARCHAR(512)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -timestamp - [TIMESTAMP] - NOT NULL - -value - [BYTEA] -======= - -xcom - -dag_run_id - [INTEGER] - NOT NULL - -key - [VARCHAR(512)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -dag_id - [VARCHAR(250)] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -timestamp - [TIMESTAMP] - NOT NULL - -value - [BYTEA] ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase + +xcom + +dag_run_id + [INTEGER] + NOT NULL + +key + [VARCHAR(512)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +dag_id + [VARCHAR(250)] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +timestamp + [TIMESTAMP] + NOT NULL + +value + [BYTEA] task_instance--xcom -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase -<<<<<<< HEAD - -0..N -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB -<<<<<<< HEAD ->>>>>>> 5b2489abf4... Add executor field to the DB -======= -======= - -0..N -1 ->>>>>>> a25a99d7cc... Repair migration after rebase ->>>>>>> 7d40bcf99e... Repair migration after rebase -======= - -0..N -1 ->>>>>>> 946158562e... Rebase yet another migration overtop + +0..N +1 task_instance--xcom -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase -<<<<<<< HEAD - -1 -1 -======= - -0..N -1 ->>>>>>> 3ce982cef8... Add executor field to the DB -<<<<<<< HEAD ->>>>>>> 5b2489abf4... Add executor field to the DB -======= -======= - -1 -1 ->>>>>>> a25a99d7cc... Repair migration after rebase ->>>>>>> 7d40bcf99e... Repair migration after rebase -======= - -1 -1 ->>>>>>> 946158562e... Rebase yet another migration overtop + +1 +1 task_instance--xcom -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -0..N -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +1 task_instance--xcom -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 rendered_task_instance_fields -<<<<<<< HEAD -<<<<<<< HEAD -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase - -rendered_task_instance_fields - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -k8s_pod_yaml - [JSON] - -rendered_fields - [JSON] - NOT NULL -<<<<<<< HEAD -======= -<<<<<<< HEAD - -rendered_task_instance_fields - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -k8s_pod_yaml - [JSON] - -rendered_fields - [JSON] - NOT NULL -======= - -rendered_task_instance_fields - -dag_id - [VARCHAR(250)] - NOT NULL - -map_index - [INTEGER] - NOT NULL - -run_id - [VARCHAR(250)] - NOT NULL - -task_id - [VARCHAR(250)] - NOT NULL - -k8s_pod_yaml - [JSON] - -rendered_fields - [JSON] - NOT NULL ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase + +rendered_task_instance_fields + +dag_id + [VARCHAR(250)] + NOT NULL + +map_index + [INTEGER] + NOT NULL + +run_id + [VARCHAR(250)] + NOT NULL + +task_id + [VARCHAR(250)] + NOT NULL + +k8s_pod_yaml + [JSON] + +rendered_fields + [JSON] + NOT NULL task_instance--rendered_task_instance_fields -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance--rendered_task_instance_fields -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance--rendered_task_instance_fields -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 task_instance--rendered_task_instance_fields -<<<<<<< HEAD -<<<<<<< HEAD - -1 -1 -======= -<<<<<<< HEAD - -1 -1 -======= - -1 -1 ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -1 -1 ->>>>>>> 7d40bcf99e... Repair migration after rebase + +1 +1 @@ -2845,25 +1485,25 @@ ab_permission_view_role - -ab_permission_view_role - -id - [INTEGER] - NOT NULL - -permission_view_id - [INTEGER] - -role_id - [INTEGER] + +ab_permission_view_role + +id + [INTEGER] + NOT NULL + +permission_view_id + [INTEGER] + +role_id + [INTEGER] ab_permission_view--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} @@ -2903,16 +1543,16 @@ ab_role--ab_user_role - -0..N -{0,1} + +0..N +{0,1} ab_role--ab_permission_view_role - -0..N -{0,1} + +0..N +{0,1} @@ -2951,9 +1591,9 @@ dataset_event--dagrun_dataset_event - -1 -1 + +1 +1 @@ -2983,27 +1623,9 @@ trigger--task_instance -<<<<<<< HEAD -<<<<<<< HEAD - -0..N -{0,1} -======= -<<<<<<< HEAD - -0..N -{0,1} -======= - -0..N -{0,1} ->>>>>>> 3ce982cef8... Add executor field to the DB ->>>>>>> 5b2489abf4... Add executor field to the DB -======= - -0..N -{0,1} ->>>>>>> 7d40bcf99e... Repair migration after rebase + +0..N +{0,1} diff --git a/docs/apache-airflow/migrations-ref.rst b/docs/apache-airflow/migrations-ref.rst index 7dc8da994b90d..ac509b0c03b49 100644 --- a/docs/apache-airflow/migrations-ref.rst +++ b/docs/apache-airflow/migrations-ref.rst @@ -39,23 +39,11 @@ Here's the list of all the Database Migrations that are executed via when you ru +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=================================+===================+===================+==============================================================+ -<<<<<<< HEAD -<<<<<<< HEAD -<<<<<<< HEAD -| ``1949afb29106`` (head) | ``ee1467d4aa35`` | ``2.9.0`` | update trigger kwargs type | +| ``677fdbb7fc54`` (head) | ``1949afb29106`` | ``2.10.0`` | add new executor field to db | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``ee1467d4aa35`` | ``b4078ac230a1`` | ``2.9.0`` | add display name for dag and task instance | +| ``1949afb29106`` | ``ee1467d4aa35`` | ``2.9.0`` | update trigger kwargs type | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ -| ``b4078ac230a1`` | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | -======= -======= ->>>>>>> 7d40bcf99e... Repair migration after rebase -<<<<<<< HEAD -| ``b4078ac230a1`` (head) | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | ->>>>>>> 5b2489abf4... Add executor field to the DB -======= -| ``677fdbb7fc54`` (head) | ``b4078ac230a1`` | ``2.10.0`` | add new executor field to db | ->>>>>>> 946158562e... Rebase yet another migration overtop +| ``ee1467d4aa35`` | ``b4078ac230a1`` | ``2.9.0`` | add display name for dag and task instance | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+ | ``b4078ac230a1`` | ``8e1c784a4fc7`` | ``2.9.0`` | Change value column type to longblob in xcom table for mysql | +---------------------------------+-------------------+-------------------+--------------------------------------------------------------+