From eedb9f2c8416ed71287252b66b58b311e434f534 Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Fri, 9 Jan 2026 14:53:48 -0800 Subject: [PATCH 01/14] Port migration and models over --- ...098_3_2_0_ui_improvements_for_deadlines.py | 636 ++++++++++++++++++ airflow-core/src/airflow/models/__init__.py | 3 + airflow-core/src/airflow/models/dag.py | 9 +- airflow-core/src/airflow/models/dagrun.py | 9 +- airflow-core/src/airflow/models/deadline.py | 22 +- .../src/airflow/models/deadline_alert.py | 105 +++ airflow-core/tests/unit/models/test_dag.py | 24 +- airflow-core/tests/unit/models/test_dagrun.py | 170 +++-- .../tests/unit/models/test_deadline.py | 54 +- .../tests/unit/models/test_deadline_alert.py | 184 +++++ 10 files changed, 1117 insertions(+), 99 deletions(-) create mode 100644 airflow-core/src/airflow/migrations/versions/0098_3_2_0_ui_improvements_for_deadlines.py create mode 100644 airflow-core/src/airflow/models/deadline_alert.py create mode 100644 airflow-core/tests/unit/models/test_deadline_alert.py diff --git a/airflow-core/src/airflow/migrations/versions/0098_3_2_0_ui_improvements_for_deadlines.py b/airflow-core/src/airflow/migrations/versions/0098_3_2_0_ui_improvements_for_deadlines.py new file mode 100644 index 0000000000000..d6c212c355ddd --- /dev/null +++ b/airflow-core/src/airflow/migrations/versions/0098_3_2_0_ui_improvements_for_deadlines.py @@ -0,0 +1,636 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Add required fields to enable UI integrations for the Deadline Alerts feature. + +This migration creates the deadline_alert table to store DeadlineAlert definitions +and migrates existing Deadline Alert data from the serialized_dag JSON structure +into the new normalized table structure. + +Revision ID: 55297ae24532 +Revises: 0b112f49112d +Create Date: 2025-10-17 16:04:55.016272 +""" + +from __future__ import annotations + +import json +import zlib +from collections import defaultdict +from typing import TYPE_CHECKING + +import sqlalchemy as sa +import uuid6 +from alembic import context, op +from sqlalchemy_utils import UUIDType + +from airflow._shared.timezones import timezone +from airflow.configuration import conf +from airflow.serialization.enums import Encoding +from airflow.utils.sqlalchemy import UtcDateTime + +if TYPE_CHECKING: + from typing import Any + + from sqlalchemy.engine import Connection + + ErrorDict = dict[str, list[str]] + +revision = "55297ae24532" +down_revision = "0b112f49112d" +branch_labels = None +depends_on = None +airflow_version = "3.2.0" + + +CALLBACK_KEY = "callback_def" +DAG_KEY = "dag" +DEADLINE_KEY = "deadline" +INTERVAL_KEY = "interval" +REFERENCE_KEY = "reference" +DEADLINE_ALERT_REQUIRED_FIELDS = {REFERENCE_KEY, CALLBACK_KEY, INTERVAL_KEY} +DEFAULT_BATCH_SIZE = 1000 +ENCODING_TYPE = "deadline_alert" + + +def upgrade() -> None: + """Make changes to enable adding DeadlineAlerts to the UI.""" + # TODO: We may finally have come up with a better naming convention. For ease of migration, + # we are going to keep deadline_alert here to match the model's name, but in the near future + # when this migration work is done we should deprecate the name DeadlineAlert (and all related + # classes, tables, etc) and replace it with DeadlineDefinition. Then we will have the + # user-provided DeadlineDefinition, and the actual instance of a Definition is (still) the Deadline. + # This feels more intuitive than DeadlineAlert defining the Deadline. + + op.create_table( + "deadline_alert", + sa.Column("id", UUIDType(binary=False), default=uuid6.uuid7), + sa.Column("created_at", UtcDateTime, nullable=False), + sa.Column("serialized_dag_id", UUIDType(binary=False), nullable=False), + sa.Column("name", sa.String(250), nullable=True), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("reference", sa.JSON(), nullable=False), + sa.Column("interval", sa.Float(), nullable=False), + sa.Column("callback_def", sa.JSON(), nullable=False), + sa.PrimaryKeyConstraint("id", name=op.f("deadline_alert_pkey")), + ) + + conn = op.get_bind() + dialect_name = conn.dialect.name + + if dialect_name == "sqlite": + conn.execute(sa.text("PRAGMA foreign_keys=OFF")) + + with op.batch_alter_table("deadline", schema=None) as batch_op: + batch_op.add_column(sa.Column("deadline_alert_id", UUIDType(binary=False), nullable=True)) + batch_op.add_column(sa.Column("created_at", UtcDateTime, nullable=True)) + batch_op.add_column(sa.Column("last_updated_at", UtcDateTime, nullable=True)) + batch_op.create_foreign_key( + batch_op.f("deadline_deadline_alert_id_fkey"), + "deadline_alert", + ["deadline_alert_id"], + ["id"], + ondelete="SET NULL", + ) + + # For migration/backcompat purposes if no timestamp is there from the migration, use now() + # then lock the columns down so all new entries require the timestamps to be provided. + now = timezone.utcnow() + conn.execute( + sa.text(""" + UPDATE deadline + SET created_at = :now, last_updated_at = :now + WHERE created_at IS NULL OR last_updated_at IS NULL + """), + {"now": now}, + ) + + with op.batch_alter_table("deadline", schema=None) as batch_op: + batch_op.alter_column("created_at", existing_type=UtcDateTime, nullable=False) + batch_op.alter_column("last_updated_at", existing_type=UtcDateTime, nullable=False) + + with op.batch_alter_table("deadline_alert", schema=None) as batch_op: + batch_op.create_foreign_key( + batch_op.f("deadline_alert_serialized_dag_id_fkey"), + "serialized_dag", + ["serialized_dag_id"], + ["id"], + ondelete="CASCADE", + ) + + if dialect_name == "sqlite": + conn.execute(sa.text("PRAGMA foreign_keys=ON")) + + migrate_existing_deadline_alert_data_from_serialized_dag() + + +def downgrade() -> None: + """Remove changes that were added to enable adding DeadlineAlerts to the UI.""" + migrate_deadline_alert_data_back_to_serialized_dag() + + conn = op.get_bind() + dialect_name = conn.dialect.name + + if dialect_name == "sqlite": + conn.execute(sa.text("PRAGMA foreign_keys=OFF")) + + with op.batch_alter_table("deadline", schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f("deadline_deadline_alert_id_fkey"), type_="foreignkey") + batch_op.drop_column("deadline_alert_id") + batch_op.drop_column("last_updated_at") + batch_op.drop_column("created_at") + + with op.batch_alter_table("deadline_alert", schema=None) as batch_op: + batch_op.drop_constraint(batch_op.f("deadline_alert_serialized_dag_id_fkey"), type_="foreignkey") + + if dialect_name == "sqlite": + conn.execute(sa.text("PRAGMA foreign_keys=ON")) + + op.drop_table("deadline_alert") + + +def get_dag_data(data: dict[str, Any] | str | None, data_compressed: bytes | None) -> dict[str, Any]: + """ + Extract and decompress DAG data regardless of storage format. + + Returns the parsed JSON data, handling both compressed and uncompressed formats. + + :param data: The uncompressed DAG data as dict or JSON string. + :param data_compressed: The compressed DAG data as bytes. + """ + if data_compressed: + decompressed = zlib.decompress(data_compressed) + return json.loads(decompressed) + return data if isinstance(data, dict) else json.loads(data) + + +def update_dag_deadline_field( + conn: Connection, + serialized_dag_id: str, + deadline_data: list[str] | list[dict[str, Any]], + dialect: str, +) -> None: + """ + Update the deadline field in serialized_dag using the appropriate format. + + Checks the existing row to determine whether to use compressed or uncompressed format, + and uses dialect-specific JSON operations when available. + + :param conn: SQLAlchemy database connection. + :param serialized_dag_id: The serialized_dag.id identifier. + :param deadline_data: List of deadline alert UUIDs (upgrade) or deadline objects (downgrade). + :param dialect: Database dialect name (e.g., 'postgresql', 'mysql', 'sqlite'). + """ + check_compressed = conn.execute( + sa.text("SELECT data_compressed FROM serialized_dag WHERE id = :serialized_dag_id"), + {"serialized_dag_id": serialized_dag_id}, + ).fetchone() + + if check_compressed and check_compressed.data_compressed: + decompressed = zlib.decompress(check_compressed.data_compressed) + dag_data = json.loads(decompressed) + dag_data[DAG_KEY][DEADLINE_KEY] = deadline_data + new_compressed = zlib.compress(json.dumps(dag_data).encode("utf-8")) + conn.execute( + sa.text("UPDATE serialized_dag SET data_compressed = :data WHERE id = :serialized_dag_id"), + {"data": new_compressed, "serialized_dag_id": serialized_dag_id}, + ) + elif dialect == "postgresql": + conn.execute( + sa.text(""" + UPDATE serialized_dag + SET data = jsonb_set( + data::jsonb, + '{dag,deadline}', + CAST(:deadline_data AS jsonb) + )::json + WHERE id = :serialized_dag_id + """), + {"serialized_dag_id": serialized_dag_id, "deadline_data": json.dumps(deadline_data)}, + ) + elif dialect == "mysql": + conn.execute( + sa.text(""" + UPDATE serialized_dag + SET data = JSON_SET( + data, + '$.dag.deadline', + CAST(:deadline_data AS JSON) + ) + WHERE id = :serialized_dag_id + """), + {"serialized_dag_id": serialized_dag_id, "deadline_data": json.dumps(deadline_data)}, + ) + else: + result = conn.execute( + sa.text("SELECT data FROM serialized_dag WHERE id = :serialized_dag_id"), + {"serialized_dag_id": serialized_dag_id}, + ).fetchone() + + if result and result.data: + dag_data = json.loads(result.data) if isinstance(result.data, str) else result.data + dag_data[DAG_KEY][DEADLINE_KEY] = deadline_data + + conn.execute( + sa.text("UPDATE serialized_dag SET data = :data WHERE id = :serialized_dag_id"), + {"data": json.dumps(dag_data), "serialized_dag_id": serialized_dag_id}, + ) + + +def validate_written_data( + conn: Connection, + deadline_alert_id: str, + expected_reference: str, + expected_interval: float, + expected_callback: str, +) -> bool: + """ + Read back the inserted data and validate that it matches what we expect. + + This provides an extra safety check for data integrity during migration. + + :param conn: SQLAlchemy database connection. + :param deadline_alert_id: The UUID of the deadline alert to validate. + :param expected_reference: Expected JSON string of the reference field. + :param expected_interval: Expected interval value as float. + :param expected_callback: Expected JSON string of the callback field. + """ + # TODO: Is this overkill? Maybe... Consider adding a config option to + # disable validation for large deployments where performance is critical?? + + validation_result = conn.execute( + sa.text(""" + SELECT reference, interval, callback_def + FROM deadline_alert + WHERE id = :alert_id + """), + {"alert_id": deadline_alert_id}, + ).fetchone() + + if not validation_result: + print(f"ERROR: Failed to read back deadline_alert for DeadlineAlert {deadline_alert_id}") + return False + + checks = [ + (REFERENCE_KEY, json.dumps(validation_result.reference, sort_keys=True), expected_reference), + (INTERVAL_KEY, validation_result.interval, expected_interval), + (CALLBACK_KEY, json.dumps(validation_result.callback_def, sort_keys=True), expected_callback), + ] + + for name, actual, expected in checks: + if actual != expected: + print(f"ERROR: Written {name} does not match expected! Written: {actual}, Expected: {expected}") + return False + + return True + + +def report_errors(errors: ErrorDict, operation: str = "migration") -> None: + if errors: + print(f"{len(errors)} Dags encountered errors: ") + for dag_id, error in errors.items(): + print(f" {dag_id}: {'; '.join(error)}") + else: + print(f"No Dags encountered errors during {operation}.") + + +def migrate_existing_deadline_alert_data_from_serialized_dag() -> None: + """Extract DeadlineAlert data from serialized Dag data and populate deadline_alert table.""" + if context.is_offline_mode(): + print( + """ + ------------ + -- WARNING: Unable to migrate DeadlineAlert data while in offline mode! + -- The deadline_alert table will remain empty in offline mode. + -- Run the migration in online mode to populate the deadline_alert table. + ------------ + """ + ) + return + + BATCH_SIZE = conf.getint("database", "migration_batch_size", fallback=DEFAULT_BATCH_SIZE) + + processed_dags: list[str] = [] + dags_with_deadlines: set[str] = set() + migrated_alerts_count: int = 0 + dags_with_errors: ErrorDict = defaultdict(list) + batch_num = 0 + last_dag_id = "" + + conn = op.get_bind() + dialect = conn.dialect.name + + total_dags = conn.execute( + sa.text("SELECT COUNT(*) FROM serialized_dag WHERE data IS NOT NULL OR data_compressed IS NOT NULL") + ).scalar() + total_batches = (total_dags + BATCH_SIZE - 1) // BATCH_SIZE + + print(f"Using migration_batch_size of {BATCH_SIZE} as set in Airflow configuration.") + print(f"Starting migration of {total_dags} Dags in {total_batches} batches.\n") + + while True: + batch_num += 1 + + result = conn.execute( + sa.text(""" + SELECT id, dag_id, data, data_compressed, created_at + FROM serialized_dag + WHERE (data IS NOT NULL OR data_compressed IS NOT NULL) + AND dag_id > :last_dag_id + ORDER BY dag_id + LIMIT :batch_size + """), + {"last_dag_id": last_dag_id, "batch_size": BATCH_SIZE}, + ) + + batch_results = list(result) + if not batch_results: + break + + print(f"Processing batch {batch_num}...") + + for serialized_dag_id, dag_id, data, data_compressed, created_at in batch_results: + processed_dags.append(dag_id) + last_dag_id = dag_id + + # Create a savepoint for this Dag to allow rollback on error. + savepoint = conn.begin_nested() + + try: + dag_data = get_dag_data(data, data_compressed) + + if dag_deadline := dag_data[DAG_KEY][DEADLINE_KEY]: + dags_with_deadlines.add(dag_id) + deadline_alerts = dag_deadline if isinstance(dag_deadline, list) else [dag_deadline] + + migrated_alert_ids = [] + + for serialized_alert in deadline_alerts: + if isinstance(serialized_alert, dict): + try: + alert_data = serialized_alert.get(Encoding.VAR, serialized_alert) + + if not DEADLINE_ALERT_REQUIRED_FIELDS.issubset(alert_data): + dags_with_errors[dag_id].append( + f"Invalid DeadlineAlert structure: {serialized_alert}" + ) + continue + + reference_data = json.dumps(alert_data[REFERENCE_KEY], sort_keys=True) + interval_data = float(alert_data.get(INTERVAL_KEY)) + callback_data = json.dumps(alert_data[CALLBACK_KEY], sort_keys=True) + deadline_alert_id = str(uuid6.uuid7()) + + conn.execute( + sa.text(""" + INSERT INTO deadline_alert ( + id, + created_at, + serialized_dag_id, + reference, + interval, + callback_def, + name, + description) + VALUES ( + :id, + :created_at, + :serialized_dag_id, + :reference, + :interval, + :callback_def, + NULL, + NULL) + """), + { + "id": deadline_alert_id, + "created_at": created_at or timezone.utcnow(), + "serialized_dag_id": serialized_dag_id, + "reference": reference_data, + "interval": interval_data, + "callback_def": callback_data, + }, + ) + + if not validate_written_data( + conn, deadline_alert_id, reference_data, interval_data, callback_data + ): + dags_with_errors[dag_id].append( + f"Invalid DeadlineAlert data: {serialized_alert}" + ) + continue + + migrated_alert_ids.append(deadline_alert_id) + migrated_alerts_count += 1 + + conn.execute( + sa.text(""" + UPDATE deadline + SET deadline_alert_id = :alert_id + WHERE dagrun_id IN ( + SELECT dr.id + FROM dag_run dr + JOIN serialized_dag sd ON dr.dag_id = sd.dag_id + WHERE sd.id = :serialized_dag_id) + AND deadline_alert_id IS NULL + """), + {"alert_id": deadline_alert_id, "serialized_dag_id": serialized_dag_id}, + ) + except Exception as e: + dags_with_errors[dag_id].append(f"Failed to process {serialized_alert}: {e}") + continue + + if migrated_alert_ids: + uuid_strings = [str(uuid_id) for uuid_id in migrated_alert_ids] + update_dag_deadline_field(conn, serialized_dag_id, uuid_strings, dialect) + + # Recalculate and update the dag_hash after modifying the deadline data to ensure + # it matches what write_dag() will compute later and avoid re-serialization. + updated_result = conn.execute( + sa.text( + "SELECT data, data_compressed " + "FROM serialized_dag " + "WHERE id = :serialized_dag_id" + ), + {"serialized_dag_id": serialized_dag_id}, + ).fetchone() + + if updated_result: + updated_dag_data = get_dag_data( + updated_result.data, updated_result.data_compressed + ) + # Import here to avoid a circular dependency issue + from airflow.models.serialized_dag import SerializedDagModel + + new_hash = SerializedDagModel.hash(updated_dag_data) + + conn.execute( + sa.text( + "UPDATE serialized_dag " + "SET dag_hash = :new_hash " + "WHERE id = :serialized_dag_id" + ), + {"new_hash": new_hash, "serialized_dag_id": serialized_dag_id}, + ) + + # Commit the savepoint if everything succeeded for this Dag. + savepoint.commit() + + except (json.JSONDecodeError, KeyError, TypeError) as e: + dags_with_errors[dag_id].append(f"Could not process serialized Dag: {e}") + savepoint.rollback() + + print(f"Batch {batch_num} of {total_batches} complete.") + + print( + f"\nProcessed {len(processed_dags)} serialized_dag records ({len(set(processed_dags))} " + f"unique Dags), {len(dags_with_deadlines)} had DeadlineAlerts." + ) + print(f"Migrated {migrated_alerts_count} DeadlineAlert configurations.") + report_errors(dags_with_errors, "migration") + + +def migrate_deadline_alert_data_back_to_serialized_dag() -> None: + """Restore DeadlineAlert data from deadline_alert table back to serialized_dag.""" + from alembic import context + + if context.is_offline_mode(): + print( + """ + ------------ + -- WARNING: Unable to restore DeadlineAlert data while in offline mode! + -- The downgrade will skip data restoration in offline mode. + -- Run the migration in online mode to restore the deadline_alert data. + ------------ + """ + ) + return + + from airflow.configuration import conf + from airflow.serialization.enums import Encoding + + BATCH_SIZE = conf.getint("database", "migration_batch_size", fallback=DEFAULT_BATCH_SIZE) + + processed_dags: list[str] = [] + dags_with_deadlines: set[str] = set() + restored_alerts_count: int = 0 + dags_with_errors: ErrorDict = defaultdict(list) + batch_num = 0 + last_dag_id = "" + + conn = op.get_bind() + dialect = conn.dialect.name + + # Count all dags - we'll filter in the loop for those with deadline data + total_dags = conn.execute( + sa.text("SELECT COUNT(*) FROM serialized_dag WHERE data IS NOT NULL OR data_compressed IS NOT NULL") + ).scalar() + + total_batches = (total_dags + BATCH_SIZE - 1) // BATCH_SIZE + + print(f"Using migration_batch_size of {BATCH_SIZE} as set in Airflow configuration.") + print(f"Starting downgrade of {total_dags} Dags with DeadlineAlerts in {total_batches} batches.\n") + + while True: + batch_num += 1 + + result = conn.execute( + sa.text(""" + SELECT id, dag_id, data, data_compressed + FROM serialized_dag + WHERE (data IS NOT NULL OR data_compressed IS NOT NULL) + AND dag_id > :last_dag_id + ORDER BY dag_id + LIMIT :batch_size + """), + {"last_dag_id": last_dag_id, "batch_size": BATCH_SIZE}, + ) + + batch_results = list(result) + if not batch_results: + break + print(f"Processing batch {batch_num}...") + + for serialized_dag_id, dag_id, data, data_compressed in batch_results: + processed_dags.append(dag_id) + last_dag_id = dag_id + + # Create a savepoint for this Dag to allow rollback on error. + savepoint = conn.begin_nested() + + try: + dag_data = get_dag_data(data, data_compressed) + deadline_uuids = dag_data[DAG_KEY][DEADLINE_KEY] + + if not isinstance(deadline_uuids, list) or not deadline_uuids: + continue + + if not all(isinstance(uuid_val, str) for uuid_val in deadline_uuids): + print(f"WARNING: Dag {dag_id} has non-string deadline values, skipping") + continue + + dags_with_deadlines.add(dag_id) + restored_deadline_objects = [] + + alert_result = conn.execute( + sa.text(""" + SELECT reference, interval, callback_def + FROM deadline_alert + WHERE serialized_dag_id = :serialized_dag_id + """), + {"serialized_dag_id": serialized_dag_id}, + ).fetchall() + + if not alert_result: + dags_with_errors[dag_id].append( + f"Could not find deadline_alert for serialized_dag {serialized_dag_id}" + ) + continue + + for alert in alert_result: + deadline_object = { + Encoding.TYPE: ENCODING_TYPE, + Encoding.VAR: { + REFERENCE_KEY: alert.reference, + INTERVAL_KEY: float(alert.interval), + CALLBACK_KEY: alert.callback_def, + }, + } + restored_deadline_objects.append(deadline_object) + restored_alerts_count += 1 + + # Replace the UUID array with the restored objects. + if restored_deadline_objects: + update_dag_deadline_field(conn, serialized_dag_id, restored_deadline_objects, dialect) + + # Commit the savepoint if everything succeeded for this Dag. + savepoint.commit() + + except Exception as e: + dags_with_errors[dag_id].append(f"Could not restore deadline: {e}") + savepoint.rollback() + + print(f"Batch {batch_num} of {total_batches} complete.") + + print( + f"\nProcessed {len(processed_dags)} serialized_dag records ({len(set(processed_dags))} " + f"unique Dags), {len(dags_with_deadlines)} had DeadlineAlerts." + ) + print(f"Restored {restored_alerts_count} DeadlineAlert configurations to original format.") + report_errors(dags_with_errors, "downgrade") diff --git a/airflow-core/src/airflow/models/__init__.py b/airflow-core/src/airflow/models/__init__.py index 39a5a7350759e..3dc1c050bae0b 100644 --- a/airflow-core/src/airflow/models/__init__.py +++ b/airflow-core/src/airflow/models/__init__.py @@ -67,6 +67,7 @@ def import_all_models(): import airflow.models.dagbag import airflow.models.dagbundle import airflow.models.dagwarning + import airflow.models.deadline_alert import airflow.models.errors import airflow.models.serialized_dag import airflow.models.taskinstancehistory @@ -107,6 +108,7 @@ def __getattr__(name): "DagWarning": "airflow.models.dagwarning", "DbCallbackRequest": "airflow.models.db_callback_request", "Deadline": "airflow.models.deadline", + "DeadlineAlert": "airflow.models.deadline_alert", "Log": "airflow.models.log", "HITLDetail": "airflow.models.hitl", "MappedOperator": "airflow.sdk.definitions.mappedoperator", @@ -135,6 +137,7 @@ def __getattr__(name): from airflow.models.dagwarning import DagWarning from airflow.models.db_callback_request import DbCallbackRequest from airflow.models.deadline import Deadline + from airflow.models.deadline_alert import DeadlineAlert from airflow.models.log import Log from airflow.models.pool import Pool from airflow.models.renderedtifields import RenderedTaskInstanceFields diff --git a/airflow-core/src/airflow/models/dag.py b/airflow-core/src/airflow/models/dag.py index a913372d816e2..d7a1ba95e572b 100644 --- a/airflow-core/src/airflow/models/dag.py +++ b/airflow-core/src/airflow/models/dag.py @@ -54,7 +54,6 @@ from airflow.models.dagbundle import DagBundleModel from airflow.models.dagrun import DagRun from airflow.models.team import Team -from airflow.sdk.definitions.deadline import DeadlineAlert from airflow.serialization.definitions.assets import SerializedAssetUniqueKey from airflow.settings import json from airflow.timetables.base import DataInterval, Timetable @@ -477,12 +476,8 @@ def next_dagrun_data_interval(self, value: tuple[datetime, datetime] | None) -> @property def deadline(self): - """Get the deserialized deadline alert.""" - if self._deadline is None: - return None - if isinstance(self._deadline, list): - return [DeadlineAlert.deserialize_deadline_alert(item) for item in self._deadline] - return DeadlineAlert.deserialize_deadline_alert(self._deadline) + """Get deadline alert UUID references.""" + return self._deadline @deadline.setter def deadline(self, value): diff --git a/airflow-core/src/airflow/models/dagrun.py b/airflow-core/src/airflow/models/dagrun.py index 3f3bae8b66b9a..5b2b0578c1918 100644 --- a/airflow-core/src/airflow/models/dagrun.py +++ b/airflow-core/src/airflow/models/dagrun.py @@ -66,6 +66,7 @@ from airflow.models import Deadline, Log from airflow.models.backfill import Backfill from airflow.models.base import Base, StringID +from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel from airflow.models.taskinstance import TaskInstance as TI from airflow.models.taskinstancehistory import TaskInstanceHistory as TIH from airflow.models.tasklog import LogTemplate @@ -1265,9 +1266,13 @@ def recalculate(self) -> _UnfinishedStates: if dag.deadline: # The dagrun has succeeded. If there were any Deadlines for it which were not breached, they are no longer needed. + deadline_alerts = [ + DeadlineAlertModel.get_by_id(alert_id, session) for alert_id in dag.deadline + ] + if any( - isinstance(d.reference, DeadlineReference.TYPES.DAGRUN) - for d in cast("list", dag.deadline) + deadline_alert.reference_class in DeadlineReference.TYPES.DAGRUN + for deadline_alert in deadline_alerts ): Deadline.prune_deadlines(session=session, conditions={DagRun.id: self.id}) diff --git a/airflow-core/src/airflow/models/deadline.py b/airflow-core/src/airflow/models/deadline.py index 1613703103549..53dba340612b6 100644 --- a/airflow-core/src/airflow/models/deadline.py +++ b/airflow-core/src/airflow/models/deadline.py @@ -40,6 +40,9 @@ from sqlalchemy.orm import Session from sqlalchemy.sql import ColumnElement + from airflow.models.callback import CallbackDefinitionProtocol + from airflow.models.deadline_alert import DeadlineAlert + logger = logging.getLogger(__name__) @@ -79,6 +82,10 @@ class Deadline(Base): __tablename__ = "deadline" id: Mapped[str] = mapped_column(UUIDType(binary=False), primary_key=True, default=uuid6.uuid7) + created_at: Mapped[datetime] = mapped_column(UtcDateTime, nullable=False, default=timezone.utcnow) + last_updated_at: Mapped[datetime] = mapped_column( + UtcDateTime, nullable=False, default=timezone.utcnow, onupdate=timezone.utcnow + ) # If the Deadline Alert is for a DAG, store the DAG run ID from the dag_run. dagrun_id: Mapped[int | None] = mapped_column( @@ -98,6 +105,12 @@ class Deadline(Base): ) callback = relationship("Callback", uselist=False, cascade="all, delete-orphan", single_parent=True) + # The DeadlineAlert that generated this deadline + deadline_alert_id: Mapped[str | None] = mapped_column( + UUIDType(binary=False), ForeignKey("deadline_alert.id", ondelete="SET NULL"), nullable=True + ) + deadline_alert: Mapped[DeadlineAlert | None] = relationship("DeadlineAlert") + __table_args__ = (Index("deadline_missed_deadline_time_idx", missed, deadline_time, unique=False),) def __init__( @@ -105,6 +118,7 @@ def __init__( deadline_time: datetime, callback: CallbackDefinitionProtocol, dagrun_id: int, + deadline_alert_id: str | None, dag_id: str | None = None, ): super().__init__() @@ -114,6 +128,7 @@ def __init__( self.callback = Callback.create_from_sdk_def( callback_def=callback, prefix=CALLBACK_METRICS_PREFIX, dag_id=dag_id ) + self.deadline_alert_id = deadline_alert_id def __repr__(self): def _determine_resource() -> tuple[str, str]: @@ -127,8 +142,11 @@ def _determine_resource() -> tuple[str, str]: resource_type, resource_details = _determine_resource() return ( - f"[{resource_type} Deadline] {resource_details} needed by " - f"{self.deadline_time} or run: {self.callback}" + f"[{resource_type} Deadline] " + f"created at {self.created_at}, " + f"{resource_details}, " + f"needed by {self.deadline_time} " + f"or run: {self.callback}" ) @classmethod diff --git a/airflow-core/src/airflow/models/deadline_alert.py b/airflow-core/src/airflow/models/deadline_alert.py new file mode 100644 index 0000000000000..2cb645815a35e --- /dev/null +++ b/airflow-core/src/airflow/models/deadline_alert.py @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from datetime import datetime +from typing import TYPE_CHECKING + +import uuid6 +from sqlalchemy import JSON, Float, ForeignKey, String, Text, select +from sqlalchemy.exc import NoResultFound +from sqlalchemy.orm import Mapped +from sqlalchemy_utils import UUIDType + +from airflow._shared.timezones import timezone +from airflow.models import Base +from airflow.models.deadline import ReferenceModels +from airflow.utils.session import NEW_SESSION, provide_session +from airflow.utils.sqlalchemy import UtcDateTime, mapped_column + +if TYPE_CHECKING: + from sqlalchemy.orm import Session + + +class DeadlineAlert(Base): + """Table containing DeadlineAlert properties.""" + + __tablename__ = "deadline_alert" + + id: Mapped[str] = mapped_column(UUIDType(binary=False), primary_key=True, default=uuid6.uuid7) + created_at: Mapped[datetime] = mapped_column(UtcDateTime, nullable=False, default=timezone.utcnow) + + serialized_dag_id: Mapped[str] = mapped_column( + UUIDType(binary=False), ForeignKey("serialized_dag.id", ondelete="CASCADE"), nullable=False + ) + + name: Mapped[str | None] = mapped_column(String(250), nullable=True) + description: Mapped[str | None] = mapped_column(Text, nullable=True) + reference: Mapped[dict] = mapped_column(JSON, nullable=False) + interval: Mapped[float] = mapped_column(Float, nullable=False) + callback_def: Mapped[dict] = mapped_column(JSON, nullable=False) + + def __repr__(self): + interval_seconds = int(self.interval) + + if interval_seconds >= 3600: + interval_display = f"{interval_seconds // 3600}h" + elif interval_seconds >= 60: + interval_display = f"{interval_seconds // 60}m" + else: + interval_display = f"{interval_seconds}s" + + return ( + f"[DeadlineAlert] " + f"id={str(self.id)[:8]}, " + f"created_at={self.created_at}, " + f"name={self.name or 'Unnamed'}, " + f"reference={self.reference}, " + f"interval={interval_display}, " + f"callback={self.callback_def}" + ) + + def __eq__(self, other): + if not isinstance(other, DeadlineAlert): + return False + return ( + self.reference == other.reference + and self.interval == other.interval + and self.callback_def == other.callback_def + ) + + def __hash__(self): + return hash((str(self.reference), self.interval, str(self.callback_def))) + + @property + def reference_class(self) -> type[ReferenceModels.BaseDeadlineReference]: + """Return the deserialized reference object.""" + return ReferenceModels.get_reference_class(self.reference[ReferenceModels.REFERENCE_TYPE_FIELD]) + + @classmethod + @provide_session + def get_by_id(cls, deadline_alert_id: str, session: Session = NEW_SESSION) -> DeadlineAlert: + """ + Retrieve a DeadlineAlert record by its UUID. + + :param deadline_alert_id: The UUID of the DeadlineAlert to retrieve + :param session: Database session + """ + result = session.scalar(select(cls).where(cls.id == deadline_alert_id)) + if result is None: + raise NoResultFound(f"No DeadlineAlert found with id {deadline_alert_id}") + return result diff --git a/airflow-core/tests/unit/models/test_dag.py b/airflow-core/tests/unit/models/test_dag.py index 78af76114d2c1..20901fed9bd6a 100644 --- a/airflow-core/tests/unit/models/test_dag.py +++ b/airflow-core/tests/unit/models/test_dag.py @@ -1833,20 +1833,22 @@ def test_dag_owner_links(self, testing_dag_bundle): pytest.param(DeadlineReference.FIXED_DATETIME(DEFAULT_DATE), "NONE", id="fixed_deadline"), ], ) - def test_dagrun_deadline(self, reference_type, reference_column, dag_maker, session): + def test_dagrun_deadline(self, reference_type, reference_column, testing_dag_bundle, session): interval = datetime.timedelta(hours=1) - with dag_maker( - dag_id="test_queued_deadline", + dag = DAG( + dag_id="test_deadline", schedule=datetime.timedelta(days=1), deadline=DeadlineAlert( reference=reference_type, interval=interval, callback=AsyncCallback(empty_callback_for_deadline), ), - ) as dag: - ... + ) + + # Sync the DAG to the database to create DeadlineAlert records + scheduler_dag = sync_dag_to_db(dag, session=session) - dr = dag.create_dagrun( + dr = scheduler_dag.create_dagrun( run_id="test_dagrun_deadline", run_type=DagRunType.SCHEDULED, state=State.QUEUED, @@ -1860,7 +1862,7 @@ def test_dagrun_deadline(self, reference_type, reference_column, dag_maker, sess assert len(dr.deadlines) == 1 assert dr.deadlines[0].deadline_time == getattr(dr, reference_column, DEFAULT_DATE) + interval - def test_dag_with_multiple_deadlines(self, dag_maker, session): + def test_dag_with_multiple_deadlines(self, testing_dag_bundle, session): """Test that a DAG with multiple deadlines stores all deadlines in the database.""" deadlines = [ DeadlineAlert( @@ -1880,14 +1882,14 @@ def test_dag_with_multiple_deadlines(self, dag_maker, session): ), ] - with dag_maker( + dag = DAG( dag_id="test_multiple_deadlines", schedule=datetime.timedelta(days=1), deadline=deadlines, - ) as dag: - ... + ) + + scheduler_dag = sync_dag_to_db(dag, session=session) - scheduler_dag = sync_dag_to_db(dag) dr = scheduler_dag.create_dagrun( run_id="test_multiple_deadlines", run_type=DagRunType.SCHEDULED, diff --git a/airflow-core/tests/unit/models/test_dagrun.py b/airflow-core/tests/unit/models/test_dagrun.py index d890579b97a41..1af654c6d8214 100644 --- a/airflow-core/tests/unit/models/test_dagrun.py +++ b/airflow-core/tests/unit/models/test_dagrun.py @@ -27,7 +27,7 @@ import pendulum import pytest -from sqlalchemy import exists, func, select +from sqlalchemy import func, select from sqlalchemy.orm import joinedload from airflow import settings @@ -37,7 +37,8 @@ from airflow.models.dag import DagModel, infer_automated_data_interval from airflow.models.dag_version import DagVersion from airflow.models.dagrun import DagRun, DagRunNote -from airflow.models.deadline import Deadline +from airflow.models.deadline import Deadline, ReferenceModels +from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel from airflow.models.serialized_dag import SerializedDagModel from airflow.models.taskinstance import TaskInstance, TaskInstanceNote, clear_task_instances from airflow.models.taskmap import TaskMap @@ -87,6 +88,28 @@ def dagbag(): return DagBag(include_examples=True) +@pytest.fixture +def deadline_test_dag(session): + """Fixture that creates and syncs a basic DAG with two tasks.""" + + def _make_dag(deadline=None, on_success_callback=None): + dag_kwargs = {"dag_id": "test_dag", "schedule": datetime.timedelta(days=1)} + if deadline: + dag_kwargs["deadline"] = deadline + if on_success_callback: + dag_kwargs["on_success_callback"] = on_success_callback + + dag = DAG(**dag_kwargs) + task_1 = EmptyOperator(task_id="task_1", dag=dag) + task_2 = EmptyOperator(task_id="task_2", dag=dag) + task_1.set_downstream(task_2) + + scheduler_dag = sync_dag_to_db(dag, session=session) + return scheduler_dag + + return _make_dag + + class TestDagRun: @pytest.fixture(autouse=True) def setup_test_cases(self): @@ -1260,96 +1283,115 @@ def test_dag_run_dag_versions_with_null_created_dag_version(self, dag_maker, ses assert isinstance(dag_run.dag_versions, list) assert len(dag_run.dag_versions) == 0 - def test_dagrun_success_deadline(self, dag_maker, session): + @mock.patch.object(Deadline, "prune_deadlines") + def test_dagrun_success_deadline(self, _, session, deadline_test_dag): def on_success_callable(context): - assert context["dag_run"].dag_id == "test_dagrun_success_callback" + assert context["dag_run"].dag_id == "test_dag" future_date = datetime.datetime.now() + datetime.timedelta(days=365) - with dag_maker( - dag_id="test_dagrun_success_callback", - schedule=datetime.timedelta(days=1), - on_success_callback=on_success_callable, + scheduler_dag = deadline_test_dag( deadline=DeadlineAlert( reference=DeadlineReference.FIXED_DATETIME(future_date), interval=datetime.timedelta(hours=1), callback=AsyncCallback(empty_callback_for_deadline), ), - ) as dag: - dag_task1 = EmptyOperator(task_id="test_state_succeeded1") - dag_task2 = EmptyOperator(task_id="test_state_succeeded2") - dag_task1.set_downstream(dag_task2) - - initial_task_states = { - "test_state_succeeded1": TaskInstanceState.SUCCESS, - "test_state_succeeded2": TaskInstanceState.SUCCESS, - } + on_success_callback=on_success_callable, + ) - # Scheduler uses Serialized DAG -- so use that instead of the Actual DAG. - dag_run = self.create_dag_run(dag=dag, task_states=initial_task_states, session=session) - dag_run = session.merge(dag_run) - dag_run.dag = dag + dag_run = self.create_dag_run( + dag=scheduler_dag, + task_states={"task_1": TaskInstanceState.SUCCESS, "task_2": TaskInstanceState.SUCCESS}, + session=session, + ) + dag_run.dag = scheduler_dag with mock.patch.object(dag_run, "handle_dag_callback") as handle_dag_callback: _, callback = dag_run.update_state() - assert handle_dag_callback.mock_calls == [mock.call(dag=dag, success=True, reason="success")] + assert handle_dag_callback.mock_calls == [ + mock.call(dag=scheduler_dag, success=True, reason="success") + ] assert dag_run.state == DagRunState.SUCCESS # Callbacks are not added until handle_callback = False is passed to dag_run.update_state() assert callback is None - def test_dagrun_success_deadline_prune(self, dag_maker, session): - """Ensure only the deadline associated with dagrun marked as success is deleted.""" - now = timezone.utcnow() - future_date = datetime.datetime.now() + datetime.timedelta(days=365) - initial_task_states = { - "test_state_succeeded1": TaskInstanceState.SUCCESS, - } + @mock.patch.object(Deadline, "prune_deadlines") + @mock.patch.object(DeadlineAlertModel, "get_by_id") + def test_dagrun_success_prunes_dagrun_deadlines( + self, mock_get_by_id, mock_prune, session, deadline_test_dag + ): + mock_deadline_alert = mock.MagicMock() + mock_deadline_alert.reference_class = ReferenceModels.FixedDatetimeDeadline + mock_get_by_id.return_value = mock_deadline_alert - with dag_maker( - dag_id="dag_1", - schedule=datetime.timedelta(days=1), - deadline=DeadlineAlert( - reference=DeadlineReference.FIXED_DATETIME(future_date), - interval=datetime.timedelta(hours=1), - callback=AsyncCallback(empty_callback_for_deadline), - ), - session=session, - ) as dag1: - EmptyOperator(task_id="test_state_succeeded1") + scheduler_dag = deadline_test_dag() - dag_run1 = self.create_dag_run( - dag=dag1, session=session, logical_date=now, task_states=initial_task_states + deadline_ids = ["deadline-uuid-1", "deadline-uuid-2"] + scheduler_dag.deadline = deadline_ids + + dag_run = self.create_dag_run( + dag=scheduler_dag, + task_states={"task_1": TaskInstanceState.SUCCESS, "task_2": TaskInstanceState.SUCCESS}, + session=session, ) + dag_run.dag = scheduler_dag + + dag_run.update_state(session=session) + + assert mock_get_by_id.call_count == len(deadline_ids) + for deadline_id in deadline_ids: + mock_get_by_id.assert_any_call(deadline_id, session) + mock_prune.assert_called_once_with(session=session, conditions={DagRun.id: dag_run.id}) + assert dag_run.state == DagRunState.SUCCESS + + @mock.patch.object(Deadline, "prune_deadlines") + @mock.patch.object(DeadlineAlertModel, "get_by_id") + def test_dagrun_success_skips_pruning_non_dagrun_deadlines( + self, mock_get_by_id, mock_prune, dag_maker, session + ): + mock_deadline_alert = mock.MagicMock() + mock_deadline_alert.reference_class = "TASK" # Not DAGRUN + mock_get_by_id.return_value = mock_deadline_alert + + deadline_id = "deadline_alert_uuid" with dag_maker( - dag_id="dag_2", + dag_id="test_dagrun_no_prune", schedule=datetime.timedelta(days=1), - deadline=DeadlineAlert( - reference=DeadlineReference.FIXED_DATETIME(future_date), - interval=datetime.timedelta(hours=1), - callback=AsyncCallback(empty_callback_for_deadline), - ), - session=session, - ) as dag2: - EmptyOperator(task_id="test_state_succeeded1") + ) as dag: + EmptyOperator(task_id="task_1") - dag_run2 = self.create_dag_run( - dag=dag2, session=session, logical_date=now, task_states=initial_task_states - ) + initial_task_states = {"task_1": TaskInstanceState.SUCCESS} - dag_run1_deadline = exists().where(Deadline.dagrun_id == dag_run1.id) - dag_run2_deadline = exists().where(Deadline.dagrun_id == dag_run2.id) + dag_run = self.create_dag_run(dag=dag, task_states=initial_task_states, session=session) + dag_run = session.merge(dag_run) + dag.deadline = [deadline_id] + dag_run.dag = dag - assert session.scalar(select(dag_run1_deadline)) - assert session.scalar(select(dag_run2_deadline)) + dag_run.update_state(session=session) - session.add(dag_run1) - dag_run1.update_state() + mock_get_by_id.assert_called_once_with(deadline_id, session) + mock_prune.assert_not_called() + assert dag_run.state == DagRunState.SUCCESS - assert not session.scalar(select(dag_run1_deadline)) - assert session.scalar(select(dag_run2_deadline)) - assert dag_run1.state == DagRunState.SUCCESS - assert dag_run2.state == DagRunState.RUNNING + @mock.patch.object(Deadline, "prune_deadlines") + def test_dagrun_success_handles_empty_deadline_list(self, mock_prune, dag_maker, session): + with dag_maker( + dag_id="test_dagrun_empty_deadlines", + schedule=datetime.timedelta(days=1), + ) as dag: + EmptyOperator(task_id="task_1") + + initial_task_states = {"task_1": TaskInstanceState.SUCCESS} + dag_run = self.create_dag_run(dag=dag, task_states=initial_task_states, session=session) + dag_run = session.merge(dag_run) + dag.deadline = [] + dag_run.dag = dag + + dag_run.update_state(session=session) + + mock_prune.assert_not_called() + assert dag_run.state == DagRunState.SUCCESS @pytest.mark.parametrize( diff --git a/airflow-core/tests/unit/models/test_deadline.py b/airflow-core/tests/unit/models/test_deadline.py index 8521585298c71..379998efbb88c 100644 --- a/airflow-core/tests/unit/models/test_deadline.py +++ b/airflow-core/tests/unit/models/test_deadline.py @@ -105,14 +105,16 @@ def dagrun(session, dag_maker): @pytest.fixture def deadline_orm(dagrun, session): - deadline = Deadline( - deadline_time=DEFAULT_DATE, - callback=AsyncCallback(TEST_CALLBACK_PATH, TEST_CALLBACK_KWARGS), - dagrun_id=dagrun.id, - ) - session.add(deadline) - session.flush() - return deadline + with time_machine.travel(DEFAULT_DATE, tick=False): + deadline = Deadline( + deadline_time=DEFAULT_DATE, + callback=AsyncCallback(TEST_CALLBACK_PATH, TEST_CALLBACK_KWARGS), + dagrun_id=dagrun.id, + deadline_alert_id=None, + ) + session.add(deadline) + session.flush() + return deadline @pytest.mark.db_test @@ -166,11 +168,36 @@ def test_prune_deadlines(self, mock_session, conditions, dagrun): else: mock_session.query.assert_not_called() - def test_repr(self, deadline_orm, dagrun): - assert ( - repr(deadline_orm) == f"[DagRun Deadline] Dag: {DAG_ID} Run: {dagrun.id} needed by " - f"{DEFAULT_DATE} or run: {deadline_orm.callback}" - ) + def test_repr_with_callback_kwargs(self, deadline_orm, dagrun): + repr_str = repr(deadline_orm) + assert "[DagRun Deadline]" in repr_str + assert f"created at {DEFAULT_DATE}" in repr_str + assert f"Dag: {DAG_ID}" in repr_str + assert f"Run: {dagrun.id}" in repr_str + assert f"needed by {DEFAULT_DATE}" in repr_str + assert TEST_CALLBACK_PATH in repr_str + assert str(TEST_CALLBACK_KWARGS) in repr_str + + def test_repr_without_callback_kwargs(self, dagrun, session): + with time_machine.travel(DEFAULT_DATE, tick=False): + # Create a new Deadline without callback kwargs. + deadline = Deadline( + deadline_time=DEFAULT_DATE, + callback=AsyncCallback(TEST_CALLBACK_PATH), + dagrun_id=dagrun.id, + deadline_alert_id=None, + ) + session.add(deadline) + session.flush() + + assert not deadline.callback.data.get("kwargs") + repr_str = repr(deadline) + assert "[DagRun Deadline]" in repr_str + assert f"created at {DEFAULT_DATE}" in repr_str + assert f"Dag: {DAG_ID}" in repr_str + assert f"Run: {dagrun.id}" in repr_str + assert f"needed by {DEFAULT_DATE}" in repr_str + assert TEST_CALLBACK_PATH in repr_str @pytest.mark.db_test def test_handle_miss(self, dagrun, session): @@ -179,6 +206,7 @@ def test_handle_miss(self, dagrun, session): callback=AsyncCallback(TEST_CALLBACK_PATH, TEST_CALLBACK_KWARGS), dagrun_id=dagrun.id, dag_id=dagrun.dag_id, + deadline_alert_id=None, ) session.add(deadline_orm) session.flush() diff --git a/airflow-core/tests/unit/models/test_deadline_alert.py b/airflow-core/tests/unit/models/test_deadline_alert.py new file mode 100644 index 0000000000000..7ee8f3e51e608 --- /dev/null +++ b/airflow-core/tests/unit/models/test_deadline_alert.py @@ -0,0 +1,184 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest +import time_machine + +from airflow.models.deadline import ReferenceModels +from airflow.models.deadline_alert import DeadlineAlert +from airflow.models.serialized_dag import SerializedDagModel +from airflow.sdk.definitions.deadline import DeadlineReference + +from tests_common.test_utils import db +from unit.models import DEFAULT_DATE + +DAG_ID = "test_deadline_alert_dag" +DEADLINE_NAME = "Test Alert" +DEADLINE_DESCRIPTION = "This is a test alert description" +DEADLINE_INTERVAL = 60 +DEADLINE_CALLBACK = {"path": "test.callback"} +SERIALIZED_DAG_ID = "serialized_dag_uuid" + + +def _clean_db(): + db.clear_db_deadline_alert() + + +@pytest.fixture +def deadline_reference(): + return DeadlineReference.DAGRUN_QUEUED_AT.serialize_reference() + + +@pytest.fixture +def deadline_alert_orm(dag_maker, session, deadline_reference): + with dag_maker(DAG_ID, session=session): + pass + + serialized_dag = session.query(SerializedDagModel).filter_by(dag_id=DAG_ID).one() + + with time_machine.travel(DEFAULT_DATE, tick=False): + alert = DeadlineAlert( + serialized_dag_id=serialized_dag.id, + name=DEADLINE_NAME, + description=DEADLINE_DESCRIPTION, + reference=deadline_reference, + interval=DEADLINE_INTERVAL, + callback_def=DEADLINE_CALLBACK, + ) + session.add(alert) + session.flush() + return alert + + +@pytest.mark.db_test +class TestDeadlineAlert: + @staticmethod + def setup_method(): + _clean_db() + + @staticmethod + def teardown_method(): + _clean_db() + + def test_deadline_alert_creation(self, deadline_alert_orm): + assert deadline_alert_orm.id is not None + assert deadline_alert_orm.created_at == DEFAULT_DATE + assert deadline_alert_orm.name == DEADLINE_NAME + assert deadline_alert_orm.description == DEADLINE_DESCRIPTION + + def test_minimal_deadline_alert_creation(self, dag_maker, session, deadline_reference): + with dag_maker(DAG_ID, session=session): + pass + + serialized_dag = session.query(SerializedDagModel).filter_by(dag_id=DAG_ID).one() + + with time_machine.travel(DEFAULT_DATE, tick=False): + deadline_alert = DeadlineAlert( + serialized_dag_id=serialized_dag.id, + reference=deadline_reference, + interval=DEADLINE_INTERVAL, + callback_def=DEADLINE_CALLBACK, + ) + session.add(deadline_alert) + session.flush() + + assert deadline_alert.id is not None + assert deadline_alert.created_at == DEFAULT_DATE + assert deadline_alert.name is None + assert deadline_alert.description is None + + def test_deadline_alert_repr(self, deadline_alert_orm, deadline_reference): + repr_str = repr(deadline_alert_orm) + assert "[DeadlineAlert]" in repr_str + assert "id=" in repr_str + assert f"created_at={DEFAULT_DATE}" in repr_str + assert f"name={DEADLINE_NAME}" in repr_str + assert f"reference={deadline_reference}" in repr_str + assert "interval=1m" in repr_str + assert repr(deadline_alert_orm.callback_def) in repr_str + + def test_deadline_alert_equality(self, session, deadline_reference): + alert1 = DeadlineAlert( + serialized_dag_id=SERIALIZED_DAG_ID, + reference=deadline_reference, + interval=DEADLINE_INTERVAL, + callback_def=DEADLINE_CALLBACK, + ) + alert2 = DeadlineAlert( + serialized_dag_id=SERIALIZED_DAG_ID, + reference=deadline_reference, + interval=DEADLINE_INTERVAL, + callback_def=DEADLINE_CALLBACK, + ) + assert alert1 == alert2 + + different_ref = DeadlineAlert( + serialized_dag_id=SERIALIZED_DAG_ID, + reference=DeadlineReference.DAGRUN_LOGICAL_DATE.serialize_reference(), + interval=DEADLINE_INTERVAL, + callback_def=DEADLINE_CALLBACK, + ) + assert alert1 != different_ref + + different_interval = DeadlineAlert( + serialized_dag_id=SERIALIZED_DAG_ID, + reference=deadline_reference, + interval=120, + callback_def=DEADLINE_CALLBACK, + ) + assert alert1 != different_interval + + different_callback = DeadlineAlert( + serialized_dag_id=SERIALIZED_DAG_ID, + reference=deadline_reference, + interval=DEADLINE_INTERVAL, + callback_def={"path": "different.callback"}, + ) + assert alert1 != different_callback + + assert alert1 != "not a deadline alert" + + def test_deadline_alert_hash(self, session, deadline_reference): + alert1 = DeadlineAlert( + serialized_dag_id=SERIALIZED_DAG_ID, + reference=deadline_reference, + interval=DEADLINE_INTERVAL, + callback_def=DEADLINE_CALLBACK, + ) + alert2 = DeadlineAlert( + serialized_dag_id=SERIALIZED_DAG_ID, + reference=deadline_reference, + interval=DEADLINE_INTERVAL, + callback_def=DEADLINE_CALLBACK, + ) + + assert hash(alert1) == hash(alert2) + + def test_deadline_alert_reference_class_property(self, deadline_alert_orm): + assert deadline_alert_orm.reference_class == ReferenceModels.DagRunQueuedAtDeadline + + def test_deadline_alert_get_by_id(self, deadline_alert_orm, session): + retrieved_alert = DeadlineAlert.get_by_id(deadline_alert_orm.id, session=session) + assert retrieved_alert == deadline_alert_orm + + def test_deadline_alert_get_by_id_not_found(self, session): + from sqlalchemy.exc import NoResultFound + + nonexistent_uuid = "00000000-0000-7000-8000-000000000000" + with pytest.raises(NoResultFound, match="No DeadlineAlert found"): + DeadlineAlert.get_by_id(nonexistent_uuid, session=session) From 570901ba4fcf62f88428380a86a97676f474e036 Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Fri, 9 Jan 2026 14:56:29 -0800 Subject: [PATCH 02/14] Port collection over --- airflow-core/src/airflow/dag_processing/collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow-core/src/airflow/dag_processing/collection.py b/airflow-core/src/airflow/dag_processing/collection.py index fd09df8fd36ec..e13957570f394 100644 --- a/airflow-core/src/airflow/dag_processing/collection.py +++ b/airflow-core/src/airflow/dag_processing/collection.py @@ -526,8 +526,8 @@ def update_dags( else: dm.max_consecutive_failed_dag_runs = dag.max_consecutive_failed_dag_runs - if dag.deadline is not None: - dm.deadline = dag.deadline + if (deadline_uuids := dag.data.get("dag", {}).get("deadline")) is not None: + dm.deadline = deadline_uuids if hasattr(dag, "has_task_concurrency_limits"): dm.has_task_concurrency_limits = dag.has_task_concurrency_limits From 911713439366386bf05bce5cf945d692b8a48659 Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Fri, 9 Jan 2026 15:45:35 -0800 Subject: [PATCH 03/14] port test_scheduler --- .../tests/unit/jobs/test_scheduler_job.py | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/airflow-core/tests/unit/jobs/test_scheduler_job.py b/airflow-core/tests/unit/jobs/test_scheduler_job.py index 2b9ed36c7cb4d..f109d620d6493 100644 --- a/airflow-core/tests/unit/jobs/test_scheduler_job.py +++ b/airflow-core/tests/unit/jobs/test_scheduler_job.py @@ -69,6 +69,7 @@ from airflow.models.dagwarning import DagWarning from airflow.models.db_callback_request import DbCallbackRequest from airflow.models.deadline import Deadline +from airflow.models.deadline_alert import DeadlineAlert from airflow.models.log import Log from airflow.models.pool import Pool from airflow.models.serialized_dag import SerializedDagModel @@ -7327,11 +7328,26 @@ def test_process_expired_deadlines(self, mock_handle_miss, session, dag_maker): EmptyOperator(task_id="empty") dagrun_id = dag_maker.create_dagrun().id + serialized_dag = session.scalar(select(SerializedDagModel).where(SerializedDagModel.dag_id == dag_id)) + assert serialized_dag is not None + + # Create a test DeadlineAlert object for Deadline + deadline_alert = DeadlineAlert( + serialized_dag_id=serialized_dag.id, + name="Test Alert", + reference={"type": "dag", "dag_id": dag_id}, + interval=300.0, # 5 minutes + callback_def={"classpath": callback_path, "kwargs": {}}, + ) + session.add(deadline_alert) + session.flush() + handled_deadline_async = Deadline( deadline_time=past_date, callback=AsyncCallback(callback_path), dagrun_id=dagrun_id, dag_id=dag_id, + deadline_alert_id=deadline_alert.id, ) handled_deadline_async.missed = True @@ -7340,20 +7356,30 @@ def test_process_expired_deadlines(self, mock_handle_miss, session, dag_maker): callback=SyncCallback(callback_path), dagrun_id=dagrun_id, dag_id=dag_id, + deadline_alert_id=deadline_alert.id, ) handled_deadline_sync.missed = True expired_deadline1 = Deadline( - deadline_time=past_date, callback=AsyncCallback(callback_path), dagrun_id=dagrun_id, dag_id=dag_id + deadline_time=past_date, + callback=AsyncCallback(callback_path), + dagrun_id=dagrun_id, + dag_id=dag_id, + deadline_alert_id=deadline_alert.id, ) expired_deadline2 = Deadline( - deadline_time=past_date, callback=SyncCallback(callback_path), dagrun_id=dagrun_id, dag_id=dag_id + deadline_time=past_date, + callback=SyncCallback(callback_path), + dagrun_id=dagrun_id, + dag_id=dag_id, + deadline_alert_id=deadline_alert.id, ) future_deadline = Deadline( deadline_time=future_date, callback=AsyncCallback(callback_path), dagrun_id=dagrun_id, dag_id=dag_id, + deadline_alert_id=deadline_alert.id, ) session.add_all( From 0687d4c16bfd3c9f5f4a7ebe71553e35b9a22842 Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Fri, 9 Jan 2026 16:13:49 -0800 Subject: [PATCH 04/14] port serialized_objects --- .../airflow/serialization/definitions/dag.py | 94 +++++++++++++++---- .../serialization/serialized_objects.py | 24 ++--- 2 files changed, 83 insertions(+), 35 deletions(-) diff --git a/airflow-core/src/airflow/serialization/definitions/dag.py b/airflow-core/src/airflow/serialization/definitions/dag.py index 6cb26f48607fb..a4eb10d10d887 100644 --- a/airflow-core/src/airflow/serialization/definitions/dag.py +++ b/airflow-core/src/airflow/serialization/definitions/dag.py @@ -40,7 +40,9 @@ from airflow.models.deadline import Deadline from airflow.models.taskinstancekey import TaskInstanceKey from airflow.models.tasklog import LogTemplate -from airflow.sdk.definitions.deadline import DeadlineReference +from airflow.observability.stats import Stats +from airflow.sdk.definitions.deadline import DeadlineReference, DeadlineAlertFields +from airflow.serialization.enums import DagAttributeTypes as DAT, Encoding from airflow.serialization.definitions.param import SerializedParamsDict from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction from airflow.utils.session import NEW_SESSION, provide_session @@ -57,6 +59,7 @@ from sqlalchemy.orm import Session from typing_extensions import TypeIs + from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel from airflow.models.taskinstance import TaskInstance from airflow.sdk import DAG from airflow.sdk.definitions.deadline import DeadlineAlert @@ -96,7 +99,7 @@ class SerializedDAG: access_control: dict[str, dict[str, Collection[str]]] | None = None catchup: bool = False dagrun_timeout: datetime.timedelta | None = None - deadline: list[DeadlineAlert] | DeadlineAlert | None = None + deadline: list[str] | None = None default_args: dict[str, Any] = attrs.field(factory=dict) description: str | None = None disable_bundle_versioning: bool = False @@ -605,27 +608,78 @@ def create_dagrun( ) if self.deadline: - for deadline in cast("list", self.deadline): - if isinstance(deadline.reference, DeadlineReference.TYPES.DAGRUN): - deadline_time = deadline.reference.evaluate_with( - session=session, - interval=deadline.interval, - dag_id=self.dag_id, - run_id=run_id, - ) - if deadline_time is not None: - session.add( - Deadline( - deadline_time=deadline_time, - callback=deadline.callback, - dagrun_id=orm_dagrun.id, - dag_id=orm_dagrun.dag_id, - ) - ) - Stats.incr("deadline_alerts.deadline_created", tags={"dag_id": self.dag_id}) + self._process_dagrun_deadline_alerts(orm_dagrun, session) return orm_dagrun + def _process_dagrun_deadline_alerts( + self, + orm_dagrun: DagRun, + session: Session, + ) -> None: + """ + Process deadline alerts for a newly created DagRun. + + Creates Deadline records for any DeadlineAlerts that reference DAGRUN. + + :param orm_dagrun: The newly created DagRun + :param session: Database session + """ + # Import here to avoid circular dependency + from airflow.models.serialized_dag import SerializedDagModel + + # Get the serialized_dag ID for this DAG + serialized_dag_id = session.scalar( + select(SerializedDagModel.id).where( + SerializedDagModel.dag_version_id == orm_dagrun.created_dag_version_id + ) + ) + + if not serialized_dag_id: + return + + # Query deadline alerts by serialized_dag_id + deadline_alert_records = ( + session.query(DeadlineAlertModel) + .filter(DeadlineAlertModel.serialized_dag_id == serialized_dag_id) + .all() + ) + + for deadline_alert in deadline_alert_records: + if not deadline_alert: + continue + deserialized_deadline_alert = DeadlineAlert.deserialize_deadline_alert( + { + Encoding.TYPE: DAT.DEADLINE_ALERT, + Encoding.VAR: { + DeadlineAlertFields.REFERENCE: deadline_alert.reference, + DeadlineAlertFields.INTERVAL: deadline_alert.interval, + DeadlineAlertFields.CALLBACK: deadline_alert.callback_def, + }, + } + ) + + if isinstance(deserialized_deadline_alert.reference, DeadlineReference.TYPES.DAGRUN): + deadline_time = deserialized_deadline_alert.reference.evaluate_with( + session=session, + interval=deserialized_deadline_alert.interval, + # TODO : Pretty sure we can drop these last two; verify after testing is complete + dag_id=self.dag_id, + run_id=orm_dagrun.run_id, + ) + + if deadline_time is not None: + session.add( + Deadline( + deadline_time=deadline_time, + callback=deserialized_deadline_alert.callback, + dagrun_id=orm_dagrun.id, + deadline_alert_id=deadline_alert.id, + dag_id=orm_dagrun.dag_id, + ) + ) + Stats.incr("deadline_alerts.deadline_created", tags={"dag_id": self.dag_id}) + @provide_session def set_task_instance_state( self, diff --git a/airflow-core/src/airflow/serialization/serialized_objects.py b/airflow-core/src/airflow/serialization/serialized_objects.py index 2f698796331ca..e10009de6131e 100644 --- a/airflow-core/src/airflow/serialization/serialized_objects.py +++ b/airflow-core/src/airflow/serialization/serialized_objects.py @@ -57,7 +57,7 @@ AssetUniqueKey, BaseAsset, ) -from airflow.sdk.definitions.deadline import DeadlineAlert +from airflow.sdk.definitions.deadline import DeadlineAlert, DeadlineAlertFields, DeadlineReference from airflow.sdk.definitions.mappedoperator import MappedOperator from airflow.sdk.definitions.operator_resources import Resources from airflow.sdk.definitions.param import Param, ParamsDict @@ -1757,11 +1757,13 @@ def serialize_dag(cls, dag: DAG) -> dict: serialized_dag["dag_dependencies"] = [x.__dict__ for x in sorted(dag_deps)] serialized_dag["task_group"] = TaskGroupSerialization.serialize_task_group(dag.task_group) - serialized_dag["deadline"] = ( - [deadline.serialize_deadline_alert() for deadline in dag.deadline] - if isinstance(dag.deadline, list) - else None - ) + if dag.deadline: + deadline_list = dag.deadline if isinstance(dag.deadline, list) else [dag.deadline] + serialized_dag["deadline"] = [ + deadline.serialize_deadline_alert() for deadline in deadline_list + ] + else: + serialized_dag["deadline"] = None # Edge info in the JSON exactly matches our internal structure serialized_dag["edge_info"] = dag.edge_info @@ -1894,15 +1896,7 @@ def _deserialize_dag_internal( if "has_on_failure_callback" in encoded_dag: dag.has_on_failure_callback = True - if "deadline" in encoded_dag and encoded_dag["deadline"] is not None: - dag.deadline = ( - [ - DeadlineAlert.deserialize_deadline_alert(deadline_data) - for deadline_data in encoded_dag["deadline"] - ] - if encoded_dag["deadline"] - else None - ) + dag.deadline = encoded_dag.get("deadline") keys_to_set_none = dag.get_serialized_fields() - encoded_dag.keys() - cls._CONSTRUCTOR_PARAMS.keys() for k in keys_to_set_none: From 7fcf504cc9b8f9cd2d5758ba1836fc54136145b5 Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Fri, 9 Jan 2026 16:22:14 -0800 Subject: [PATCH 05/14] port serialized_dag --- .../src/airflow/models/serialized_dag.py | 82 ++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/airflow-core/src/airflow/models/serialized_dag.py b/airflow-core/src/airflow/models/serialized_dag.py index 0b0154dfa9e0f..9b221f898a39a 100644 --- a/airflow-core/src/airflow/models/serialized_dag.py +++ b/airflow-core/src/airflow/models/serialized_dag.py @@ -44,9 +44,12 @@ from airflow.models.dag_version import DagVersion from airflow.models.dagcode import DagCode from airflow.models.dagrun import DagRun +from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel from airflow.serialization.dag_dependency import DagDependency from airflow.serialization.definitions.assets import SerializedAssetUniqueKey as UKey +from airflow.serialization.enums import Encoding from airflow.serialization.serialized_objects import DagSerialization +from airflow.sdk.definitions.deadline import DeadlineAlertFields # TODO: move to airflow.serialization.deadline? from airflow.settings import json from airflow.utils.hashlib_wrapper import md5 from airflow.utils.session import NEW_SESSION, provide_session @@ -323,6 +326,13 @@ class SerializedDagModel(Base): ) dag_version = relationship("DagVersion", back_populates="serialized_dag") + deadline_alerts = relationship( + "DeadlineAlert", + foreign_keys="DeadlineAlert.serialized_dag_id", + cascade="all, delete-orphan", + passive_deletes=True, + ) + load_op_links = True def __init__(self, dag: LazyDeserializedDAG) -> None: @@ -380,6 +390,57 @@ def _sort_serialized_dag_dict(cls, serialized_dag: Any): return [cls._sort_serialized_dag_dict(i) for i in serialized_dag] return serialized_dag + @classmethod + def _generate_deadline_uuids(cls, dag_data: dict[str, Any]) -> dict[str, dict]: + """ + Generate UUIDs for DeadlineAlerts and replace dicts with list[UUID] in dag_data. + + This modifies dag_data in place, replacing deadline alert definitions with UUID strings. + Called before SerializedDagModel creation to ensure UUIDs are included in the hash. + + :param dag_data: The serialized DAG data dictionary + :return: Mapping of UUID strings to deadline alert data dicts + """ + uuid_mapping: dict[str, dict] = {} + + dag_deadline_data = dag_data.get("dag", {}).get("deadline") + if not dag_deadline_data: + return uuid_mapping + + for deadline_alert in dag_deadline_data: + deadline_data = deadline_alert.get(Encoding.VAR, deadline_alert) + + deadline_uuid = str(uuid6.uuid7()) + uuid_mapping[deadline_uuid] = deadline_data + + dag_data["dag"]["deadline"] = list(uuid_mapping.keys()) + + return uuid_mapping + + @classmethod + def _create_deadline_alert_records( + cls, + serialized_dag: SerializedDagModel, + uuid_mapping: dict[str, dict], + ) -> None: + """ + Create DeadlineAlert records in the database and appends them to serialized_dag. + + :param serialized_dag: The SerializedDagModel instance (not yet flushed) + :param uuid_mapping: Mapping of UUID strings to deadline alert data dicts + """ + if not uuid_mapping: + return + + for uuid_str, deadline_data in uuid_mapping.items(): + alert = DeadlineAlertModel( + id=uuid_str, + reference=deadline_data[DeadlineAlertFields.REFERENCE], + interval=deadline_data[DeadlineAlertFields.INTERVAL], + callback_def=deadline_data[DeadlineAlertFields.CALLBACK], + ) + serialized_dag.deadline_alerts.append(alert) + @classmethod @provide_session def write_dag( @@ -419,7 +480,6 @@ def write_dag( return False log.debug("Checking if DAG (%s) changed", dag.dag_id) - new_serialized_dag = cls(dag) serialized_dag_hash = session.scalars( select(cls.dag_hash).where(cls.dag_id == dag.dag_id).order_by(cls.created_at.desc()) ).first() @@ -431,6 +491,25 @@ def write_dag( .limit(1) ) + if dag.data.get("dag", {}).get("deadline"): + # If this DAG has been serialized before then reuse deadline UUIDs to preserve the hash, + # otherwise we have new serialized dags getting generated constantly. + existing_serialized_dag = session.scalar( + select(cls).where(cls.dag_id == dag.dag_id).order_by(cls.created_at.desc()).limit(1) + ) + + if existing_serialized_dag and ( + existing_deadline_uuids := existing_serialized_dag.data.get("dag", {}).get("deadline") + ): + dag.data["dag"]["deadline"] = existing_deadline_uuids + deadline_uuid_mapping = {} + else: + deadline_uuid_mapping = cls._generate_deadline_uuids(dag.data) + else: + deadline_uuid_mapping = {} + + new_serialized_dag = cls(dag) + if ( serialized_dag_hash == new_serialized_dag.dag_hash and dag_version @@ -479,6 +558,7 @@ def write_dag( log.debug("Writing Serialized DAG: %s to the DB", dag.dag_id) new_serialized_dag.dag_version = dagv session.add(new_serialized_dag) + cls._create_deadline_alert_records(new_serialized_dag, deadline_uuid_mapping) log.debug("DAG: %s written to the DB", dag.dag_id) DagCode.write_code(dagv, dag.fileloc, session=session) return True From 50bd728192733ad5073ebc62c78c0849bd13347d Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Fri, 9 Jan 2026 16:28:03 -0800 Subject: [PATCH 06/14] move DeadlineAlertFields into the new serialization/definitions/deadline --- .../src/airflow/models/serialized_dag.py | 2 +- .../airflow/serialization/definitions/dag.py | 3 +- .../serialization/definitions/deadline.py | 30 +++++++++++++++++++ .../serialization/serialized_objects.py | 2 +- .../serialization/test_serialized_objects.py | 2 +- .../src/airflow/sdk/definitions/deadline.py | 14 +-------- 6 files changed, 36 insertions(+), 17 deletions(-) create mode 100644 airflow-core/src/airflow/serialization/definitions/deadline.py diff --git a/airflow-core/src/airflow/models/serialized_dag.py b/airflow-core/src/airflow/models/serialized_dag.py index 9b221f898a39a..c8b43ad1f387e 100644 --- a/airflow-core/src/airflow/models/serialized_dag.py +++ b/airflow-core/src/airflow/models/serialized_dag.py @@ -49,7 +49,7 @@ from airflow.serialization.definitions.assets import SerializedAssetUniqueKey as UKey from airflow.serialization.enums import Encoding from airflow.serialization.serialized_objects import DagSerialization -from airflow.sdk.definitions.deadline import DeadlineAlertFields # TODO: move to airflow.serialization.deadline? +from airflow.serialization.definitions.deadline import DeadlineAlertFields from airflow.settings import json from airflow.utils.hashlib_wrapper import md5 from airflow.utils.session import NEW_SESSION, provide_session diff --git a/airflow-core/src/airflow/serialization/definitions/dag.py b/airflow-core/src/airflow/serialization/definitions/dag.py index a4eb10d10d887..8825b6057e9ed 100644 --- a/airflow-core/src/airflow/serialization/definitions/dag.py +++ b/airflow-core/src/airflow/serialization/definitions/dag.py @@ -41,7 +41,8 @@ from airflow.models.taskinstancekey import TaskInstanceKey from airflow.models.tasklog import LogTemplate from airflow.observability.stats import Stats -from airflow.sdk.definitions.deadline import DeadlineReference, DeadlineAlertFields +from airflow.sdk.definitions.deadline import DeadlineReference +from airflow.serialization.definitions.deadline import DeadlineAlertFields from airflow.serialization.enums import DagAttributeTypes as DAT, Encoding from airflow.serialization.definitions.param import SerializedParamsDict from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction diff --git a/airflow-core/src/airflow/serialization/definitions/deadline.py b/airflow-core/src/airflow/serialization/definitions/deadline.py new file mode 100644 index 0000000000000..b48ea4883aa36 --- /dev/null +++ b/airflow-core/src/airflow/serialization/definitions/deadline.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + + +class DeadlineAlertFields: + """ + Define field names used in DeadlineAlert serialization/deserialization. + + These constants provide a single source of truth for the field names used when + serializing DeadlineAlert instances to and from their dictionary representation. + """ + + REFERENCE = "reference" + INTERVAL = "interval" + CALLBACK = "callback" diff --git a/airflow-core/src/airflow/serialization/serialized_objects.py b/airflow-core/src/airflow/serialization/serialized_objects.py index e10009de6131e..4d477f665e395 100644 --- a/airflow-core/src/airflow/serialization/serialized_objects.py +++ b/airflow-core/src/airflow/serialization/serialized_objects.py @@ -57,7 +57,7 @@ AssetUniqueKey, BaseAsset, ) -from airflow.sdk.definitions.deadline import DeadlineAlert, DeadlineAlertFields, DeadlineReference +from airflow.sdk.definitions.deadline import DeadlineAlert from airflow.sdk.definitions.mappedoperator import MappedOperator from airflow.sdk.definitions.operator_resources import Resources from airflow.sdk.definitions.param import Param, ParamsDict diff --git a/airflow-core/tests/unit/serialization/test_serialized_objects.py b/airflow-core/tests/unit/serialization/test_serialized_objects.py index 82769d00a69b1..b69a9a29dbe41 100644 --- a/airflow-core/tests/unit/serialization/test_serialized_objects.py +++ b/airflow-core/tests/unit/serialization/test_serialized_objects.py @@ -61,9 +61,9 @@ from airflow.sdk.definitions.deadline import ( AsyncCallback, DeadlineAlert, - DeadlineAlertFields, DeadlineReference, ) +from airflow.serialization.definitions.deadline import DeadlineAlertFields from airflow.sdk.definitions.decorators import task from airflow.sdk.definitions.operator_resources import Resources from airflow.sdk.definitions.param import Param diff --git a/task-sdk/src/airflow/sdk/definitions/deadline.py b/task-sdk/src/airflow/sdk/definitions/deadline.py index 28ba0d8168889..c0f2389717348 100644 --- a/task-sdk/src/airflow/sdk/definitions/deadline.py +++ b/task-sdk/src/airflow/sdk/definitions/deadline.py @@ -23,6 +23,7 @@ from airflow.models.deadline import DeadlineReferenceType, ReferenceModels from airflow.sdk.definitions.callback import AsyncCallback, Callback from airflow.sdk.serde import deserialize, serialize +from airflow.serialization.definitions.deadline import DeadlineAlertFields from airflow.serialization.enums import DagAttributeTypes as DAT, Encoding if TYPE_CHECKING: @@ -34,19 +35,6 @@ DeadlineReferenceTypes: TypeAlias = tuple[type[ReferenceModels.BaseDeadlineReference], ...] -class DeadlineAlertFields: - """ - Define field names used in DeadlineAlert serialization/deserialization. - - These constants provide a single source of truth for the field names used when - serializing DeadlineAlert instances to and from their dictionary representation. - """ - - REFERENCE = "reference" - INTERVAL = "interval" - CALLBACK = "callback" - - class DeadlineAlert: """Store Deadline values needed to calculate the need-by timestamp and the callback information.""" From cfbaf8b87e574b78b94fa42f45a03ea4f3e49b0c Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Mon, 12 Jan 2026 12:20:54 -0800 Subject: [PATCH 07/14] finish serdag migration --- airflow-core/src/airflow/models/serialized_dag.py | 2 +- airflow-core/src/airflow/serialization/definitions/dag.py | 7 +++---- .../tests/unit/serialization/test_serialized_objects.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/airflow-core/src/airflow/models/serialized_dag.py b/airflow-core/src/airflow/models/serialized_dag.py index c8b43ad1f387e..2fe3528b5c598 100644 --- a/airflow-core/src/airflow/models/serialized_dag.py +++ b/airflow-core/src/airflow/models/serialized_dag.py @@ -47,9 +47,9 @@ from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel from airflow.serialization.dag_dependency import DagDependency from airflow.serialization.definitions.assets import SerializedAssetUniqueKey as UKey +from airflow.serialization.definitions.deadline import DeadlineAlertFields from airflow.serialization.enums import Encoding from airflow.serialization.serialized_objects import DagSerialization -from airflow.serialization.definitions.deadline import DeadlineAlertFields from airflow.settings import json from airflow.utils.hashlib_wrapper import md5 from airflow.utils.session import NEW_SESSION, provide_session diff --git a/airflow-core/src/airflow/serialization/definitions/dag.py b/airflow-core/src/airflow/serialization/definitions/dag.py index 8825b6057e9ed..f86d55163ea1a 100644 --- a/airflow-core/src/airflow/serialization/definitions/dag.py +++ b/airflow-core/src/airflow/serialization/definitions/dag.py @@ -38,13 +38,14 @@ from airflow.models.dag_version import DagVersion from airflow.models.dagrun import DagRun from airflow.models.deadline import Deadline +from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel from airflow.models.taskinstancekey import TaskInstanceKey from airflow.models.tasklog import LogTemplate from airflow.observability.stats import Stats -from airflow.sdk.definitions.deadline import DeadlineReference +from airflow.sdk.definitions.deadline import DeadlineAlert, DeadlineReference from airflow.serialization.definitions.deadline import DeadlineAlertFields -from airflow.serialization.enums import DagAttributeTypes as DAT, Encoding from airflow.serialization.definitions.param import SerializedParamsDict +from airflow.serialization.enums import DagAttributeTypes as DAT, Encoding from airflow.timetables.base import DagRunInfo, DataInterval, TimeRestriction from airflow.utils.session import NEW_SESSION, provide_session from airflow.utils.state import DagRunState, TaskInstanceState @@ -60,10 +61,8 @@ from sqlalchemy.orm import Session from typing_extensions import TypeIs - from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel from airflow.models.taskinstance import TaskInstance from airflow.sdk import DAG - from airflow.sdk.definitions.deadline import DeadlineAlert from airflow.serialization.definitions.taskgroup import SerializedTaskGroup from airflow.serialization.serialized_objects import LazyDeserializedDAG, SerializedOperator from airflow.timetables.base import Timetable diff --git a/airflow-core/tests/unit/serialization/test_serialized_objects.py b/airflow-core/tests/unit/serialization/test_serialized_objects.py index b69a9a29dbe41..64c3128461abb 100644 --- a/airflow-core/tests/unit/serialization/test_serialized_objects.py +++ b/airflow-core/tests/unit/serialization/test_serialized_objects.py @@ -63,7 +63,6 @@ DeadlineAlert, DeadlineReference, ) -from airflow.serialization.definitions.deadline import DeadlineAlertFields from airflow.sdk.definitions.decorators import task from airflow.sdk.definitions.operator_resources import Resources from airflow.sdk.definitions.param import Param @@ -77,6 +76,7 @@ SerializedAssetBase, SerializedAssetRef, ) +from airflow.serialization.definitions.deadline import DeadlineAlertFields from airflow.serialization.encoders import ensure_serialized_asset from airflow.serialization.enums import DagAttributeTypes as DAT, Encoding from airflow.serialization.serialized_objects import ( From b5c3771acb6d65e47a0bf12bd5e3ff22ea1dc392 Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Mon, 12 Jan 2026 13:45:00 -0800 Subject: [PATCH 08/14] update migration chain and erd again --- airflow-core/docs/img/airflow_erd.sha256 | 2 +- airflow-core/docs/img/airflow_erd.svg | 4516 +++++++++-------- airflow-core/docs/migrations-ref.rst | 5 +- ...99_3_2_0_ui_improvements_for_deadlines.py} | 4 +- airflow-core/src/airflow/utils/db.py | 2 +- 5 files changed, 2302 insertions(+), 2227 deletions(-) rename airflow-core/src/airflow/migrations/versions/{0098_3_2_0_ui_improvements_for_deadlines.py => 0099_3_2_0_ui_improvements_for_deadlines.py} (99%) diff --git a/airflow-core/docs/img/airflow_erd.sha256 b/airflow-core/docs/img/airflow_erd.sha256 index e4b17237c3c7b..53b16b6a8a666 100644 --- a/airflow-core/docs/img/airflow_erd.sha256 +++ b/airflow-core/docs/img/airflow_erd.sha256 @@ -1 +1 @@ -174cc04e341bf41946034e5ba04dcc1da7b5b2c7e09cd74b378e07bb87de2ee6 \ No newline at end of file +c86a1afe46a98ddece8a3bac7de0215dec8e5ca7a70765fd1584eb43f23fbf54 \ No newline at end of file diff --git a/airflow-core/docs/img/airflow_erd.svg b/airflow-core/docs/img/airflow_erd.svg index 49f9a5c66a75b..5878b88806bb5 100644 --- a/airflow-core/docs/img/airflow_erd.svg +++ b/airflow-core/docs/img/airflow_erd.svg @@ -4,2662 +4,2734 @@ - - + + %3 - + job - -job - -id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - -end_date - - [TIMESTAMP] - -executor_class - - [VARCHAR(500)] - -hostname - - [VARCHAR(500)] - -job_type - - [VARCHAR(30)] - -latest_heartbeat - - [TIMESTAMP] - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -unixname - - [VARCHAR(1000)] + +job + +id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + +end_date + + [TIMESTAMP] + +executor_class + + [VARCHAR(500)] + +hostname + + [VARCHAR(500)] + +job_type + + [VARCHAR(30)] + +latest_heartbeat + + [TIMESTAMP] + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +unixname + + [VARCHAR(1000)] partitioned_asset_key_log - -partitioned_asset_key_log - -id - - [INTEGER] - NOT NULL - -asset_event_id - - [INTEGER] - NOT NULL - -asset_id - - [INTEGER] - NOT NULL - -asset_partition_dag_run_id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -source_partition_key - - [VARCHAR(250)] - NOT NULL - -target_dag_id - - [VARCHAR(250)] - NOT NULL - -target_partition_key - - [VARCHAR(250)] - NOT NULL + +partitioned_asset_key_log + +id + + [INTEGER] + NOT NULL + +asset_event_id + + [INTEGER] + NOT NULL + +asset_id + + [INTEGER] + NOT NULL + +asset_partition_dag_run_id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +source_partition_key + + [VARCHAR(250)] + NOT NULL + +target_dag_id + + [VARCHAR(250)] + NOT NULL + +target_partition_key + + [VARCHAR(250)] + NOT NULL log - -log - -id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - -dttm - - [TIMESTAMP] - NOT NULL - -event - - [VARCHAR(60)] - NOT NULL - -extra - - [TEXT] - -logical_date - - [TIMESTAMP] - -map_index - - [INTEGER] - -owner - - [VARCHAR(500)] - -owner_display_name - - [VARCHAR(500)] - -run_id - - [VARCHAR(250)] - -task_id - - [VARCHAR(250)] - -try_number - - [INTEGER] + +log + +id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + +dttm + + [TIMESTAMP] + NOT NULL + +event + + [VARCHAR(60)] + NOT NULL + +extra + + [TEXT] + +logical_date + + [TIMESTAMP] + +map_index + + [INTEGER] + +owner + + [VARCHAR(500)] + +owner_display_name + + [VARCHAR(500)] + +run_id + + [VARCHAR(250)] + +task_id + + [VARCHAR(250)] + +try_number + + [INTEGER] dag_priority_parsing_request - -dag_priority_parsing_request - -id - - [VARCHAR(32)] - NOT NULL - -bundle_name - - [VARCHAR(250)] - NOT NULL - -relative_fileloc - - [VARCHAR(2000)] - NOT NULL + +dag_priority_parsing_request + +id + + [VARCHAR(32)] + NOT NULL + +bundle_name + + [VARCHAR(250)] + NOT NULL + +relative_fileloc + + [VARCHAR(2000)] + NOT NULL import_error - -import_error - -id - - [INTEGER] - NOT NULL - -bundle_name - - [VARCHAR(250)] - -filename - - [VARCHAR(1024)] - -stacktrace - - [TEXT] - -timestamp - - [TIMESTAMP] + +import_error + +id + + [INTEGER] + NOT NULL + +bundle_name + + [VARCHAR(250)] + +filename + + [VARCHAR(1024)] + +stacktrace + + [TEXT] + +timestamp + + [TIMESTAMP] dag_bundle - -dag_bundle - -name - - [VARCHAR(250)] - NOT NULL - -active - - [BOOLEAN] - -last_refreshed - - [TIMESTAMP] - -signed_url_template - - [VARCHAR(200)] - -template_params - - [JSON] - -version - - [VARCHAR(200)] + +dag_bundle + +name + + [VARCHAR(250)] + NOT NULL + +active + + [BOOLEAN] + +last_refreshed + + [TIMESTAMP] + +signed_url_template + + [VARCHAR(200)] + +template_params + + [JSON] + +version + + [VARCHAR(200)] dag_bundle_team - -dag_bundle_team - -dag_bundle_name - - [VARCHAR(250)] - NOT NULL - -team_name - - [VARCHAR(50)] - NOT NULL + +dag_bundle_team + +dag_bundle_name + + [VARCHAR(250)] + NOT NULL + +team_name + + [VARCHAR(50)] + NOT NULL dag_bundle:name--dag_bundle_team:dag_bundle_name - -0..N -1 + +0..N +1 dag - -dag - -dag_id - - [VARCHAR(250)] - NOT NULL - -asset_expression - - [JSON] - -bundle_name - - [VARCHAR(250)] - NOT NULL - -bundle_version - - [VARCHAR(200)] - -dag_display_name - - [VARCHAR(2000)] - -deadline - - [JSON] - -description - - [TEXT] - -exceeds_max_non_backfill - - [BOOLEAN] - NOT NULL - -fail_fast - - [BOOLEAN] - NOT NULL - -fileloc - - [VARCHAR(2000)] - -has_import_errors - - [BOOLEAN] - NOT NULL - -has_task_concurrency_limits - - [BOOLEAN] - NOT NULL - -is_paused - - [BOOLEAN] - NOT NULL - -is_stale - - [BOOLEAN] - NOT NULL - -last_expired - - [TIMESTAMP] - -last_parse_duration - - [DOUBLE PRECISION] - -last_parsed_time - - [TIMESTAMP] - -max_active_runs - - [INTEGER] - -max_active_tasks - - [INTEGER] - NOT NULL - -max_consecutive_failed_dag_runs - - [INTEGER] - NOT NULL - -next_dagrun - - [TIMESTAMP] - -next_dagrun_create_after - - [TIMESTAMP] - -next_dagrun_data_interval_end - - [TIMESTAMP] - -next_dagrun_data_interval_start - - [TIMESTAMP] - -owners - - [VARCHAR(2000)] - -relative_fileloc - - [VARCHAR(2000)] - -timetable_description - - [VARCHAR(1000)] - -timetable_summary - - [TEXT] - -timetable_type - - [VARCHAR(255)] - NOT NULL + +dag + +dag_id + + [VARCHAR(250)] + NOT NULL + +asset_expression + + [JSON] + +bundle_name + + [VARCHAR(250)] + NOT NULL + +bundle_version + + [VARCHAR(200)] + +dag_display_name + + [VARCHAR(2000)] + +deadline + + [JSON] + +description + + [TEXT] + +exceeds_max_non_backfill + + [BOOLEAN] + NOT NULL + +fail_fast + + [BOOLEAN] + NOT NULL + +fileloc + + [VARCHAR(2000)] + +has_import_errors + + [BOOLEAN] + NOT NULL + +has_task_concurrency_limits + + [BOOLEAN] + NOT NULL + +is_paused + + [BOOLEAN] + NOT NULL + +is_stale + + [BOOLEAN] + NOT NULL + +last_expired + + [TIMESTAMP] + +last_parse_duration + + [DOUBLE PRECISION] + +last_parsed_time + + [TIMESTAMP] + +max_active_runs + + [INTEGER] + +max_active_tasks + + [INTEGER] + NOT NULL + +max_consecutive_failed_dag_runs + + [INTEGER] + NOT NULL + +next_dagrun + + [TIMESTAMP] + +next_dagrun_create_after + + [TIMESTAMP] + +next_dagrun_data_interval_end + + [TIMESTAMP] + +next_dagrun_data_interval_start + + [TIMESTAMP] + +owners + + [VARCHAR(2000)] + +relative_fileloc + + [VARCHAR(2000)] + +timetable_description + + [VARCHAR(1000)] + +timetable_summary + + [TEXT] + +timetable_type + + [VARCHAR(255)] + NOT NULL dag_bundle:name--dag:bundle_name - -0..N -1 + +0..N +1 team - -team - -name - - [VARCHAR(50)] - NOT NULL + +team + +name + + [VARCHAR(50)] + NOT NULL team:name--dag_bundle_team:team_name - -0..N -1 + +0..N +1 connection - -connection - -id - - [INTEGER] - NOT NULL - -conn_id - - [VARCHAR(250)] - NOT NULL - -conn_type - - [VARCHAR(500)] - NOT NULL - -description - - [TEXT] - -extra - - [TEXT] - -host - - [VARCHAR(500)] - -is_encrypted - - [BOOLEAN] - NOT NULL - -is_extra_encrypted - - [BOOLEAN] - NOT NULL - -login - - [TEXT] - -password - - [TEXT] - -port - - [INTEGER] - -schema - - [VARCHAR(500)] - -team_name - - [VARCHAR(50)] + +connection + +id + + [INTEGER] + NOT NULL + +conn_id + + [VARCHAR(250)] + NOT NULL + +conn_type + + [VARCHAR(500)] + NOT NULL + +description + + [TEXT] + +extra + + [TEXT] + +host + + [VARCHAR(500)] + +is_encrypted + + [BOOLEAN] + NOT NULL + +is_extra_encrypted + + [BOOLEAN] + NOT NULL + +login + + [TEXT] + +password + + [TEXT] + +port + + [INTEGER] + +schema + + [VARCHAR(500)] + +team_name + + [VARCHAR(50)] team:name--connection:team_name - -0..N -{0,1} + +0..N +{0,1} slot_pool - -slot_pool - -id - - [INTEGER] - NOT NULL - -description - - [TEXT] - -include_deferred - - [BOOLEAN] - NOT NULL - -pool - - [VARCHAR(256)] - NOT NULL - -slots - - [INTEGER] - NOT NULL - -team_name - - [VARCHAR(50)] + +slot_pool + +id + + [INTEGER] + NOT NULL + +description + + [TEXT] + +include_deferred + + [BOOLEAN] + NOT NULL + +pool + + [VARCHAR(256)] + NOT NULL + +slots + + [INTEGER] + NOT NULL + +team_name + + [VARCHAR(50)] team:name--slot_pool:team_name - -0..N -{0,1} + +0..N +{0,1} variable - -variable - -id - - [INTEGER] - NOT NULL - -description - - [TEXT] - -is_encrypted - - [BOOLEAN] - NOT NULL - -key - - [VARCHAR(250)] - NOT NULL - -team_name - - [VARCHAR(50)] - -val - - [TEXT] - NOT NULL + +variable + +id + + [INTEGER] + NOT NULL + +description + + [TEXT] + +is_encrypted + + [BOOLEAN] + NOT NULL + +key + + [VARCHAR(250)] + NOT NULL + +team_name + + [VARCHAR(50)] + +val + + [TEXT] + NOT NULL team:name--variable:team_name - -0..N -{0,1} + +0..N +{0,1} trigger - -trigger - -id - - [INTEGER] - NOT NULL - -classpath - - [VARCHAR(1000)] - NOT NULL - -created_date - - [TIMESTAMP] - NOT NULL - -kwargs - - [TEXT] - NOT NULL - -queue - - [VARCHAR(256)] - -triggerer_id - - [INTEGER] + +trigger + +id + + [INTEGER] + NOT NULL + +classpath + + [VARCHAR(1000)] + NOT NULL + +created_date + + [TIMESTAMP] + NOT NULL + +kwargs + + [TEXT] + NOT NULL + +queue + + [VARCHAR(256)] + +triggerer_id + + [INTEGER] callback - -callback - -id - - [UUID] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -data - - [JSONB] - NOT NULL - -fetch_method - - [VARCHAR(20)] - NOT NULL - -output - - [TEXT] - -priority_weight - - [INTEGER] - NOT NULL - -state - - [VARCHAR(10)] - -trigger_id - - [INTEGER] - -type - - [VARCHAR(20)] - NOT NULL + +callback + +id + + [UUID] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +data + + [JSONB] + NOT NULL + +fetch_method + + [VARCHAR(20)] + NOT NULL + +output + + [TEXT] + +priority_weight + + [INTEGER] + NOT NULL + +state + + [VARCHAR(10)] + +trigger_id + + [INTEGER] + +type + + [VARCHAR(20)] + NOT NULL trigger:id--callback:trigger_id - -0..N -{0,1} + +0..N +{0,1} asset_watcher - -asset_watcher - -asset_id - - [INTEGER] - NOT NULL - -trigger_id - - [INTEGER] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL + +asset_watcher + +asset_id + + [INTEGER] + NOT NULL + +trigger_id + + [INTEGER] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL - + trigger:id--asset_watcher:trigger_id - -0..N -1 + +0..N +1 - + task_instance - -task_instance - -id - - [UUID] - NOT NULL - -context_carrier - - [JSONB] - -custom_operator_name - - [VARCHAR(1000)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - -duration - - [DOUBLE PRECISION] - -end_date - - [TIMESTAMP] - -executor - - [VARCHAR(1000)] - -executor_config - - [BYTEA] - NOT NULL - -external_executor_id - - [VARCHAR(250)] - -hostname - - [VARCHAR(1000)] - NOT NULL - -last_heartbeat_at - - [TIMESTAMP] - -map_index - - [INTEGER] - NOT NULL - -max_tries - - [INTEGER] - NOT NULL + +task_instance -next_kwargs - - [JSONB] +id + + [UUID] + NOT NULL -next_method - - [VARCHAR(1000)] +context_carrier + + [JSONB] -operator - - [VARCHAR(1000)] +custom_operator_name + + [VARCHAR(1000)] + NOT NULL -pid - - [INTEGER] +dag_id + + [VARCHAR(250)] + NOT NULL -pool - - [VARCHAR(256)] - NOT NULL +dag_version_id + + [UUID] -pool_slots - - [INTEGER] - NOT NULL +duration + + [DOUBLE PRECISION] -priority_weight - - [INTEGER] - NOT NULL +end_date + + [TIMESTAMP] -queue - - [VARCHAR(256)] - NOT NULL +executor + + [VARCHAR(1000)] -queued_by_job_id - - [INTEGER] +executor_config + + [BYTEA] + NOT NULL -queued_dttm - - [TIMESTAMP] +external_executor_id + + [VARCHAR(250)] -rendered_map_index - - [VARCHAR(250)] +hostname + + [VARCHAR(1000)] + NOT NULL -run_id - - [VARCHAR(250)] - NOT NULL +last_heartbeat_at + + [TIMESTAMP] -scheduled_dttm - - [TIMESTAMP] +map_index + + [INTEGER] + NOT NULL -span_status - - [VARCHAR(250)] - NOT NULL +max_tries + + [INTEGER] + NOT NULL -start_date - - [TIMESTAMP] +next_kwargs + + [JSONB] -state - - [VARCHAR(20)] +next_method + + [VARCHAR(1000)] -task_display_name - - [VARCHAR(2000)] +operator + + [VARCHAR(1000)] -task_id - - [VARCHAR(250)] - NOT NULL +pid + + [INTEGER] -trigger_id - - [INTEGER] +pool + + [VARCHAR(256)] + NOT NULL -trigger_timeout - - [TIMESTAMP] +pool_slots + + [INTEGER] + NOT NULL -try_number - - [INTEGER] - NOT NULL +priority_weight + + [INTEGER] + NOT NULL -unixname - - [VARCHAR(1000)] - NOT NULL +queue + + [VARCHAR(256)] + NOT NULL -updated_at - - [TIMESTAMP] +queued_by_job_id + + [INTEGER] + +queued_dttm + + [TIMESTAMP] + +rendered_map_index + + [VARCHAR(250)] + +run_id + + [VARCHAR(250)] + NOT NULL + +scheduled_dttm + + [TIMESTAMP] + +span_status + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +task_display_name + + [VARCHAR(2000)] + +task_id + + [VARCHAR(250)] + NOT NULL + +trigger_id + + [INTEGER] + +trigger_timeout + + [TIMESTAMP] + +try_number + + [INTEGER] + NOT NULL + +unixname + + [VARCHAR(1000)] + NOT NULL + +updated_at + + [TIMESTAMP] - + trigger:id--task_instance:trigger_id - -0..N -{0,1} + +0..N +{0,1} - + deadline - -deadline - -id - - [UUID] - NOT NULL - -callback_id - - [UUID] - NOT NULL - -dagrun_id - - [INTEGER] - -deadline_time - - [TIMESTAMP] - NOT NULL - -missed - - [BOOLEAN] - NOT NULL + +deadline + +id + + [UUID] + NOT NULL + +callback_id + + [UUID] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +dagrun_id + + [INTEGER] + +deadline_alert_id + + [UUID] + +deadline_time + + [TIMESTAMP] + NOT NULL + +last_updated_at + + [TIMESTAMP] + NOT NULL + +missed + + [BOOLEAN] + NOT NULL - + callback:id--deadline:callback_id - -0..N -1 + +0..N +1 asset_alias - -asset_alias - -id - - [INTEGER] - NOT NULL - -group - - [VARCHAR(1500)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL + +asset_alias + +id + + [INTEGER] + NOT NULL + +group + + [VARCHAR(1500)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL asset_alias_asset - -asset_alias_asset - -alias_id - - [INTEGER] - NOT NULL - -asset_id - - [INTEGER] - NOT NULL + +asset_alias_asset + +alias_id + + [INTEGER] + NOT NULL + +asset_id + + [INTEGER] + NOT NULL - + asset_alias:id--asset_alias_asset:alias_id - -0..N -1 + +0..N +1 asset_alias_asset_event - -asset_alias_asset_event - -alias_id - - [INTEGER] - NOT NULL - -event_id - - [INTEGER] - NOT NULL + +asset_alias_asset_event + +alias_id + + [INTEGER] + NOT NULL + +event_id + + [INTEGER] + NOT NULL - + asset_alias:id--asset_alias_asset_event:alias_id - -0..N -1 + +0..N +1 dag_schedule_asset_alias_reference - -dag_schedule_asset_alias_reference - -alias_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_alias_reference + +alias_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL asset_alias:id--dag_schedule_asset_alias_reference:alias_id - -0..N -1 + +0..N +1 asset - -asset - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -extra - - [JSON] - NOT NULL - -group - - [VARCHAR(1500)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL + +asset + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +extra + + [JSON] + NOT NULL + +group + + [VARCHAR(1500)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL - + asset:id--asset_alias_asset:asset_id - -0..N -1 + +0..N +1 - + asset:id--asset_watcher:asset_id - -0..N -1 + +0..N +1 asset_active - -asset_active - -name - - [VARCHAR(1500)] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL + +asset_active + +name + + [VARCHAR(1500)] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL asset:name--asset_active:name - -1 -1 + +1 +1 asset:uri--asset_active:uri - -1 -1 + +1 +1 dag_schedule_asset_reference - -dag_schedule_asset_reference - -asset_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_reference + +asset_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL - + asset:id--dag_schedule_asset_reference:asset_id - -0..N -1 + +0..N +1 task_outlet_asset_reference - -task_outlet_asset_reference - -asset_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +task_outlet_asset_reference + +asset_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL asset:id--task_outlet_asset_reference:asset_id - -0..N -1 + +0..N +1 task_inlet_asset_reference - -task_inlet_asset_reference - -asset_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +task_inlet_asset_reference + +asset_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL - + asset:id--task_inlet_asset_reference:asset_id - -0..N -1 + +0..N +1 asset_dag_run_queue - -asset_dag_run_queue - -asset_id - - [INTEGER] - NOT NULL - -target_dag_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL + +asset_dag_run_queue + +asset_id + + [INTEGER] + NOT NULL + +target_dag_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL - + asset:id--asset_dag_run_queue:asset_id - -0..N -1 + +0..N +1 asset_event - -asset_event - -id - - [INTEGER] - NOT NULL - -asset_id - - [INTEGER] - NOT NULL - -extra - - [JSON] - NOT NULL - -partition_key - - [VARCHAR(250)] - -source_dag_id - - [VARCHAR(250)] - -source_map_index - - [INTEGER] - -source_run_id - - [VARCHAR(250)] - -source_task_id - - [VARCHAR(250)] - -timestamp - - [TIMESTAMP] - NOT NULL + +asset_event + +id + + [INTEGER] + NOT NULL + +asset_id + + [INTEGER] + NOT NULL + +extra + + [JSON] + NOT NULL + +partition_key + + [VARCHAR(250)] + +source_dag_id + + [VARCHAR(250)] + +source_map_index + + [INTEGER] + +source_run_id + + [VARCHAR(250)] + +source_task_id + + [VARCHAR(250)] + +timestamp + + [TIMESTAMP] + NOT NULL - + asset_event:id--asset_alias_asset_event:event_id - -0..N -1 + +0..N +1 - + dagrun_asset_event - -dagrun_asset_event - -dag_run_id - - [INTEGER] - NOT NULL - -event_id - - [INTEGER] - NOT NULL + +dagrun_asset_event + +dag_run_id + + [INTEGER] + NOT NULL + +event_id + + [INTEGER] + NOT NULL - + asset_event:id--dagrun_asset_event:event_id - -0..N -1 + +0..N +1 dag_schedule_asset_name_reference - -dag_schedule_asset_name_reference - -dag_id - - [VARCHAR(250)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_name_reference + +dag_id + + [VARCHAR(250)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL dag:dag_id--dag_schedule_asset_name_reference:dag_id - -0..N -1 + +0..N +1 dag_schedule_asset_uri_reference - -dag_schedule_asset_uri_reference - -dag_id - - [VARCHAR(250)] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_uri_reference + +dag_id + + [VARCHAR(250)] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL dag:dag_id--dag_schedule_asset_uri_reference:dag_id - -0..N -1 + +0..N +1 dag:dag_id--dag_schedule_asset_alias_reference:dag_id - -0..N -1 + +0..N +1 - + dag:dag_id--dag_schedule_asset_reference:dag_id - -0..N -1 + +0..N +1 dag:dag_id--task_outlet_asset_reference:dag_id - -0..N -1 + +0..N +1 - + dag:dag_id--task_inlet_asset_reference:dag_id - -0..N -1 + +0..N +1 - + dag:dag_id--asset_dag_run_queue:target_dag_id - -0..N -1 + +0..N +1 dag_version - -dag_version - -id - - [UUID] - NOT NULL - -bundle_name - - [VARCHAR(250)] - -bundle_version - - [VARCHAR(250)] - -created_at - - [TIMESTAMP] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -last_updated - - [TIMESTAMP] - NOT NULL - -version_number - - [INTEGER] - NOT NULL + +dag_version + +id + + [UUID] + NOT NULL + +bundle_name + + [VARCHAR(250)] + +bundle_version + + [VARCHAR(250)] + +created_at + + [TIMESTAMP] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +last_updated + + [TIMESTAMP] + NOT NULL + +version_number + + [INTEGER] + NOT NULL dag:dag_id--dag_version:dag_id - -0..N -1 + +0..N +1 dag_tag - -dag_tag - -dag_id - - [VARCHAR(250)] - NOT NULL - -name - - [VARCHAR(100)] - NOT NULL + +dag_tag + +dag_id + + [VARCHAR(250)] + NOT NULL + +name + + [VARCHAR(100)] + NOT NULL dag:dag_id--dag_tag:dag_id - -0..N -1 + +0..N +1 dag_owner_attributes - -dag_owner_attributes - -dag_id - - [VARCHAR(250)] - NOT NULL - -owner - - [VARCHAR(500)] - NOT NULL - -link - - [VARCHAR(500)] - NOT NULL + +dag_owner_attributes + +dag_id + + [VARCHAR(250)] + NOT NULL + +owner + + [VARCHAR(500)] + NOT NULL + +link + + [VARCHAR(500)] + NOT NULL dag:dag_id--dag_owner_attributes:dag_id - -0..N -1 + +0..N +1 dag_warning - -dag_warning - -dag_id - - [VARCHAR(250)] - NOT NULL - -warning_type - - [VARCHAR(50)] - NOT NULL - -message - - [TEXT] - NOT NULL - -timestamp - - [TIMESTAMP] - NOT NULL + +dag_warning + +dag_id + + [VARCHAR(250)] + NOT NULL + +warning_type + + [VARCHAR(50)] + NOT NULL + +message + + [TEXT] + NOT NULL + +timestamp + + [TIMESTAMP] + NOT NULL dag:dag_id--dag_warning:dag_id - -0..N -1 + +0..N +1 dag_favorite - -dag_favorite - -dag_id - - [VARCHAR(250)] - NOT NULL - -user_id - - [VARCHAR(250)] - NOT NULL + +dag_favorite + +dag_id + + [VARCHAR(250)] + NOT NULL + +user_id + + [VARCHAR(250)] + NOT NULL dag:dag_id--dag_favorite:dag_id - -0..N -1 + +0..N +1 dag_run - -dag_run - -id - - [INTEGER] - NOT NULL - -backfill_id - - [INTEGER] - -bundle_version - - [VARCHAR(250)] - -clear_number - - [INTEGER] - NOT NULL - -conf - - [JSONB] - -context_carrier - - [JSONB] - -created_dag_version_id - - [UUID] - -creating_job_id - - [INTEGER] - -dag_id - - [VARCHAR(250)] - NOT NULL - -data_interval_end - - [TIMESTAMP] - -data_interval_start - - [TIMESTAMP] - -end_date - - [TIMESTAMP] - -last_scheduling_decision - - [TIMESTAMP] - -log_template_id - - [INTEGER] - NOT NULL - -logical_date - - [TIMESTAMP] - -partition_key - - [VARCHAR(250)] - -queued_at - - [TIMESTAMP] - -run_after - - [TIMESTAMP] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -run_type - - [VARCHAR(50)] - NOT NULL - -scheduled_by_job_id - - [INTEGER] - -span_status - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(50)] - NOT NULL - -triggered_by - - [VARCHAR(50)] - -triggering_user_name - - [VARCHAR(512)] - -updated_at - - [TIMESTAMP] - NOT NULL + +dag_run + +id + + [INTEGER] + NOT NULL + +backfill_id + + [INTEGER] + +bundle_version + + [VARCHAR(250)] + +clear_number + + [INTEGER] + NOT NULL + +conf + + [JSONB] + +context_carrier + + [JSONB] + +created_dag_version_id + + [UUID] + +creating_job_id + + [INTEGER] + +dag_id + + [VARCHAR(250)] + NOT NULL + +data_interval_end + + [TIMESTAMP] + +data_interval_start + + [TIMESTAMP] + +end_date + + [TIMESTAMP] + +last_scheduling_decision + + [TIMESTAMP] + +log_template_id + + [INTEGER] + NOT NULL + +logical_date + + [TIMESTAMP] + +partition_key + + [VARCHAR(250)] + +queued_at + + [TIMESTAMP] + +run_after + + [TIMESTAMP] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +run_type + + [VARCHAR(50)] + NOT NULL + +scheduled_by_job_id + + [INTEGER] + +span_status + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(50)] + NOT NULL + +triggered_by + + [VARCHAR(50)] + +triggering_user_name + + [VARCHAR(512)] + +updated_at + + [TIMESTAMP] + NOT NULL - + dag_version:id--dag_run:created_dag_version_id - -0..N -{0,1} + +0..N +{0,1} dag_code - -dag_code - -id - - [UUID] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - NOT NULL - -fileloc - - [VARCHAR(2000)] - NOT NULL - -last_updated - - [TIMESTAMP] - NOT NULL - -source_code - - [TEXT] - NOT NULL - -source_code_hash - - [VARCHAR(32)] - NOT NULL + +dag_code + +id + + [UUID] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + NOT NULL + +fileloc + + [VARCHAR(2000)] + NOT NULL + +last_updated + + [TIMESTAMP] + NOT NULL + +source_code + + [TEXT] + NOT NULL + +source_code_hash + + [VARCHAR(32)] + NOT NULL dag_version:id--dag_code:dag_version_id - -0..N -1 + +0..N +1 serialized_dag - -serialized_dag - -id - - [UUID] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -dag_hash - - [VARCHAR(32)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - NOT NULL - -data - - [JSONB] - -data_compressed - - [BYTEA] - -last_updated - - [TIMESTAMP] - NOT NULL + +serialized_dag + +id + + [UUID] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +dag_hash + + [VARCHAR(32)] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + NOT NULL + +data + + [JSONB] + +data_compressed + + [BYTEA] + +last_updated + + [TIMESTAMP] + NOT NULL dag_version:id--serialized_dag:dag_version_id - -0..N -1 + +0..N +1 - + dag_version:id--task_instance:dag_version_id - -0..N -{0,1} + +0..N +{0,1} log_template - -log_template - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -elasticsearch_id - - [TEXT] - NOT NULL - -filename - - [TEXT] - NOT NULL + +log_template + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +elasticsearch_id + + [TEXT] + NOT NULL + +filename + + [TEXT] + NOT NULL - + log_template:id--dag_run:log_template_id - -0..N -1 - - - -dag_run:id--deadline:dagrun_id - -0..N -{0,1} + +0..N +1 - + dag_run:id--dagrun_asset_event:dag_run_id - -0..N -1 + +0..N +1 - + asset_partition_dag_run - -asset_partition_dag_run - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -created_dag_run_id - - [INTEGER] - -partition_key - - [VARCHAR(250)] - NOT NULL - -target_dag_id - - [VARCHAR(250)] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +asset_partition_dag_run + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +created_dag_run_id + + [INTEGER] + +partition_key + + [VARCHAR(250)] + NOT NULL + +target_dag_id + + [VARCHAR(250)] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL - + dag_run:id--asset_partition_dag_run:created_dag_run_id - -0..N -{0,1} + +0..N +{0,1} - -dag_run:run_id--task_instance:run_id - -0..N -1 + +dag_run:dag_id--task_instance:dag_id + +0..N +1 - -dag_run:dag_id--task_instance:dag_id - -0..N -1 + +dag_run:run_id--task_instance:run_id + +0..N +1 - + backfill_dag_run - -backfill_dag_run - -id - - [INTEGER] - NOT NULL - -backfill_id - - [INTEGER] - NOT NULL - -dag_run_id - - [INTEGER] - -exception_reason - - [VARCHAR(250)] - -logical_date - - [TIMESTAMP] - NOT NULL - -sort_ordinal - - [INTEGER] - NOT NULL + +backfill_dag_run + +id + + [INTEGER] + NOT NULL + +backfill_id + + [INTEGER] + NOT NULL + +dag_run_id + + [INTEGER] + +exception_reason + + [VARCHAR(250)] + +logical_date + + [TIMESTAMP] + NOT NULL + +sort_ordinal + + [INTEGER] + NOT NULL - + dag_run:id--backfill_dag_run:dag_run_id - -0..N -{0,1} + +0..N +{0,1} dag_run_note - -dag_run_note - -dag_run_id - - [INTEGER] - NOT NULL - -content - - [VARCHAR(1000)] - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -user_id - - [VARCHAR(128)] + +dag_run_note + +dag_run_id + + [INTEGER] + NOT NULL + +content + + [VARCHAR(1000)] + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +user_id + + [VARCHAR(128)] - + dag_run:id--dag_run_note:dag_run_id - -1 -1 + +1 +1 + + + +dag_run:id--deadline:dagrun_id + +0..N +{0,1} backfill - -backfill - -id - - [INTEGER] - NOT NULL - -completed_at - - [TIMESTAMP] - -created_at - - [TIMESTAMP] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_run_conf - - [JSON] - NOT NULL - -from_date - - [TIMESTAMP] - NOT NULL - -is_paused - - [BOOLEAN] - -max_active_runs - - [INTEGER] - NOT NULL - -reprocess_behavior - - [VARCHAR(250)] - NOT NULL - -to_date - - [TIMESTAMP] - NOT NULL - -triggering_user_name - - [VARCHAR(512)] - -updated_at - - [TIMESTAMP] - NOT NULL + +backfill + +id + + [INTEGER] + NOT NULL + +completed_at + + [TIMESTAMP] + +created_at + + [TIMESTAMP] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_run_conf + + [JSON] + NOT NULL + +from_date + + [TIMESTAMP] + NOT NULL + +is_paused + + [BOOLEAN] + +max_active_runs + + [INTEGER] + NOT NULL + +reprocess_behavior + + [VARCHAR(250)] + NOT NULL + +to_date + + [TIMESTAMP] + NOT NULL + +triggering_user_name + + [VARCHAR(512)] + +updated_at + + [TIMESTAMP] + NOT NULL backfill:id--dag_run:backfill_id - -0..N -{0,1} + +0..N +{0,1} - + backfill:id--backfill_dag_run:backfill_id - -0..N -1 + +0..N +1 + + + +deadline_alert + +deadline_alert + +id + + [UUID] + NOT NULL + +callback_def + + [JSON] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +description + + [TEXT] + +interval + + [DOUBLE PRECISION] + NOT NULL + +name + + [VARCHAR(250)] + +reference + + [JSON] + NOT NULL + +serialized_dag_id + + [UUID] + NOT NULL + + + +serialized_dag:id--deadline_alert:serialized_dag_id + +0..N +1 - + hitl_detail - -hitl_detail - -ti_id - - [UUID] - NOT NULL - -assignees - - [JSON] - -body - - [TEXT] - -chosen_options - - [JSON] - -created_at - - [TIMESTAMP] - NOT NULL - -defaults - - [JSON] - -multiple - - [BOOLEAN] - -options - - [JSON] - NOT NULL - -params - - [JSON] - NOT NULL - -params_input - - [JSON] - NOT NULL - -responded_at - - [TIMESTAMP] - -responded_by - - [JSON] - -subject - - [TEXT] - NOT NULL + +hitl_detail + +ti_id + + [UUID] + NOT NULL + +assignees + + [JSON] + +body + + [TEXT] + +chosen_options + + [JSON] + +created_at + + [TIMESTAMP] + NOT NULL + +defaults + + [JSON] + +multiple + + [BOOLEAN] + +options + + [JSON] + NOT NULL + +params + + [JSON] + NOT NULL + +params_input + + [JSON] + NOT NULL + +responded_at + + [TIMESTAMP] + +responded_by + + [JSON] + +subject + + [TEXT] + NOT NULL - + task_instance:id--hitl_detail:ti_id - -1 -1 + +1 +1 - + task_map - -task_map - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -keys - - [JSONB] - -length - - [INTEGER] - NOT NULL - - - -task_instance:dag_id--task_map:dag_id - -0..N -1 - - - -task_instance:task_id--task_map:task_id - -0..N -1 + +task_map + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +keys + + [JSONB] + +length + + [INTEGER] + NOT NULL task_instance:map_index--task_map:map_index - -0..N -1 + +0..N +1 task_instance:run_id--task_map:run_id - -0..N -1 + +0..N +1 + + + +task_instance:task_id--task_map:task_id + +0..N +1 + + + +task_instance:dag_id--task_map:dag_id + +0..N +1 - + task_reschedule - -task_reschedule - -id - - [INTEGER] - NOT NULL - -duration - - [INTEGER] - NOT NULL - -end_date - - [TIMESTAMP] - NOT NULL - -reschedule_date - - [TIMESTAMP] - NOT NULL - -start_date - - [TIMESTAMP] - NOT NULL - -ti_id - - [UUID] - NOT NULL + +task_reschedule + +id + + [INTEGER] + NOT NULL + +duration + + [INTEGER] + NOT NULL + +end_date + + [TIMESTAMP] + NOT NULL + +reschedule_date + + [TIMESTAMP] + NOT NULL + +start_date + + [TIMESTAMP] + NOT NULL + +ti_id + + [UUID] + NOT NULL - + task_instance:id--task_reschedule:ti_id - -0..N -1 + +0..N +1 - + xcom - -xcom - -dag_run_id - - [INTEGER] - NOT NULL - -key - - [VARCHAR(512)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -timestamp - - [TIMESTAMP] - NOT NULL - -value - - [JSONB] + +xcom + +dag_run_id + + [INTEGER] + NOT NULL + +key + + [VARCHAR(512)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +timestamp + + [TIMESTAMP] + NOT NULL + +value + + [JSONB] - -task_instance:run_id--xcom:run_id - -0..N -1 - - - + task_instance:task_id--xcom:task_id - -0..N -1 + +0..N +1 - + task_instance:dag_id--xcom:dag_id - -0..N -1 + +0..N +1 - + task_instance:map_index--xcom:map_index - -0..N -1 + +0..N +1 + + + +task_instance:run_id--xcom:run_id + +0..N +1 - + task_instance_note - -task_instance_note - -ti_id - - [UUID] - NOT NULL - -content - - [VARCHAR(1000)] - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -user_id - - [VARCHAR(128)] + +task_instance_note + +ti_id + + [UUID] + NOT NULL + +content + + [VARCHAR(1000)] + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +user_id + + [VARCHAR(128)] - + task_instance:id--task_instance_note:ti_id - -1 -1 + +1 +1 - + task_instance_history - -task_instance_history - -task_instance_id - - [UUID] - NOT NULL - -context_carrier - - [JSONB] - -custom_operator_name - - [VARCHAR(1000)] - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - -duration - - [DOUBLE PRECISION] - -end_date - - [TIMESTAMP] - -executor - - [VARCHAR(1000)] - -executor_config - - [BYTEA] - -external_executor_id - - [VARCHAR(250)] - -hostname - - [VARCHAR(1000)] - -map_index - - [INTEGER] - NOT NULL - -max_tries - - [INTEGER] - -next_kwargs - - [JSONB] - -next_method - - [VARCHAR(1000)] - -operator - - [VARCHAR(1000)] - -pid - - [INTEGER] - -pool - - [VARCHAR(256)] - NOT NULL - -pool_slots - - [INTEGER] - NOT NULL - -priority_weight - - [INTEGER] - -queue - - [VARCHAR(256)] - -queued_by_job_id - - [INTEGER] - -queued_dttm - - [TIMESTAMP] - -rendered_map_index - - [VARCHAR(250)] - -run_id - - [VARCHAR(250)] - NOT NULL - -scheduled_dttm - - [TIMESTAMP] - -span_status - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -task_display_name - - [VARCHAR(2000)] - -task_id - - [VARCHAR(250)] - NOT NULL - -trigger_id - - [INTEGER] - -trigger_timeout - - [TIMESTAMP] - -try_number - - [INTEGER] - NOT NULL - -unixname - - [VARCHAR(1000)] - -updated_at - - [TIMESTAMP] + +task_instance_history + +task_instance_id + + [UUID] + NOT NULL + +context_carrier + + [JSONB] + +custom_operator_name + + [VARCHAR(1000)] + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + +duration + + [DOUBLE PRECISION] + +end_date + + [TIMESTAMP] + +executor + + [VARCHAR(1000)] + +executor_config + + [BYTEA] + +external_executor_id + + [VARCHAR(250)] + +hostname + + [VARCHAR(1000)] + +map_index + + [INTEGER] + NOT NULL + +max_tries + + [INTEGER] + +next_kwargs + + [JSONB] + +next_method + + [VARCHAR(1000)] + +operator + + [VARCHAR(1000)] + +pid + + [INTEGER] + +pool + + [VARCHAR(256)] + NOT NULL + +pool_slots + + [INTEGER] + NOT NULL + +priority_weight + + [INTEGER] + +queue + + [VARCHAR(256)] + +queued_by_job_id + + [INTEGER] + +queued_dttm + + [TIMESTAMP] + +rendered_map_index + + [VARCHAR(250)] + +run_id + + [VARCHAR(250)] + NOT NULL + +scheduled_dttm + + [TIMESTAMP] + +span_status + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +task_display_name + + [VARCHAR(2000)] + +task_id + + [VARCHAR(250)] + NOT NULL + +trigger_id + + [INTEGER] + +trigger_timeout + + [TIMESTAMP] + +try_number + + [INTEGER] + NOT NULL + +unixname + + [VARCHAR(1000)] + +updated_at + + [TIMESTAMP] - -task_instance:run_id--task_instance_history:run_id - -0..N -1 + +task_instance:dag_id--task_instance_history:dag_id + +0..N +1 - -task_instance:task_id--task_instance_history:task_id - -0..N -1 + +task_instance:run_id--task_instance_history:run_id + +0..N +1 - + task_instance:map_index--task_instance_history:map_index - -0..N -1 + +0..N +1 - -task_instance:dag_id--task_instance_history:dag_id - -0..N -1 + +task_instance:task_id--task_instance_history:task_id + +0..N +1 - + rendered_task_instance_fields - -rendered_task_instance_fields - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -k8s_pod_yaml - - [JSON] - -rendered_fields - - [JSON] - NOT NULL + +rendered_task_instance_fields + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +k8s_pod_yaml + + [JSON] + +rendered_fields + + [JSON] + NOT NULL - + task_instance:map_index--rendered_task_instance_fields:map_index - -0..N -1 + +0..N +1 - + task_instance:task_id--rendered_task_instance_fields:task_id - -0..N -1 + +0..N +1 - -task_instance:run_id--rendered_task_instance_fields:run_id - -0..N -1 + +task_instance:dag_id--rendered_task_instance_fields:dag_id + +0..N +1 - -task_instance:dag_id--rendered_task_instance_fields:dag_id - -0..N -1 + +task_instance:run_id--rendered_task_instance_fields:run_id + +0..N +1 + + + +deadline_alert:id--deadline:deadline_alert_id + +0..N +{0,1} - + hitl_detail_history - -hitl_detail_history - -ti_history_id - - [UUID] - NOT NULL - -assignees - - [JSON] - -body - - [TEXT] - -chosen_options - - [JSON] - -created_at - - [TIMESTAMP] - NOT NULL - -defaults - - [JSON] - -multiple - - [BOOLEAN] - -options - - [JSON] - NOT NULL - -params - - [JSON] - NOT NULL - -params_input - - [JSON] - NOT NULL - -responded_at - - [TIMESTAMP] - -responded_by - - [JSON] - -subject - - [TEXT] - NOT NULL + +hitl_detail_history + +ti_history_id + + [UUID] + NOT NULL + +assignees + + [JSON] + +body + + [TEXT] + +chosen_options + + [JSON] + +created_at + + [TIMESTAMP] + NOT NULL + +defaults + + [JSON] + +multiple + + [BOOLEAN] + +options + + [JSON] + NOT NULL + +params + + [JSON] + NOT NULL + +params_input + + [JSON] + NOT NULL + +responded_at + + [TIMESTAMP] + +responded_by + + [JSON] + +subject + + [TEXT] + NOT NULL - + task_instance_history:task_instance_id--hitl_detail_history:ti_history_id - -1 -1 + +1 +1 - + alembic_version - -alembic_version - -version_num - - [VARCHAR(32)] - NOT NULL + +alembic_version + +version_num + + [VARCHAR(32)] + NOT NULL diff --git a/airflow-core/docs/migrations-ref.rst b/airflow-core/docs/migrations-ref.rst index 3b2080410e228..dc0be922e079d 100644 --- a/airflow-core/docs/migrations-ref.rst +++ b/airflow-core/docs/migrations-ref.rst @@ -39,7 +39,10 @@ Here's the list of all the Database Migrations that are executed via when you ru +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | Revision ID | Revises ID | Airflow Version | Description | +=========================+==================+===================+==============================================================+ -| ``e79fc784f145`` (head) | ``0b112f49112d`` | ``3.2.0`` | add timetable_type to dag table for filtering. | +| ``55297ae24532`` (head) | ``e79fc784f145`` | ``3.2.0`` | Add required fields to enable UI integrations for the | +| | | | Deadline Alerts feature. | ++-------------------------+------------------+-------------------+--------------------------------------------------------------+ +| ``e79fc784f145`` | ``0b112f49112d`` | ``3.2.0`` | add timetable_type to dag table for filtering. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ | ``0b112f49112d`` | ``c47f2e1ab9d4`` | ``3.2.0`` | Add exceeds max runs flag to dag model. | +-------------------------+------------------+-------------------+--------------------------------------------------------------+ diff --git a/airflow-core/src/airflow/migrations/versions/0098_3_2_0_ui_improvements_for_deadlines.py b/airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py similarity index 99% rename from airflow-core/src/airflow/migrations/versions/0098_3_2_0_ui_improvements_for_deadlines.py rename to airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py index d6c212c355ddd..f5b6307a12270 100644 --- a/airflow-core/src/airflow/migrations/versions/0098_3_2_0_ui_improvements_for_deadlines.py +++ b/airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py @@ -24,7 +24,7 @@ into the new normalized table structure. Revision ID: 55297ae24532 -Revises: 0b112f49112d +Revises: e79fc784f145 Create Date: 2025-10-17 16:04:55.016272 """ @@ -53,7 +53,7 @@ ErrorDict = dict[str, list[str]] revision = "55297ae24532" -down_revision = "0b112f49112d" +down_revision = "e79fc784f145" branch_labels = None depends_on = None airflow_version = "3.2.0" diff --git a/airflow-core/src/airflow/utils/db.py b/airflow-core/src/airflow/utils/db.py index d444689e26b56..d318d3a1b8206 100644 --- a/airflow-core/src/airflow/utils/db.py +++ b/airflow-core/src/airflow/utils/db.py @@ -112,7 +112,7 @@ class MappedClassProtocol(Protocol): "3.0.0": "29ce7909c52b", "3.0.3": "fe199e1abd77", "3.1.0": "cc92b33c6709", - "3.2.0": "e79fc784f145", + "3.2.0": "55297ae24532", } # Prefix used to identify tables holding data moved during migration. From df13101b0a82b03678d6090d7b595f969b9019ea Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Mon, 12 Jan 2026 15:16:51 -0800 Subject: [PATCH 09/14] static checks - add more defensive programming --- airflow-core/src/airflow/models/serialized_dag.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/airflow-core/src/airflow/models/serialized_dag.py b/airflow-core/src/airflow/models/serialized_dag.py index 2fe3528b5c598..a019ed6377f45 100644 --- a/airflow-core/src/airflow/models/serialized_dag.py +++ b/airflow-core/src/airflow/models/serialized_dag.py @@ -498,8 +498,10 @@ def write_dag( select(cls).where(cls.dag_id == dag.dag_id).order_by(cls.created_at.desc()).limit(1) ) - if existing_serialized_dag and ( - existing_deadline_uuids := existing_serialized_dag.data.get("dag", {}).get("deadline") + if ( + existing_serialized_dag + and existing_serialized_dag.data + and (existing_deadline_uuids := existing_serialized_dag.data.get("dag", {}).get("deadline")) ): dag.data["dag"]["deadline"] = existing_deadline_uuids deadline_uuid_mapping = {} From 242d888e5e1045d13fe28c9f4c932543928ee77d Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Tue, 13 Jan 2026 09:38:22 -0800 Subject: [PATCH 10/14] missed a helper when I ported the tests over --- devel-common/src/tests_common/test_utils/db.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/devel-common/src/tests_common/test_utils/db.py b/devel-common/src/tests_common/test_utils/db.py index f154c601959d9..d29cc06141be9 100644 --- a/devel-common/src/tests_common/test_utils/db.py +++ b/devel-common/src/tests_common/test_utils/db.py @@ -280,6 +280,14 @@ def clear_db_deadline(): session.execute(delete(Deadline)) +def clear_db_deadline_alert(): + with create_session() as session: + if AIRFLOW_V_3_2_PLUS: + from airflow.models.deadline_alert import DeadlineAlert + + session.execute(delete(DeadlineAlert)) + + def drop_tables_with_prefix(prefix): with create_session() as session: metadata = reflect_tables(None, session) From 11e5ab23faa7e79fbc96d8656f902675225e05a2 Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Tue, 13 Jan 2026 15:45:30 -0800 Subject: [PATCH 11/14] Remove model import per new CI test --- ...099_3_2_0_ui_improvements_for_deadlines.py | 48 +++++++++++++++++-- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py b/airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py index f5b6307a12270..93c1a225d5294 100644 --- a/airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py +++ b/airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py @@ -33,7 +33,7 @@ import json import zlib from collections import defaultdict -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Iterable import sqlalchemy as sa import uuid6 @@ -43,6 +43,7 @@ from airflow._shared.timezones import timezone from airflow.configuration import conf from airflow.serialization.enums import Encoding +from airflow.utils.hashlib_wrapper import md5 from airflow.utils.sqlalchemy import UtcDateTime if TYPE_CHECKING: @@ -310,6 +311,47 @@ def report_errors(errors: ErrorDict, operation: str = "migration") -> None: print(f"No Dags encountered errors during {operation}.") +def hash_dag(dag_data): + """ + Hash the data to get the dag_hash. + + Copied from airflow.models.serialized_dag.SerializedDagModel since we can't import it anymore. + """ + dag_data = _sort_serialized_dag_dict(dag_data) + data_ = dag_data.copy() + # Remove fileloc from the hash so changes to fileloc + # does not affect the hash. In 3.0+, a combination of + # bundle_path and relative fileloc more correctly determines the + # dag file location. + data_["dag"].pop("fileloc", None) + data_json = json.dumps(data_, sort_keys=True).encode("utf-8") + return md5(data_json).hexdigest() + + +def _sort_serialized_dag_dict(serialized_dag: Any): + """ + Recursively sort json_dict and its nested dictionaries and lists. + + Copied from airflow.models.serialized_dag.SerializedDagModel since we can't import it anymore. + """ + if isinstance(serialized_dag, dict): + return {k: _sort_serialized_dag_dict(v) for k, v in sorted(serialized_dag.items())} + if isinstance(serialized_dag, list): + if all(isinstance(i, dict) for i in serialized_dag): + if all( + isinstance(i.get("__var", {}), Iterable) and "task_id" in i.get("__var", {}) + for i in serialized_dag + ): + return sorted( + [_sort_serialized_dag_dict(i) for i in serialized_dag], + key=lambda x: x["__var"]["task_id"], + ) + elif all(isinstance(item, str) for item in serialized_dag): + return sorted(serialized_dag) + return [_sort_serialized_dag_dict(i) for i in serialized_dag] + return serialized_dag + + def migrate_existing_deadline_alert_data_from_serialized_dag() -> None: """Extract DeadlineAlert data from serialized Dag data and populate deadline_alert table.""" if context.is_offline_mode(): @@ -476,9 +518,7 @@ def migrate_existing_deadline_alert_data_from_serialized_dag() -> None: updated_result.data, updated_result.data_compressed ) # Import here to avoid a circular dependency issue - from airflow.models.serialized_dag import SerializedDagModel - - new_hash = SerializedDagModel.hash(updated_dag_data) + new_hash = hash_dag(updated_dag_data) conn.execute( sa.text( From b8cf48f15b93c7efa21ec940ab558f783690f302 Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Wed, 14 Jan 2026 13:16:17 -0800 Subject: [PATCH 12/14] static checks again --- airflow-core/docs/img/airflow_erd.sha256 | 2 +- airflow-core/docs/img/airflow_erd.svg | 3218 ++++++++--------- ...099_3_2_0_ui_improvements_for_deadlines.py | 3 +- 3 files changed, 1612 insertions(+), 1611 deletions(-) diff --git a/airflow-core/docs/img/airflow_erd.sha256 b/airflow-core/docs/img/airflow_erd.sha256 index 53b16b6a8a666..31ce3fc87ce1a 100644 --- a/airflow-core/docs/img/airflow_erd.sha256 +++ b/airflow-core/docs/img/airflow_erd.sha256 @@ -1 +1 @@ -c86a1afe46a98ddece8a3bac7de0215dec8e5ca7a70765fd1584eb43f23fbf54 \ No newline at end of file +3a0e13a0db9d0b9fdcd76730e9817846682549ebc7e23e02bf3ffc8efdfee42e \ No newline at end of file diff --git a/airflow-core/docs/img/airflow_erd.svg b/airflow-core/docs/img/airflow_erd.svg index 5878b88806bb5..976cd3c6de371 100644 --- a/airflow-core/docs/img/airflow_erd.svg +++ b/airflow-core/docs/img/airflow_erd.svg @@ -4,572 +4,572 @@ - - + + %3 - + job - -job - -id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - -end_date - - [TIMESTAMP] - -executor_class - - [VARCHAR(500)] - -hostname - - [VARCHAR(500)] - -job_type - - [VARCHAR(30)] - -latest_heartbeat - - [TIMESTAMP] - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -unixname - - [VARCHAR(1000)] + +job + +id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + +end_date + + [TIMESTAMP] + +executor_class + + [VARCHAR(500)] + +hostname + + [VARCHAR(500)] + +job_type + + [VARCHAR(30)] + +latest_heartbeat + + [TIMESTAMP] + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +unixname + + [VARCHAR(1000)] partitioned_asset_key_log - -partitioned_asset_key_log - -id - - [INTEGER] - NOT NULL - -asset_event_id - - [INTEGER] - NOT NULL - -asset_id - - [INTEGER] - NOT NULL - -asset_partition_dag_run_id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -source_partition_key - - [VARCHAR(250)] - NOT NULL - -target_dag_id - - [VARCHAR(250)] - NOT NULL - -target_partition_key - - [VARCHAR(250)] - NOT NULL + +partitioned_asset_key_log + +id + + [INTEGER] + NOT NULL + +asset_event_id + + [INTEGER] + NOT NULL + +asset_id + + [INTEGER] + NOT NULL + +asset_partition_dag_run_id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +source_partition_key + + [VARCHAR(250)] + NOT NULL + +target_dag_id + + [VARCHAR(250)] + NOT NULL + +target_partition_key + + [VARCHAR(250)] + NOT NULL log - -log - -id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - -dttm - - [TIMESTAMP] - NOT NULL - -event - - [VARCHAR(60)] - NOT NULL - -extra - - [TEXT] - -logical_date - - [TIMESTAMP] - -map_index - - [INTEGER] - -owner - - [VARCHAR(500)] - -owner_display_name - - [VARCHAR(500)] - -run_id - - [VARCHAR(250)] - -task_id - - [VARCHAR(250)] - -try_number - - [INTEGER] + +log + +id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + +dttm + + [TIMESTAMP] + NOT NULL + +event + + [VARCHAR(60)] + NOT NULL + +extra + + [TEXT] + +logical_date + + [TIMESTAMP] + +map_index + + [INTEGER] + +owner + + [VARCHAR(500)] + +owner_display_name + + [VARCHAR(500)] + +run_id + + [VARCHAR(250)] + +task_id + + [VARCHAR(250)] + +try_number + + [INTEGER] dag_priority_parsing_request - -dag_priority_parsing_request - -id - - [VARCHAR(32)] - NOT NULL - -bundle_name - - [VARCHAR(250)] - NOT NULL - -relative_fileloc - - [VARCHAR(2000)] - NOT NULL + +dag_priority_parsing_request + +id + + [VARCHAR(32)] + NOT NULL + +bundle_name + + [VARCHAR(250)] + NOT NULL + +relative_fileloc + + [VARCHAR(2000)] + NOT NULL import_error - -import_error - -id - - [INTEGER] - NOT NULL - -bundle_name - - [VARCHAR(250)] - -filename - - [VARCHAR(1024)] - -stacktrace - - [TEXT] - -timestamp - - [TIMESTAMP] + +import_error + +id + + [INTEGER] + NOT NULL + +bundle_name + + [VARCHAR(250)] + +filename + + [VARCHAR(1024)] + +stacktrace + + [TEXT] + +timestamp + + [TIMESTAMP] dag_bundle - -dag_bundle - -name - - [VARCHAR(250)] - NOT NULL - -active - - [BOOLEAN] - -last_refreshed - - [TIMESTAMP] - -signed_url_template - - [VARCHAR(200)] - -template_params - - [JSON] - -version - - [VARCHAR(200)] + +dag_bundle + +name + + [VARCHAR(250)] + NOT NULL + +active + + [BOOLEAN] + +last_refreshed + + [TIMESTAMP] + +signed_url_template + + [VARCHAR(200)] + +template_params + + [JSON] + +version + + [VARCHAR(200)] dag_bundle_team - -dag_bundle_team - -dag_bundle_name - - [VARCHAR(250)] - NOT NULL - -team_name - - [VARCHAR(50)] - NOT NULL + +dag_bundle_team + +dag_bundle_name + + [VARCHAR(250)] + NOT NULL + +team_name + + [VARCHAR(50)] + NOT NULL dag_bundle:name--dag_bundle_team:dag_bundle_name - -0..N -1 + +0..N +1 dag - -dag - -dag_id - - [VARCHAR(250)] - NOT NULL - -asset_expression - - [JSON] - -bundle_name - - [VARCHAR(250)] - NOT NULL - -bundle_version - - [VARCHAR(200)] - -dag_display_name - - [VARCHAR(2000)] - -deadline - - [JSON] - -description - - [TEXT] - -exceeds_max_non_backfill - - [BOOLEAN] - NOT NULL - -fail_fast - - [BOOLEAN] - NOT NULL - -fileloc - - [VARCHAR(2000)] - -has_import_errors - - [BOOLEAN] - NOT NULL - -has_task_concurrency_limits - - [BOOLEAN] - NOT NULL - -is_paused - - [BOOLEAN] - NOT NULL - -is_stale - - [BOOLEAN] - NOT NULL - -last_expired - - [TIMESTAMP] - -last_parse_duration - - [DOUBLE PRECISION] - -last_parsed_time - - [TIMESTAMP] - -max_active_runs - - [INTEGER] - -max_active_tasks - - [INTEGER] - NOT NULL - -max_consecutive_failed_dag_runs - - [INTEGER] - NOT NULL - -next_dagrun - - [TIMESTAMP] - -next_dagrun_create_after - - [TIMESTAMP] - -next_dagrun_data_interval_end - - [TIMESTAMP] - -next_dagrun_data_interval_start - - [TIMESTAMP] - -owners - - [VARCHAR(2000)] - -relative_fileloc - - [VARCHAR(2000)] - -timetable_description - - [VARCHAR(1000)] - -timetable_summary - - [TEXT] - -timetable_type - - [VARCHAR(255)] - NOT NULL + +dag + +dag_id + + [VARCHAR(250)] + NOT NULL + +asset_expression + + [JSON] + +bundle_name + + [VARCHAR(250)] + NOT NULL + +bundle_version + + [VARCHAR(200)] + +dag_display_name + + [VARCHAR(2000)] + +deadline + + [JSON] + +description + + [TEXT] + +exceeds_max_non_backfill + + [BOOLEAN] + NOT NULL + +fail_fast + + [BOOLEAN] + NOT NULL + +fileloc + + [VARCHAR(2000)] + +has_import_errors + + [BOOLEAN] + NOT NULL + +has_task_concurrency_limits + + [BOOLEAN] + NOT NULL + +is_paused + + [BOOLEAN] + NOT NULL + +is_stale + + [BOOLEAN] + NOT NULL + +last_expired + + [TIMESTAMP] + +last_parse_duration + + [DOUBLE PRECISION] + +last_parsed_time + + [TIMESTAMP] + +max_active_runs + + [INTEGER] + +max_active_tasks + + [INTEGER] + NOT NULL + +max_consecutive_failed_dag_runs + + [INTEGER] + NOT NULL + +next_dagrun + + [TIMESTAMP] + +next_dagrun_create_after + + [TIMESTAMP] + +next_dagrun_data_interval_end + + [TIMESTAMP] + +next_dagrun_data_interval_start + + [TIMESTAMP] + +owners + + [VARCHAR(2000)] + +relative_fileloc + + [VARCHAR(2000)] + +timetable_description + + [VARCHAR(1000)] + +timetable_summary + + [TEXT] + +timetable_type + + [VARCHAR(255)] + NOT NULL dag_bundle:name--dag:bundle_name - -0..N -1 + +0..N +1 team - -team - -name - - [VARCHAR(50)] - NOT NULL + +team + +name + + [VARCHAR(50)] + NOT NULL team:name--dag_bundle_team:team_name - -0..N -1 + +0..N +1 connection - -connection - -id - - [INTEGER] - NOT NULL - -conn_id - - [VARCHAR(250)] - NOT NULL - -conn_type - - [VARCHAR(500)] - NOT NULL - -description - - [TEXT] - -extra - - [TEXT] - -host - - [VARCHAR(500)] - -is_encrypted - - [BOOLEAN] - NOT NULL - -is_extra_encrypted - - [BOOLEAN] - NOT NULL - -login - - [TEXT] - -password - - [TEXT] - -port - - [INTEGER] - -schema - - [VARCHAR(500)] - -team_name - - [VARCHAR(50)] + +connection + +id + + [INTEGER] + NOT NULL + +conn_id + + [VARCHAR(250)] + NOT NULL + +conn_type + + [VARCHAR(500)] + NOT NULL + +description + + [TEXT] + +extra + + [TEXT] + +host + + [VARCHAR(500)] + +is_encrypted + + [BOOLEAN] + NOT NULL + +is_extra_encrypted + + [BOOLEAN] + NOT NULL + +login + + [TEXT] + +password + + [TEXT] + +port + + [INTEGER] + +schema + + [VARCHAR(500)] + +team_name + + [VARCHAR(50)] team:name--connection:team_name - -0..N -{0,1} + +0..N +{0,1} slot_pool - -slot_pool - -id - - [INTEGER] - NOT NULL - -description - - [TEXT] - -include_deferred - - [BOOLEAN] - NOT NULL - -pool - - [VARCHAR(256)] - NOT NULL - -slots - - [INTEGER] - NOT NULL - -team_name - - [VARCHAR(50)] + +slot_pool + +id + + [INTEGER] + NOT NULL + +description + + [TEXT] + +include_deferred + + [BOOLEAN] + NOT NULL + +pool + + [VARCHAR(256)] + NOT NULL + +slots + + [INTEGER] + NOT NULL + +team_name + + [VARCHAR(50)] team:name--slot_pool:team_name - -0..N -{0,1} + +0..N +{0,1} variable - -variable - -id - - [INTEGER] - NOT NULL - -description - - [TEXT] - -is_encrypted - - [BOOLEAN] - NOT NULL - -key - - [VARCHAR(250)] - NOT NULL - -team_name - - [VARCHAR(50)] - -val - - [TEXT] - NOT NULL + +variable + +id + + [INTEGER] + NOT NULL + +description + + [TEXT] + +is_encrypted + + [BOOLEAN] + NOT NULL + +key + + [VARCHAR(250)] + NOT NULL + +team_name + + [VARCHAR(50)] + +val + + [TEXT] + NOT NULL team:name--variable:team_name - -0..N -{0,1} + +0..N +{0,1} @@ -691,178 +691,178 @@ task_instance - -task_instance - -id - - [UUID] - NOT NULL - -context_carrier - - [JSONB] - -custom_operator_name - - [VARCHAR(1000)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - -duration - - [DOUBLE PRECISION] - -end_date - - [TIMESTAMP] - -executor - - [VARCHAR(1000)] - -executor_config - - [BYTEA] - NOT NULL - -external_executor_id - - [VARCHAR(250)] - -hostname - - [VARCHAR(1000)] - NOT NULL - -last_heartbeat_at - - [TIMESTAMP] - -map_index - - [INTEGER] - NOT NULL - -max_tries - - [INTEGER] - NOT NULL - -next_kwargs - - [JSONB] - -next_method - - [VARCHAR(1000)] - -operator - - [VARCHAR(1000)] - -pid - - [INTEGER] - -pool - - [VARCHAR(256)] - NOT NULL - -pool_slots - - [INTEGER] - NOT NULL - -priority_weight - - [INTEGER] - NOT NULL - -queue - - [VARCHAR(256)] - NOT NULL - -queued_by_job_id - - [INTEGER] - -queued_dttm - - [TIMESTAMP] - -rendered_map_index - - [VARCHAR(250)] - -run_id - - [VARCHAR(250)] - NOT NULL - -scheduled_dttm - - [TIMESTAMP] - -span_status - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -task_display_name - - [VARCHAR(2000)] - -task_id - - [VARCHAR(250)] - NOT NULL - -trigger_id - - [INTEGER] - -trigger_timeout - - [TIMESTAMP] - -try_number - - [INTEGER] - NOT NULL - -unixname - - [VARCHAR(1000)] - NOT NULL - -updated_at - - [TIMESTAMP] + +task_instance + +id + + [UUID] + NOT NULL + +context_carrier + + [JSONB] + +custom_operator_name + + [VARCHAR(1000)] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + +duration + + [DOUBLE PRECISION] + +end_date + + [TIMESTAMP] + +executor + + [VARCHAR(1000)] + +executor_config + + [BYTEA] + NOT NULL + +external_executor_id + + [VARCHAR(250)] + +hostname + + [VARCHAR(1000)] + NOT NULL + +last_heartbeat_at + + [TIMESTAMP] + +map_index + + [INTEGER] + NOT NULL + +max_tries + + [INTEGER] + NOT NULL + +next_kwargs + + [JSONB] + +next_method + + [VARCHAR(1000)] + +operator + + [VARCHAR(1000)] + +pid + + [INTEGER] + +pool + + [VARCHAR(256)] + NOT NULL + +pool_slots + + [INTEGER] + NOT NULL + +priority_weight + + [INTEGER] + NOT NULL + +queue + + [VARCHAR(256)] + NOT NULL + +queued_by_job_id + + [INTEGER] + +queued_dttm + + [TIMESTAMP] + +rendered_map_index + + [VARCHAR(250)] + +run_id + + [VARCHAR(250)] + NOT NULL + +scheduled_dttm + + [TIMESTAMP] + +span_status + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +task_display_name + + [VARCHAR(2000)] + +task_id + + [VARCHAR(250)] + NOT NULL + +trigger_id + + [INTEGER] + +trigger_timeout + + [TIMESTAMP] + +try_number + + [INTEGER] + NOT NULL + +unixname + + [VARCHAR(1000)] + NOT NULL + +updated_at + + [TIMESTAMP] - + trigger:id--task_instance:trigger_id - -0..N + +0..N {0,1} @@ -910,7 +910,7 @@ NOT NULL - + callback:id--deadline:callback_id 0..N @@ -919,775 +919,775 @@ asset_alias - -asset_alias - -id - - [INTEGER] - NOT NULL - -group - - [VARCHAR(1500)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL + +asset_alias + +id + + [INTEGER] + NOT NULL + +group + + [VARCHAR(1500)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL asset_alias_asset - -asset_alias_asset - -alias_id - - [INTEGER] - NOT NULL - -asset_id - - [INTEGER] - NOT NULL + +asset_alias_asset + +alias_id + + [INTEGER] + NOT NULL + +asset_id + + [INTEGER] + NOT NULL asset_alias:id--asset_alias_asset:alias_id - -0..N -1 + +0..N +1 asset_alias_asset_event - -asset_alias_asset_event - -alias_id - - [INTEGER] - NOT NULL - -event_id - - [INTEGER] - NOT NULL + +asset_alias_asset_event + +alias_id + + [INTEGER] + NOT NULL + +event_id + + [INTEGER] + NOT NULL asset_alias:id--asset_alias_asset_event:alias_id - -0..N -1 + +0..N +1 dag_schedule_asset_alias_reference - -dag_schedule_asset_alias_reference - -alias_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_alias_reference + +alias_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL asset_alias:id--dag_schedule_asset_alias_reference:alias_id - -0..N -1 + +0..N +1 asset - -asset - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -extra - - [JSON] - NOT NULL - -group - - [VARCHAR(1500)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL + +asset + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +extra + + [JSON] + NOT NULL + +group + + [VARCHAR(1500)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL asset:id--asset_alias_asset:asset_id - -0..N -1 + +0..N +1 asset:id--asset_watcher:asset_id - + 0..N -1 +1 asset_active - -asset_active - -name - - [VARCHAR(1500)] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL + +asset_active + +name + + [VARCHAR(1500)] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL asset:name--asset_active:name - -1 -1 + +1 +1 asset:uri--asset_active:uri - -1 -1 + +1 +1 dag_schedule_asset_reference - -dag_schedule_asset_reference - -asset_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_reference + +asset_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL asset:id--dag_schedule_asset_reference:asset_id - -0..N -1 + +0..N +1 task_outlet_asset_reference - -task_outlet_asset_reference - -asset_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +task_outlet_asset_reference + +asset_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL asset:id--task_outlet_asset_reference:asset_id - -0..N -1 + +0..N +1 task_inlet_asset_reference - -task_inlet_asset_reference - -asset_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +task_inlet_asset_reference + +asset_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL asset:id--task_inlet_asset_reference:asset_id - -0..N -1 + +0..N +1 asset_dag_run_queue - -asset_dag_run_queue - -asset_id - - [INTEGER] - NOT NULL - -target_dag_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL + +asset_dag_run_queue + +asset_id + + [INTEGER] + NOT NULL + +target_dag_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL asset:id--asset_dag_run_queue:asset_id - -0..N -1 + +0..N +1 asset_event - -asset_event - -id - - [INTEGER] - NOT NULL - -asset_id - - [INTEGER] - NOT NULL - -extra - - [JSON] - NOT NULL - -partition_key - - [VARCHAR(250)] - -source_dag_id - - [VARCHAR(250)] - -source_map_index - - [INTEGER] - -source_run_id - - [VARCHAR(250)] - -source_task_id - - [VARCHAR(250)] - -timestamp - - [TIMESTAMP] - NOT NULL + +asset_event + +id + + [INTEGER] + NOT NULL + +asset_id + + [INTEGER] + NOT NULL + +extra + + [JSON] + NOT NULL + +partition_key + + [VARCHAR(250)] + +source_dag_id + + [VARCHAR(250)] + +source_map_index + + [INTEGER] + +source_run_id + + [VARCHAR(250)] + +source_task_id + + [VARCHAR(250)] + +timestamp + + [TIMESTAMP] + NOT NULL asset_event:id--asset_alias_asset_event:event_id - -0..N -1 + +0..N +1 dagrun_asset_event - -dagrun_asset_event - -dag_run_id - - [INTEGER] - NOT NULL - -event_id - - [INTEGER] - NOT NULL + +dagrun_asset_event + +dag_run_id + + [INTEGER] + NOT NULL + +event_id + + [INTEGER] + NOT NULL - + asset_event:id--dagrun_asset_event:event_id - -0..N -1 + +0..N +1 dag_schedule_asset_name_reference - -dag_schedule_asset_name_reference - -dag_id - - [VARCHAR(250)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_name_reference + +dag_id + + [VARCHAR(250)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL dag:dag_id--dag_schedule_asset_name_reference:dag_id - -0..N -1 + +0..N +1 dag_schedule_asset_uri_reference - -dag_schedule_asset_uri_reference - -dag_id - - [VARCHAR(250)] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_uri_reference + +dag_id + + [VARCHAR(250)] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL dag:dag_id--dag_schedule_asset_uri_reference:dag_id - -0..N -1 + +0..N +1 dag:dag_id--dag_schedule_asset_alias_reference:dag_id - -0..N -1 + +0..N +1 dag:dag_id--dag_schedule_asset_reference:dag_id - -0..N -1 + +0..N +1 dag:dag_id--task_outlet_asset_reference:dag_id - -0..N -1 + +0..N +1 dag:dag_id--task_inlet_asset_reference:dag_id - -0..N -1 + +0..N +1 dag:dag_id--asset_dag_run_queue:target_dag_id - -0..N -1 + +0..N +1 dag_version - -dag_version - -id - - [UUID] - NOT NULL - -bundle_name - - [VARCHAR(250)] - -bundle_version - - [VARCHAR(250)] - -created_at - - [TIMESTAMP] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -last_updated - - [TIMESTAMP] - NOT NULL - -version_number - - [INTEGER] - NOT NULL + +dag_version + +id + + [UUID] + NOT NULL + +bundle_name + + [VARCHAR(250)] + +bundle_version + + [VARCHAR(250)] + +created_at + + [TIMESTAMP] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +last_updated + + [TIMESTAMP] + NOT NULL + +version_number + + [INTEGER] + NOT NULL dag:dag_id--dag_version:dag_id - -0..N -1 + +0..N +1 dag_tag - -dag_tag - -dag_id - - [VARCHAR(250)] - NOT NULL - -name - - [VARCHAR(100)] - NOT NULL + +dag_tag + +dag_id + + [VARCHAR(250)] + NOT NULL + +name + + [VARCHAR(100)] + NOT NULL dag:dag_id--dag_tag:dag_id - -0..N -1 + +0..N +1 dag_owner_attributes - -dag_owner_attributes - -dag_id - - [VARCHAR(250)] - NOT NULL - -owner - - [VARCHAR(500)] - NOT NULL - -link - - [VARCHAR(500)] - NOT NULL + +dag_owner_attributes + +dag_id + + [VARCHAR(250)] + NOT NULL + +owner + + [VARCHAR(500)] + NOT NULL + +link + + [VARCHAR(500)] + NOT NULL dag:dag_id--dag_owner_attributes:dag_id - -0..N -1 + +0..N +1 dag_warning - -dag_warning - -dag_id - - [VARCHAR(250)] - NOT NULL - -warning_type - - [VARCHAR(50)] - NOT NULL - -message - - [TEXT] - NOT NULL - -timestamp - - [TIMESTAMP] - NOT NULL + +dag_warning + +dag_id + + [VARCHAR(250)] + NOT NULL + +warning_type + + [VARCHAR(50)] + NOT NULL + +message + + [TEXT] + NOT NULL + +timestamp + + [TIMESTAMP] + NOT NULL dag:dag_id--dag_warning:dag_id - -0..N -1 + +0..N +1 dag_favorite - -dag_favorite - -dag_id - - [VARCHAR(250)] - NOT NULL - -user_id - - [VARCHAR(250)] - NOT NULL + +dag_favorite + +dag_id + + [VARCHAR(250)] + NOT NULL + +user_id + + [VARCHAR(250)] + NOT NULL dag:dag_id--dag_favorite:dag_id - -0..N -1 + +0..N +1 dag_run - -dag_run - -id - - [INTEGER] - NOT NULL - -backfill_id - - [INTEGER] - -bundle_version - - [VARCHAR(250)] - -clear_number - - [INTEGER] - NOT NULL - -conf - - [JSONB] - -context_carrier - - [JSONB] - -created_dag_version_id - - [UUID] - -creating_job_id - - [INTEGER] - -dag_id - - [VARCHAR(250)] - NOT NULL - -data_interval_end - - [TIMESTAMP] - -data_interval_start - - [TIMESTAMP] - -end_date - - [TIMESTAMP] - -last_scheduling_decision - - [TIMESTAMP] - -log_template_id - - [INTEGER] - NOT NULL - -logical_date - - [TIMESTAMP] - -partition_key - - [VARCHAR(250)] - -queued_at - - [TIMESTAMP] - -run_after - - [TIMESTAMP] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -run_type - - [VARCHAR(50)] - NOT NULL - -scheduled_by_job_id - - [INTEGER] - -span_status - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(50)] - NOT NULL - -triggered_by - - [VARCHAR(50)] - -triggering_user_name - - [VARCHAR(512)] - -updated_at - - [TIMESTAMP] - NOT NULL + +dag_run + +id + + [INTEGER] + NOT NULL + +backfill_id + + [INTEGER] + +bundle_version + + [VARCHAR(250)] + +clear_number + + [INTEGER] + NOT NULL + +conf + + [JSONB] + +context_carrier + + [JSONB] + +created_dag_version_id + + [UUID] + +creating_job_id + + [INTEGER] + +dag_id + + [VARCHAR(250)] + NOT NULL + +data_interval_end + + [TIMESTAMP] + +data_interval_start + + [TIMESTAMP] + +end_date + + [TIMESTAMP] + +last_scheduling_decision + + [TIMESTAMP] + +log_template_id + + [INTEGER] + NOT NULL + +logical_date + + [TIMESTAMP] + +partition_key + + [VARCHAR(250)] + +queued_at + + [TIMESTAMP] + +run_after + + [TIMESTAMP] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +run_type + + [VARCHAR(50)] + NOT NULL + +scheduled_by_job_id + + [INTEGER] + +span_status + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(50)] + NOT NULL + +triggered_by + + [VARCHAR(50)] + +triggering_user_name + + [VARCHAR(512)] + +updated_at + + [TIMESTAMP] + NOT NULL dag_version:id--dag_run:created_dag_version_id - -0..N -{0,1} + +0..N +{0,1} @@ -1738,9 +1738,9 @@ dag_version:id--dag_code:dag_version_id - + 0..N -1 +1 @@ -1789,112 +1789,112 @@ dag_version:id--serialized_dag:dag_version_id - + 0..N -1 +1 - + dag_version:id--task_instance:dag_version_id - -0..N -{0,1} + +0..N +{0,1} log_template - -log_template - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -elasticsearch_id - - [TEXT] - NOT NULL - -filename - - [TEXT] - NOT NULL + +log_template + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +elasticsearch_id + + [TEXT] + NOT NULL + +filename + + [TEXT] + NOT NULL - + log_template:id--dag_run:log_template_id - -0..N -1 + +0..N +1 - + dag_run:id--dagrun_asset_event:dag_run_id - -0..N -1 + +0..N +1 asset_partition_dag_run - -asset_partition_dag_run - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -created_dag_run_id - - [INTEGER] - -partition_key - - [VARCHAR(250)] - NOT NULL - -target_dag_id - - [VARCHAR(250)] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +asset_partition_dag_run + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +created_dag_run_id + + [INTEGER] + +partition_key + + [VARCHAR(250)] + NOT NULL + +target_dag_id + + [VARCHAR(250)] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL dag_run:id--asset_partition_dag_run:created_dag_run_id - -0..N -{0,1} + +0..N +{0,1} - -dag_run:dag_id--task_instance:dag_id - -0..N -1 + +dag_run:run_id--task_instance:run_id + +0..N +1 - -dag_run:run_id--task_instance:run_id - -0..N -1 + +dag_run:dag_id--task_instance:dag_id + +0..N +1 @@ -1933,129 +1933,129 @@ dag_run:id--backfill_dag_run:dag_run_id - + 0..N -{0,1} +{0,1} dag_run_note - -dag_run_note - -dag_run_id - - [INTEGER] - NOT NULL - -content - - [VARCHAR(1000)] - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -user_id - - [VARCHAR(128)] + +dag_run_note + +dag_run_id + + [INTEGER] + NOT NULL + +content + + [VARCHAR(1000)] + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +user_id + + [VARCHAR(128)] dag_run:id--dag_run_note:dag_run_id - -1 -1 + +1 +1 - + dag_run:id--deadline:dagrun_id - + 0..N -{0,1} +{0,1} backfill - -backfill - -id - - [INTEGER] - NOT NULL - -completed_at - - [TIMESTAMP] - -created_at - - [TIMESTAMP] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_run_conf - - [JSON] - NOT NULL - -from_date - - [TIMESTAMP] - NOT NULL - -is_paused - - [BOOLEAN] - -max_active_runs - - [INTEGER] - NOT NULL - -reprocess_behavior - - [VARCHAR(250)] - NOT NULL - -to_date - - [TIMESTAMP] - NOT NULL - -triggering_user_name - - [VARCHAR(512)] - -updated_at - - [TIMESTAMP] - NOT NULL + +backfill + +id + + [INTEGER] + NOT NULL + +completed_at + + [TIMESTAMP] + +created_at + + [TIMESTAMP] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_run_conf + + [JSON] + NOT NULL + +from_date + + [TIMESTAMP] + NOT NULL + +is_paused + + [BOOLEAN] + +max_active_runs + + [INTEGER] + NOT NULL + +reprocess_behavior + + [VARCHAR(250)] + NOT NULL + +to_date + + [TIMESTAMP] + NOT NULL + +triggering_user_name + + [VARCHAR(512)] + +updated_at + + [TIMESTAMP] + NOT NULL - + backfill:id--dag_run:backfill_id - -0..N -{0,1} + +0..N +{0,1} backfill:id--backfill_dag_run:backfill_id - + 0..N -1 +1 @@ -2175,72 +2175,72 @@ task_instance:id--hitl_detail:ti_id - + 1 -1 +1 task_map - -task_map - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -keys - - [JSONB] - -length - - [INTEGER] - NOT NULL + +task_map + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +keys + + [JSONB] + +length + + [INTEGER] + NOT NULL -task_instance:map_index--task_map:map_index - -0..N -1 +task_instance:run_id--task_map:run_id + +0..N +1 -task_instance:run_id--task_map:run_id - -0..N -1 +task_instance:dag_id--task_map:dag_id + +0..N +1 -task_instance:task_id--task_map:task_id - -0..N -1 +task_instance:map_index--task_map:map_index + +0..N +1 -task_instance:dag_id--task_map:dag_id - -0..N -1 +task_instance:task_id--task_map:task_id + +0..N +1 @@ -2281,82 +2281,82 @@ task_instance:id--task_reschedule:ti_id - + 0..N -1 +1 xcom - -xcom - -dag_run_id - - [INTEGER] - NOT NULL - -key - - [VARCHAR(512)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -timestamp - - [TIMESTAMP] - NOT NULL - -value - - [JSONB] + +xcom + +dag_run_id + + [INTEGER] + NOT NULL + +key + + [VARCHAR(512)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +timestamp + + [TIMESTAMP] + NOT NULL + +value + + [JSONB] -task_instance:task_id--xcom:task_id - -0..N -1 +task_instance:run_id--xcom:run_id + +0..N +1 -task_instance:dag_id--xcom:dag_id - -0..N -1 +task_instance:map_index--xcom:map_index + +0..N +1 -task_instance:map_index--xcom:map_index - -0..N -1 +task_instance:task_id--xcom:task_id + +0..N +1 -task_instance:run_id--xcom:run_id - -0..N -1 +task_instance:dag_id--xcom:dag_id + +0..N +1 @@ -2390,9 +2390,9 @@ task_instance:id--task_instance_note:ti_id - + 1 -1 +1 @@ -2555,97 +2555,97 @@ -task_instance:dag_id--task_instance_history:dag_id - -0..N -1 +task_instance:map_index--task_instance_history:map_index + +0..N +1 -task_instance:run_id--task_instance_history:run_id - -0..N -1 +task_instance:dag_id--task_instance_history:dag_id + +0..N +1 -task_instance:map_index--task_instance_history:map_index - -0..N -1 +task_instance:task_id--task_instance_history:task_id + +0..N +1 -task_instance:task_id--task_instance_history:task_id - -0..N -1 +task_instance:run_id--task_instance_history:run_id + +0..N +1 rendered_task_instance_fields - -rendered_task_instance_fields - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL + +rendered_task_instance_fields -run_id - - [VARCHAR(250)] - NOT NULL +dag_id + + [VARCHAR(250)] + NOT NULL -task_id - - [VARCHAR(250)] - NOT NULL +map_index + + [INTEGER] + NOT NULL -k8s_pod_yaml - - [JSON] +run_id + + [VARCHAR(250)] + NOT NULL -rendered_fields - - [JSON] - NOT NULL +task_id + + [VARCHAR(250)] + NOT NULL + +k8s_pod_yaml + + [JSON] + +rendered_fields + + [JSON] + NOT NULL -task_instance:map_index--rendered_task_instance_fields:map_index - -0..N -1 +task_instance:run_id--rendered_task_instance_fields:run_id + +0..N +1 -task_instance:task_id--rendered_task_instance_fields:task_id - -0..N -1 +task_instance:map_index--rendered_task_instance_fields:map_index + +0..N +1 task_instance:dag_id--rendered_task_instance_fields:dag_id - -0..N -1 + +0..N +1 -task_instance:run_id--rendered_task_instance_fields:run_id - -0..N -1 +task_instance:task_id--rendered_task_instance_fields:task_id + +0..N +1 - + deadline_alert:id--deadline:deadline_alert_id 0..N @@ -2725,13 +2725,13 @@ alembic_version - -alembic_version - -version_num - - [VARCHAR(32)] - NOT NULL + +alembic_version + +version_num + + [VARCHAR(32)] + NOT NULL diff --git a/airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py b/airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py index 93c1a225d5294..14a1ac855b147 100644 --- a/airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py +++ b/airflow-core/src/airflow/migrations/versions/0099_3_2_0_ui_improvements_for_deadlines.py @@ -33,7 +33,8 @@ import json import zlib from collections import defaultdict -from typing import TYPE_CHECKING, Iterable +from collections.abc import Iterable +from typing import TYPE_CHECKING import sqlalchemy as sa import uuid6 From 2ee570fa1fbd83dce186874193eef6296fcd6640 Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Thu, 15 Jan 2026 08:41:03 -0800 Subject: [PATCH 13/14] another merge conflict --- airflow-core/docs/img/airflow_erd.sha256 | 2 +- airflow-core/docs/img/airflow_erd.svg | 3234 ++++++++--------- .../airflow/serialization/definitions/dag.py | 3 +- 3 files changed, 1619 insertions(+), 1620 deletions(-) diff --git a/airflow-core/docs/img/airflow_erd.sha256 b/airflow-core/docs/img/airflow_erd.sha256 index 31ce3fc87ce1a..ecbb6455edd62 100644 --- a/airflow-core/docs/img/airflow_erd.sha256 +++ b/airflow-core/docs/img/airflow_erd.sha256 @@ -1 +1 @@ -3a0e13a0db9d0b9fdcd76730e9817846682549ebc7e23e02bf3ffc8efdfee42e \ No newline at end of file +ae3f3b6fd90d208442c42ecd3a3318a3da2ca4a5d8bb1691e57d44d245114e0b \ No newline at end of file diff --git a/airflow-core/docs/img/airflow_erd.svg b/airflow-core/docs/img/airflow_erd.svg index 976cd3c6de371..6807ea1786f75 100644 --- a/airflow-core/docs/img/airflow_erd.svg +++ b/airflow-core/docs/img/airflow_erd.svg @@ -4,572 +4,572 @@ - - + + %3 - + job - -job - -id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - -end_date - - [TIMESTAMP] - -executor_class - - [VARCHAR(500)] - -hostname - - [VARCHAR(500)] - -job_type - - [VARCHAR(30)] - -latest_heartbeat - - [TIMESTAMP] - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -unixname - - [VARCHAR(1000)] + +job + +id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + +end_date + + [TIMESTAMP] + +executor_class + + [VARCHAR(500)] + +hostname + + [VARCHAR(500)] + +job_type + + [VARCHAR(30)] + +latest_heartbeat + + [TIMESTAMP] + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +unixname + + [VARCHAR(1000)] partitioned_asset_key_log - -partitioned_asset_key_log - -id - - [INTEGER] - NOT NULL - -asset_event_id - - [INTEGER] - NOT NULL - -asset_id - - [INTEGER] - NOT NULL - -asset_partition_dag_run_id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -source_partition_key - - [VARCHAR(250)] - NOT NULL - -target_dag_id - - [VARCHAR(250)] - NOT NULL - -target_partition_key - - [VARCHAR(250)] - NOT NULL + +partitioned_asset_key_log + +id + + [INTEGER] + NOT NULL + +asset_event_id + + [INTEGER] + NOT NULL + +asset_id + + [INTEGER] + NOT NULL + +asset_partition_dag_run_id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +source_partition_key + + [VARCHAR(250)] + NOT NULL + +target_dag_id + + [VARCHAR(250)] + NOT NULL + +target_partition_key + + [VARCHAR(250)] + NOT NULL log - -log - -id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - -dttm - - [TIMESTAMP] - NOT NULL - -event - - [VARCHAR(60)] - NOT NULL - -extra - - [TEXT] - -logical_date - - [TIMESTAMP] - -map_index - - [INTEGER] - -owner - - [VARCHAR(500)] - -owner_display_name - - [VARCHAR(500)] - -run_id - - [VARCHAR(250)] - -task_id - - [VARCHAR(250)] - -try_number - - [INTEGER] + +log + +id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + +dttm + + [TIMESTAMP] + NOT NULL + +event + + [VARCHAR(60)] + NOT NULL + +extra + + [TEXT] + +logical_date + + [TIMESTAMP] + +map_index + + [INTEGER] + +owner + + [VARCHAR(500)] + +owner_display_name + + [VARCHAR(500)] + +run_id + + [VARCHAR(250)] + +task_id + + [VARCHAR(250)] + +try_number + + [INTEGER] dag_priority_parsing_request - -dag_priority_parsing_request - -id - - [VARCHAR(32)] - NOT NULL - -bundle_name - - [VARCHAR(250)] - NOT NULL - -relative_fileloc - - [VARCHAR(2000)] - NOT NULL + +dag_priority_parsing_request + +id + + [VARCHAR(32)] + NOT NULL + +bundle_name + + [VARCHAR(250)] + NOT NULL + +relative_fileloc + + [VARCHAR(2000)] + NOT NULL import_error - -import_error - -id - - [INTEGER] - NOT NULL - -bundle_name - - [VARCHAR(250)] - -filename - - [VARCHAR(1024)] - -stacktrace - - [TEXT] - -timestamp - - [TIMESTAMP] + +import_error + +id + + [INTEGER] + NOT NULL + +bundle_name + + [VARCHAR(250)] + +filename + + [VARCHAR(1024)] + +stacktrace + + [TEXT] + +timestamp + + [TIMESTAMP] dag_bundle - -dag_bundle - -name - - [VARCHAR(250)] - NOT NULL - -active - - [BOOLEAN] - -last_refreshed - - [TIMESTAMP] - -signed_url_template - - [VARCHAR(200)] - -template_params - - [JSON] - -version - - [VARCHAR(200)] + +dag_bundle + +name + + [VARCHAR(250)] + NOT NULL + +active + + [BOOLEAN] + +last_refreshed + + [TIMESTAMP] + +signed_url_template + + [VARCHAR(200)] + +template_params + + [JSON] + +version + + [VARCHAR(200)] dag_bundle_team - -dag_bundle_team - -dag_bundle_name - - [VARCHAR(250)] - NOT NULL - -team_name - - [VARCHAR(50)] - NOT NULL + +dag_bundle_team + +dag_bundle_name + + [VARCHAR(250)] + NOT NULL + +team_name + + [VARCHAR(50)] + NOT NULL dag_bundle:name--dag_bundle_team:dag_bundle_name - -0..N -1 + +0..N +1 dag - -dag - -dag_id - - [VARCHAR(250)] - NOT NULL - -asset_expression - - [JSON] - -bundle_name - - [VARCHAR(250)] - NOT NULL - -bundle_version - - [VARCHAR(200)] - -dag_display_name - - [VARCHAR(2000)] - -deadline - - [JSON] - -description - - [TEXT] - -exceeds_max_non_backfill - - [BOOLEAN] - NOT NULL - -fail_fast - - [BOOLEAN] - NOT NULL - -fileloc - - [VARCHAR(2000)] - -has_import_errors - - [BOOLEAN] - NOT NULL - -has_task_concurrency_limits - - [BOOLEAN] - NOT NULL - -is_paused - - [BOOLEAN] - NOT NULL - -is_stale - - [BOOLEAN] - NOT NULL - -last_expired - - [TIMESTAMP] - -last_parse_duration - - [DOUBLE PRECISION] - -last_parsed_time - - [TIMESTAMP] - -max_active_runs - - [INTEGER] - -max_active_tasks - - [INTEGER] - NOT NULL - -max_consecutive_failed_dag_runs - - [INTEGER] - NOT NULL - -next_dagrun - - [TIMESTAMP] - -next_dagrun_create_after - - [TIMESTAMP] - -next_dagrun_data_interval_end - - [TIMESTAMP] - -next_dagrun_data_interval_start - - [TIMESTAMP] - -owners - - [VARCHAR(2000)] - -relative_fileloc - - [VARCHAR(2000)] - -timetable_description - - [VARCHAR(1000)] - -timetable_summary - - [TEXT] - -timetable_type - - [VARCHAR(255)] - NOT NULL + +dag + +dag_id + + [VARCHAR(250)] + NOT NULL + +asset_expression + + [JSON] + +bundle_name + + [VARCHAR(250)] + NOT NULL + +bundle_version + + [VARCHAR(200)] + +dag_display_name + + [VARCHAR(2000)] + +deadline + + [JSON] + +description + + [TEXT] + +exceeds_max_non_backfill + + [BOOLEAN] + NOT NULL + +fail_fast + + [BOOLEAN] + NOT NULL + +fileloc + + [VARCHAR(2000)] + +has_import_errors + + [BOOLEAN] + NOT NULL + +has_task_concurrency_limits + + [BOOLEAN] + NOT NULL + +is_paused + + [BOOLEAN] + NOT NULL + +is_stale + + [BOOLEAN] + NOT NULL + +last_expired + + [TIMESTAMP] + +last_parse_duration + + [DOUBLE PRECISION] + +last_parsed_time + + [TIMESTAMP] + +max_active_runs + + [INTEGER] + +max_active_tasks + + [INTEGER] + NOT NULL + +max_consecutive_failed_dag_runs + + [INTEGER] + NOT NULL + +next_dagrun + + [TIMESTAMP] + +next_dagrun_create_after + + [TIMESTAMP] + +next_dagrun_data_interval_end + + [TIMESTAMP] + +next_dagrun_data_interval_start + + [TIMESTAMP] + +owners + + [VARCHAR(2000)] + +relative_fileloc + + [VARCHAR(2000)] + +timetable_description + + [VARCHAR(1000)] + +timetable_summary + + [TEXT] + +timetable_type + + [VARCHAR(255)] + NOT NULL dag_bundle:name--dag:bundle_name - -0..N -1 + +0..N +1 team - -team - -name - - [VARCHAR(50)] - NOT NULL + +team + +name + + [VARCHAR(50)] + NOT NULL team:name--dag_bundle_team:team_name - -0..N -1 + +0..N +1 connection - -connection - -id - - [INTEGER] - NOT NULL - -conn_id - - [VARCHAR(250)] - NOT NULL - -conn_type - - [VARCHAR(500)] - NOT NULL - -description - - [TEXT] - -extra - - [TEXT] - -host - - [VARCHAR(500)] - -is_encrypted - - [BOOLEAN] - NOT NULL - -is_extra_encrypted - - [BOOLEAN] - NOT NULL - -login - - [TEXT] - -password - - [TEXT] - -port - - [INTEGER] - -schema - - [VARCHAR(500)] - -team_name - - [VARCHAR(50)] + +connection + +id + + [INTEGER] + NOT NULL + +conn_id + + [VARCHAR(250)] + NOT NULL + +conn_type + + [VARCHAR(500)] + NOT NULL + +description + + [TEXT] + +extra + + [TEXT] + +host + + [VARCHAR(500)] + +is_encrypted + + [BOOLEAN] + NOT NULL + +is_extra_encrypted + + [BOOLEAN] + NOT NULL + +login + + [TEXT] + +password + + [TEXT] + +port + + [INTEGER] + +schema + + [VARCHAR(500)] + +team_name + + [VARCHAR(50)] team:name--connection:team_name - -0..N -{0,1} + +0..N +{0,1} slot_pool - -slot_pool - -id - - [INTEGER] - NOT NULL - -description - - [TEXT] - -include_deferred - - [BOOLEAN] - NOT NULL - -pool - - [VARCHAR(256)] - NOT NULL - -slots - - [INTEGER] - NOT NULL - -team_name - - [VARCHAR(50)] + +slot_pool + +id + + [INTEGER] + NOT NULL + +description + + [TEXT] + +include_deferred + + [BOOLEAN] + NOT NULL + +pool + + [VARCHAR(256)] + NOT NULL + +slots + + [INTEGER] + NOT NULL + +team_name + + [VARCHAR(50)] team:name--slot_pool:team_name - -0..N -{0,1} + +0..N +{0,1} variable - -variable - -id - - [INTEGER] - NOT NULL - -description - - [TEXT] - -is_encrypted - - [BOOLEAN] - NOT NULL - -key - - [VARCHAR(250)] - NOT NULL - -team_name - - [VARCHAR(50)] - -val - - [TEXT] - NOT NULL + +variable + +id + + [INTEGER] + NOT NULL + +description + + [TEXT] + +is_encrypted + + [BOOLEAN] + NOT NULL + +key + + [VARCHAR(250)] + NOT NULL + +team_name + + [VARCHAR(50)] + +val + + [TEXT] + NOT NULL team:name--variable:team_name - -0..N -{0,1} + +0..N +{0,1} @@ -682,7 +682,7 @@ NOT NULL - + trigger:id--asset_watcher:trigger_id 0..N @@ -691,178 +691,178 @@ task_instance - -task_instance - -id - - [UUID] - NOT NULL - -context_carrier - - [JSONB] - -custom_operator_name - - [VARCHAR(1000)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_version_id - - [UUID] - -duration - - [DOUBLE PRECISION] - -end_date - - [TIMESTAMP] - -executor - - [VARCHAR(1000)] - -executor_config - - [BYTEA] - NOT NULL - -external_executor_id - - [VARCHAR(250)] - -hostname - - [VARCHAR(1000)] - NOT NULL - -last_heartbeat_at - - [TIMESTAMP] - -map_index - - [INTEGER] - NOT NULL - -max_tries - - [INTEGER] - NOT NULL - -next_kwargs - - [JSONB] - -next_method - - [VARCHAR(1000)] - -operator - - [VARCHAR(1000)] - -pid - - [INTEGER] - -pool - - [VARCHAR(256)] - NOT NULL - -pool_slots - - [INTEGER] - NOT NULL - -priority_weight - - [INTEGER] - NOT NULL - -queue - - [VARCHAR(256)] - NOT NULL - -queued_by_job_id - - [INTEGER] - -queued_dttm - - [TIMESTAMP] - -rendered_map_index - - [VARCHAR(250)] - -run_id - - [VARCHAR(250)] - NOT NULL - -scheduled_dttm - - [TIMESTAMP] - -span_status - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(20)] - -task_display_name - - [VARCHAR(2000)] - -task_id - - [VARCHAR(250)] - NOT NULL - -trigger_id - - [INTEGER] - -trigger_timeout - - [TIMESTAMP] - -try_number - - [INTEGER] - NOT NULL - -unixname - - [VARCHAR(1000)] - NOT NULL - -updated_at - - [TIMESTAMP] + +task_instance + +id + + [UUID] + NOT NULL + +context_carrier + + [JSONB] + +custom_operator_name + + [VARCHAR(1000)] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_version_id + + [UUID] + +duration + + [DOUBLE PRECISION] + +end_date + + [TIMESTAMP] + +executor + + [VARCHAR(1000)] + +executor_config + + [BYTEA] + NOT NULL + +external_executor_id + + [VARCHAR(250)] + +hostname + + [VARCHAR(1000)] + NOT NULL + +last_heartbeat_at + + [TIMESTAMP] + +map_index + + [INTEGER] + NOT NULL + +max_tries + + [INTEGER] + NOT NULL + +next_kwargs + + [JSONB] + +next_method + + [VARCHAR(1000)] + +operator + + [VARCHAR(1000)] + +pid + + [INTEGER] + +pool + + [VARCHAR(256)] + NOT NULL + +pool_slots + + [INTEGER] + NOT NULL + +priority_weight + + [INTEGER] + NOT NULL + +queue + + [VARCHAR(256)] + NOT NULL + +queued_by_job_id + + [INTEGER] + +queued_dttm + + [TIMESTAMP] + +rendered_map_index + + [VARCHAR(250)] + +run_id + + [VARCHAR(250)] + NOT NULL + +scheduled_dttm + + [TIMESTAMP] + +span_status + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(20)] + +task_display_name + + [VARCHAR(2000)] + +task_id + + [VARCHAR(250)] + NOT NULL + +trigger_id + + [INTEGER] + +trigger_timeout + + [TIMESTAMP] + +try_number + + [INTEGER] + NOT NULL + +unixname + + [VARCHAR(1000)] + NOT NULL + +updated_at + + [TIMESTAMP] - + trigger:id--task_instance:trigger_id - -0..N + +0..N {0,1} @@ -910,7 +910,7 @@ NOT NULL - + callback:id--deadline:callback_id 0..N @@ -919,775 +919,775 @@ asset_alias - -asset_alias - -id - - [INTEGER] - NOT NULL - -group - - [VARCHAR(1500)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL + +asset_alias + +id + + [INTEGER] + NOT NULL + +group + + [VARCHAR(1500)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL asset_alias_asset - -asset_alias_asset - -alias_id - - [INTEGER] - NOT NULL - -asset_id - - [INTEGER] - NOT NULL + +asset_alias_asset + +alias_id + + [INTEGER] + NOT NULL + +asset_id + + [INTEGER] + NOT NULL - + asset_alias:id--asset_alias_asset:alias_id - -0..N -1 + +0..N +1 asset_alias_asset_event - -asset_alias_asset_event - -alias_id - - [INTEGER] - NOT NULL - -event_id - - [INTEGER] - NOT NULL + +asset_alias_asset_event + +alias_id + + [INTEGER] + NOT NULL + +event_id + + [INTEGER] + NOT NULL asset_alias:id--asset_alias_asset_event:alias_id - -0..N -1 + +0..N +1 dag_schedule_asset_alias_reference - -dag_schedule_asset_alias_reference - -alias_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_alias_reference + +alias_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL asset_alias:id--dag_schedule_asset_alias_reference:alias_id - -0..N -1 + +0..N +1 asset - -asset - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -extra - - [JSON] - NOT NULL - -group - - [VARCHAR(1500)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL + +asset + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +extra + + [JSON] + NOT NULL + +group + + [VARCHAR(1500)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL - + asset:id--asset_alias_asset:asset_id - -0..N -1 + +0..N +1 - + asset:id--asset_watcher:asset_id - + 0..N -1 +1 asset_active - -asset_active - -name - - [VARCHAR(1500)] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL + +asset_active + +name + + [VARCHAR(1500)] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL -asset:name--asset_active:name - -1 -1 +asset:uri--asset_active:uri + +1 +1 -asset:uri--asset_active:uri - -1 -1 +asset:name--asset_active:name + +1 +1 dag_schedule_asset_reference - -dag_schedule_asset_reference - -asset_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_reference + +asset_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL - + asset:id--dag_schedule_asset_reference:asset_id - -0..N -1 + +0..N +1 task_outlet_asset_reference - -task_outlet_asset_reference - -asset_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +task_outlet_asset_reference + +asset_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL - + asset:id--task_outlet_asset_reference:asset_id - -0..N -1 + +0..N +1 task_inlet_asset_reference - -task_inlet_asset_reference - -asset_id - - [INTEGER] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +task_inlet_asset_reference + +asset_id + + [INTEGER] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL - + asset:id--task_inlet_asset_reference:asset_id - -0..N -1 + +0..N +1 asset_dag_run_queue - -asset_dag_run_queue - -asset_id - - [INTEGER] - NOT NULL - -target_dag_id - - [VARCHAR(250)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL + +asset_dag_run_queue + +asset_id + + [INTEGER] + NOT NULL + +target_dag_id + + [VARCHAR(250)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL - + asset:id--asset_dag_run_queue:asset_id - -0..N -1 + +0..N +1 asset_event - -asset_event - -id - - [INTEGER] - NOT NULL - -asset_id - - [INTEGER] - NOT NULL - -extra - - [JSON] - NOT NULL - -partition_key - - [VARCHAR(250)] - -source_dag_id - - [VARCHAR(250)] - -source_map_index - - [INTEGER] - -source_run_id - - [VARCHAR(250)] - -source_task_id - - [VARCHAR(250)] - -timestamp - - [TIMESTAMP] - NOT NULL + +asset_event + +id + + [INTEGER] + NOT NULL + +asset_id + + [INTEGER] + NOT NULL + +extra + + [JSON] + NOT NULL + +partition_key + + [VARCHAR(250)] + +source_dag_id + + [VARCHAR(250)] + +source_map_index + + [INTEGER] + +source_run_id + + [VARCHAR(250)] + +source_task_id + + [VARCHAR(250)] + +timestamp + + [TIMESTAMP] + NOT NULL asset_event:id--asset_alias_asset_event:event_id - -0..N -1 + +0..N +1 dagrun_asset_event - -dagrun_asset_event - -dag_run_id - - [INTEGER] - NOT NULL - -event_id - - [INTEGER] - NOT NULL + +dagrun_asset_event + +dag_run_id + + [INTEGER] + NOT NULL + +event_id + + [INTEGER] + NOT NULL - + asset_event:id--dagrun_asset_event:event_id - -0..N -1 + +0..N +1 dag_schedule_asset_name_reference - -dag_schedule_asset_name_reference - -dag_id - - [VARCHAR(250)] - NOT NULL - -name - - [VARCHAR(1500)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_name_reference + +dag_id + + [VARCHAR(250)] + NOT NULL + +name + + [VARCHAR(1500)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL dag:dag_id--dag_schedule_asset_name_reference:dag_id - -0..N -1 + +0..N +1 dag_schedule_asset_uri_reference - -dag_schedule_asset_uri_reference - -dag_id - - [VARCHAR(250)] - NOT NULL - -uri - - [VARCHAR(1500)] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL + +dag_schedule_asset_uri_reference + +dag_id + + [VARCHAR(250)] + NOT NULL + +uri + + [VARCHAR(1500)] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL dag:dag_id--dag_schedule_asset_uri_reference:dag_id - -0..N -1 + +0..N +1 dag:dag_id--dag_schedule_asset_alias_reference:dag_id - -0..N -1 + +0..N +1 - + dag:dag_id--dag_schedule_asset_reference:dag_id - -0..N -1 + +0..N +1 - + dag:dag_id--task_outlet_asset_reference:dag_id - -0..N -1 + +0..N +1 - + dag:dag_id--task_inlet_asset_reference:dag_id - -0..N -1 + +0..N +1 - + dag:dag_id--asset_dag_run_queue:target_dag_id - -0..N -1 + +0..N +1 dag_version - -dag_version - -id - - [UUID] - NOT NULL - -bundle_name - - [VARCHAR(250)] - -bundle_version - - [VARCHAR(250)] - -created_at - - [TIMESTAMP] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -last_updated - - [TIMESTAMP] - NOT NULL - -version_number - - [INTEGER] - NOT NULL + +dag_version + +id + + [UUID] + NOT NULL + +bundle_name + + [VARCHAR(250)] + +bundle_version + + [VARCHAR(250)] + +created_at + + [TIMESTAMP] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +last_updated + + [TIMESTAMP] + NOT NULL + +version_number + + [INTEGER] + NOT NULL dag:dag_id--dag_version:dag_id - -0..N -1 + +0..N +1 dag_tag - -dag_tag - -dag_id - - [VARCHAR(250)] - NOT NULL - -name - - [VARCHAR(100)] - NOT NULL + +dag_tag + +dag_id + + [VARCHAR(250)] + NOT NULL + +name + + [VARCHAR(100)] + NOT NULL dag:dag_id--dag_tag:dag_id - -0..N -1 + +0..N +1 dag_owner_attributes - -dag_owner_attributes - -dag_id - - [VARCHAR(250)] - NOT NULL - -owner - - [VARCHAR(500)] - NOT NULL - -link - - [VARCHAR(500)] - NOT NULL + +dag_owner_attributes + +dag_id + + [VARCHAR(250)] + NOT NULL + +owner + + [VARCHAR(500)] + NOT NULL + +link + + [VARCHAR(500)] + NOT NULL dag:dag_id--dag_owner_attributes:dag_id - -0..N -1 + +0..N +1 dag_warning - -dag_warning - -dag_id - - [VARCHAR(250)] - NOT NULL - -warning_type - - [VARCHAR(50)] - NOT NULL - -message - - [TEXT] - NOT NULL - -timestamp - - [TIMESTAMP] - NOT NULL + +dag_warning + +dag_id + + [VARCHAR(250)] + NOT NULL + +warning_type + + [VARCHAR(50)] + NOT NULL + +message + + [TEXT] + NOT NULL + +timestamp + + [TIMESTAMP] + NOT NULL dag:dag_id--dag_warning:dag_id - -0..N -1 + +0..N +1 dag_favorite - -dag_favorite - -dag_id - - [VARCHAR(250)] - NOT NULL - -user_id - - [VARCHAR(250)] - NOT NULL + +dag_favorite + +dag_id + + [VARCHAR(250)] + NOT NULL + +user_id + + [VARCHAR(250)] + NOT NULL dag:dag_id--dag_favorite:dag_id - -0..N -1 + +0..N +1 dag_run - -dag_run - -id - - [INTEGER] - NOT NULL - -backfill_id - - [INTEGER] - -bundle_version - - [VARCHAR(250)] - -clear_number - - [INTEGER] - NOT NULL - -conf - - [JSONB] - -context_carrier - - [JSONB] - -created_dag_version_id - - [UUID] - -creating_job_id - - [INTEGER] - -dag_id - - [VARCHAR(250)] - NOT NULL - -data_interval_end - - [TIMESTAMP] - -data_interval_start - - [TIMESTAMP] - -end_date - - [TIMESTAMP] - -last_scheduling_decision - - [TIMESTAMP] - -log_template_id - - [INTEGER] - NOT NULL - -logical_date - - [TIMESTAMP] - -partition_key - - [VARCHAR(250)] - -queued_at - - [TIMESTAMP] - -run_after - - [TIMESTAMP] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -run_type - - [VARCHAR(50)] - NOT NULL - -scheduled_by_job_id - - [INTEGER] - -span_status - - [VARCHAR(250)] - NOT NULL - -start_date - - [TIMESTAMP] - -state - - [VARCHAR(50)] - NOT NULL - -triggered_by - - [VARCHAR(50)] - -triggering_user_name - - [VARCHAR(512)] - -updated_at - - [TIMESTAMP] - NOT NULL + +dag_run + +id + + [INTEGER] + NOT NULL + +backfill_id + + [INTEGER] + +bundle_version + + [VARCHAR(250)] + +clear_number + + [INTEGER] + NOT NULL + +conf + + [JSONB] + +context_carrier + + [JSONB] + +created_dag_version_id + + [UUID] + +creating_job_id + + [INTEGER] + +dag_id + + [VARCHAR(250)] + NOT NULL + +data_interval_end + + [TIMESTAMP] + +data_interval_start + + [TIMESTAMP] + +end_date + + [TIMESTAMP] + +last_scheduling_decision + + [TIMESTAMP] + +log_template_id + + [INTEGER] + NOT NULL + +logical_date + + [TIMESTAMP] + +partition_key + + [VARCHAR(250)] + +queued_at + + [TIMESTAMP] + +run_after + + [TIMESTAMP] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +run_type + + [VARCHAR(50)] + NOT NULL + +scheduled_by_job_id + + [INTEGER] + +span_status + + [VARCHAR(250)] + NOT NULL + +start_date + + [TIMESTAMP] + +state + + [VARCHAR(50)] + NOT NULL + +triggered_by + + [VARCHAR(50)] + +triggering_user_name + + [VARCHAR(512)] + +updated_at + + [TIMESTAMP] + NOT NULL dag_version:id--dag_run:created_dag_version_id - -0..N -{0,1} + +0..N +{0,1} @@ -1738,9 +1738,9 @@ dag_version:id--dag_code:dag_version_id - + 0..N -1 +1 @@ -1789,112 +1789,112 @@ dag_version:id--serialized_dag:dag_version_id - + 0..N -1 +1 - + dag_version:id--task_instance:dag_version_id - -0..N -{0,1} + +0..N +{0,1} log_template - -log_template - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -elasticsearch_id - - [TEXT] - NOT NULL - -filename - - [TEXT] - NOT NULL + +log_template + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +elasticsearch_id + + [TEXT] + NOT NULL + +filename + + [TEXT] + NOT NULL - + log_template:id--dag_run:log_template_id - -0..N -1 + +0..N +1 - + dag_run:id--dagrun_asset_event:dag_run_id - -0..N -1 + +0..N +1 asset_partition_dag_run - -asset_partition_dag_run - -id - - [INTEGER] - NOT NULL - -created_at - - [TIMESTAMP] - NOT NULL - -created_dag_run_id - - [INTEGER] - -partition_key - - [VARCHAR(250)] - NOT NULL - -target_dag_id - - [VARCHAR(250)] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL + +asset_partition_dag_run + +id + + [INTEGER] + NOT NULL + +created_at + + [TIMESTAMP] + NOT NULL + +created_dag_run_id + + [INTEGER] + +partition_key + + [VARCHAR(250)] + NOT NULL + +target_dag_id + + [VARCHAR(250)] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL dag_run:id--asset_partition_dag_run:created_dag_run_id - -0..N -{0,1} + +0..N +{0,1} - + dag_run:run_id--task_instance:run_id - -0..N -1 + +0..N +1 dag_run:dag_id--task_instance:dag_id - -0..N -1 + +0..N +1 @@ -1931,131 +1931,131 @@ NOT NULL - + dag_run:id--backfill_dag_run:dag_run_id - + 0..N -{0,1} +{0,1} dag_run_note - -dag_run_note - -dag_run_id - - [INTEGER] - NOT NULL - -content - - [VARCHAR(1000)] - -created_at - - [TIMESTAMP] - NOT NULL - -updated_at - - [TIMESTAMP] - NOT NULL - -user_id - - [VARCHAR(128)] + +dag_run_note + +dag_run_id + + [INTEGER] + NOT NULL + +content + + [VARCHAR(1000)] + +created_at + + [TIMESTAMP] + NOT NULL + +updated_at + + [TIMESTAMP] + NOT NULL + +user_id + + [VARCHAR(128)] dag_run:id--dag_run_note:dag_run_id - -1 -1 + +1 +1 - + dag_run:id--deadline:dagrun_id - + 0..N -{0,1} +{0,1} backfill - -backfill - -id - - [INTEGER] - NOT NULL - -completed_at - - [TIMESTAMP] - -created_at - - [TIMESTAMP] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -dag_run_conf - - [JSON] - NOT NULL - -from_date - - [TIMESTAMP] - NOT NULL - -is_paused - - [BOOLEAN] - -max_active_runs - - [INTEGER] - NOT NULL - -reprocess_behavior - - [VARCHAR(250)] - NOT NULL - -to_date - - [TIMESTAMP] - NOT NULL - -triggering_user_name - - [VARCHAR(512)] - -updated_at - - [TIMESTAMP] - NOT NULL + +backfill + +id + + [INTEGER] + NOT NULL + +completed_at + + [TIMESTAMP] + +created_at + + [TIMESTAMP] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +dag_run_conf + + [JSON] + NOT NULL + +from_date + + [TIMESTAMP] + NOT NULL + +is_paused + + [BOOLEAN] + +max_active_runs + + [INTEGER] + NOT NULL + +reprocess_behavior + + [VARCHAR(250)] + NOT NULL + +to_date + + [TIMESTAMP] + NOT NULL + +triggering_user_name + + [VARCHAR(512)] + +updated_at + + [TIMESTAMP] + NOT NULL - + backfill:id--dag_run:backfill_id - -0..N -{0,1} + +0..N +{0,1} - + backfill:id--backfill_dag_run:backfill_id - + 0..N -1 +1 @@ -2175,72 +2175,72 @@ task_instance:id--hitl_detail:ti_id - + 1 -1 +1 task_map - -task_map - -dag_id - - [VARCHAR(250)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -keys - - [JSONB] - -length - - [INTEGER] - NOT NULL + +task_map + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +keys + + [JSONB] + +length + + [INTEGER] + NOT NULL task_instance:run_id--task_map:run_id - -0..N -1 + +0..N +1 task_instance:dag_id--task_map:dag_id - -0..N -1 + +0..N +1 -task_instance:map_index--task_map:map_index - -0..N -1 +task_instance:task_id--task_map:task_id + +0..N +1 -task_instance:task_id--task_map:task_id - -0..N -1 +task_instance:map_index--task_map:map_index + +0..N +1 @@ -2281,82 +2281,82 @@ task_instance:id--task_reschedule:ti_id - + 0..N -1 +1 xcom - -xcom - -dag_run_id - - [INTEGER] - NOT NULL - -key - - [VARCHAR(512)] - NOT NULL - -map_index - - [INTEGER] - NOT NULL - -task_id - - [VARCHAR(250)] - NOT NULL - -dag_id - - [VARCHAR(250)] - NOT NULL - -run_id - - [VARCHAR(250)] - NOT NULL - -timestamp - - [TIMESTAMP] - NOT NULL - -value - - [JSONB] + +xcom + +dag_run_id + + [INTEGER] + NOT NULL + +key + + [VARCHAR(512)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL + +task_id + + [VARCHAR(250)] + NOT NULL + +dag_id + + [VARCHAR(250)] + NOT NULL + +run_id + + [VARCHAR(250)] + NOT NULL + +timestamp + + [TIMESTAMP] + NOT NULL + +value + + [JSONB] -task_instance:run_id--xcom:run_id - -0..N -1 +task_instance:task_id--xcom:task_id + +0..N +1 task_instance:map_index--xcom:map_index - -0..N -1 + +0..N +1 -task_instance:task_id--xcom:task_id - -0..N -1 +task_instance:dag_id--xcom:dag_id + +0..N +1 -task_instance:dag_id--xcom:dag_id - -0..N -1 +task_instance:run_id--xcom:run_id + +0..N +1 @@ -2390,9 +2390,9 @@ task_instance:id--task_instance_note:ti_id - + 1 -1 +1 @@ -2555,97 +2555,97 @@ -task_instance:map_index--task_instance_history:map_index - -0..N -1 +task_instance:run_id--task_instance_history:run_id + +0..N +1 task_instance:dag_id--task_instance_history:dag_id - + 0..N -1 +1 -task_instance:task_id--task_instance_history:task_id - -0..N -1 +task_instance:map_index--task_instance_history:map_index + +0..N +1 -task_instance:run_id--task_instance_history:run_id - -0..N -1 +task_instance:task_id--task_instance_history:task_id + +0..N +1 rendered_task_instance_fields - -rendered_task_instance_fields + +rendered_task_instance_fields + +dag_id + + [VARCHAR(250)] + NOT NULL + +map_index + + [INTEGER] + NOT NULL -dag_id - - [VARCHAR(250)] - NOT NULL +run_id + + [VARCHAR(250)] + NOT NULL -map_index - - [INTEGER] - NOT NULL +task_id + + [VARCHAR(250)] + NOT NULL -run_id - - [VARCHAR(250)] - NOT NULL +k8s_pod_yaml + + [JSON] -task_id - - [VARCHAR(250)] - NOT NULL - -k8s_pod_yaml - - [JSON] - -rendered_fields - - [JSON] - NOT NULL +rendered_fields + + [JSON] + NOT NULL -task_instance:run_id--rendered_task_instance_fields:run_id - -0..N -1 +task_instance:dag_id--rendered_task_instance_fields:dag_id + +0..N +1 task_instance:map_index--rendered_task_instance_fields:map_index - -0..N -1 + +0..N +1 -task_instance:dag_id--rendered_task_instance_fields:dag_id - -0..N -1 +task_instance:task_id--rendered_task_instance_fields:task_id + +0..N +1 -task_instance:task_id--rendered_task_instance_fields:task_id - -0..N -1 +task_instance:run_id--rendered_task_instance_fields:run_id + +0..N +1 - + deadline_alert:id--deadline:deadline_alert_id 0..N @@ -2725,13 +2725,13 @@ alembic_version - -alembic_version - -version_num - - [VARCHAR(32)] - NOT NULL + +alembic_version + +version_num + + [VARCHAR(32)] + NOT NULL diff --git a/airflow-core/src/airflow/serialization/definitions/dag.py b/airflow-core/src/airflow/serialization/definitions/dag.py index f86d55163ea1a..a82b17e7b4474 100644 --- a/airflow-core/src/airflow/serialization/definitions/dag.py +++ b/airflow-core/src/airflow/serialization/definitions/dag.py @@ -30,7 +30,6 @@ import structlog from sqlalchemy import func, or_, select, tuple_ -from airflow._shared.observability.metrics.stats import Stats from airflow._shared.timezones.timezone import coerce_datetime from airflow.configuration import conf as airflow_conf from airflow.exceptions import AirflowException, TaskNotFound @@ -41,7 +40,7 @@ from airflow.models.deadline_alert import DeadlineAlert as DeadlineAlertModel from airflow.models.taskinstancekey import TaskInstanceKey from airflow.models.tasklog import LogTemplate -from airflow.observability.stats import Stats +from airflow.sdk._shared.observability.metrics.stats import Stats from airflow.sdk.definitions.deadline import DeadlineAlert, DeadlineReference from airflow.serialization.definitions.deadline import DeadlineAlertFields from airflow.serialization.definitions.param import SerializedParamsDict From b28b5a6be5c7b4e2216a2c85c7df6f4f67729f01 Mon Sep 17 00:00:00 2001 From: ferruzzi Date: Mon, 19 Jan 2026 12:18:22 -0800 Subject: [PATCH 14/14] fix some alchemy 2 syntax --- .../src/airflow/serialization/definitions/dag.py | 8 +++----- airflow-core/tests/unit/models/test_deadline_alert.py | 9 +++++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/airflow-core/src/airflow/serialization/definitions/dag.py b/airflow-core/src/airflow/serialization/definitions/dag.py index a82b17e7b4474..83c83e43634a8 100644 --- a/airflow-core/src/airflow/serialization/definitions/dag.py +++ b/airflow-core/src/airflow/serialization/definitions/dag.py @@ -638,11 +638,9 @@ def _process_dagrun_deadline_alerts( return # Query deadline alerts by serialized_dag_id - deadline_alert_records = ( - session.query(DeadlineAlertModel) - .filter(DeadlineAlertModel.serialized_dag_id == serialized_dag_id) - .all() - ) + deadline_alert_records = session.scalars( + select(DeadlineAlertModel).where(DeadlineAlertModel.serialized_dag_id == serialized_dag_id) + ).all() for deadline_alert in deadline_alert_records: if not deadline_alert: diff --git a/airflow-core/tests/unit/models/test_deadline_alert.py b/airflow-core/tests/unit/models/test_deadline_alert.py index 7ee8f3e51e608..17954e9ab3d58 100644 --- a/airflow-core/tests/unit/models/test_deadline_alert.py +++ b/airflow-core/tests/unit/models/test_deadline_alert.py @@ -18,6 +18,7 @@ import pytest import time_machine +from sqlalchemy import select from airflow.models.deadline import ReferenceModels from airflow.models.deadline_alert import DeadlineAlert @@ -49,7 +50,9 @@ def deadline_alert_orm(dag_maker, session, deadline_reference): with dag_maker(DAG_ID, session=session): pass - serialized_dag = session.query(SerializedDagModel).filter_by(dag_id=DAG_ID).one() + serialized_dag = session.execute( + select(SerializedDagModel).where(SerializedDagModel.dag_id == DAG_ID) + ).scalar_one() with time_machine.travel(DEFAULT_DATE, tick=False): alert = DeadlineAlert( @@ -85,7 +88,9 @@ def test_minimal_deadline_alert_creation(self, dag_maker, session, deadline_refe with dag_maker(DAG_ID, session=session): pass - serialized_dag = session.query(SerializedDagModel).filter_by(dag_id=DAG_ID).one() + serialized_dag = session.execute( + select(SerializedDagModel).where(SerializedDagModel.dag_id == DAG_ID) + ).scalar_one() with time_machine.travel(DEFAULT_DATE, tick=False): deadline_alert = DeadlineAlert(