From d958b2a2f0260dbd6c5f981498c37f9080937690 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 13:40:19 +0000 Subject: [PATCH 01/21] feat: add Microsoft Fabric support to elementary CLI - Add dbt-fabric as optional dependency with fabric extra in pyproject.toml - Update packages.yml to point to Fabric-enabled dbt-data-reliability branch - Add fabric__get_adapter_unique_id macro (uses target.server) - Add Fabric transient error patterns to transient_errors.py - Add Fabric to data type mappings in test schema.yml - Update package-lock.yml for new dbt-data-reliability revision Co-Authored-By: Itamar Hartstein --- elementary/clients/dbt/transient_errors.py | 8 ++++++++ .../dbt_project/macros/get_adapter_type_and_unique_id.sql | 4 ++++ elementary/monitor/dbt_project/package-lock.yml | 8 +++++--- elementary/monitor/dbt_project/packages.yml | 2 +- pyproject.toml | 4 +++- tests/e2e_dbt_project/models/schema.yml | 4 ++-- 6 files changed, 23 insertions(+), 7 deletions(-) diff --git a/elementary/clients/dbt/transient_errors.py b/elementary/clients/dbt/transient_errors.py index 14882ab5e..3874193d4 100644 --- a/elementary/clients/dbt/transient_errors.py +++ b/elementary/clients/dbt/transient_errors.py @@ -109,6 +109,14 @@ # DuckDB runs in-process; transient errors are rare. # Common patterns (connection reset, broken pipe) are in _COMMON. ), + "fabric": ( + "connection timed out", + "could not connect to the server", + "ssl syscall error", + "communication link failure", + "tcp provider: an existing connection was forcibly closed", + "login timeout expired", + ), } # Pre-computed union of all adapter-specific patterns for the fallback path diff --git a/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql b/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql index 2a021d3b5..3a9a3c186 100644 --- a/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql +++ b/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql @@ -21,3 +21,7 @@ {% macro duckdb__get_adapter_unique_id() %} {{ return(target.path) }} {% endmacro %} + +{% macro fabric__get_adapter_unique_id() %} + {{ return(target.server) }} +{% endmacro %} diff --git a/elementary/monitor/dbt_project/package-lock.yml b/elementary/monitor/dbt_project/package-lock.yml index 48102b18e..df46d6613 100644 --- a/elementary/monitor/dbt_project/package-lock.yml +++ b/elementary/monitor/dbt_project/package-lock.yml @@ -1,6 +1,8 @@ packages: - - package: dbt-labs/dbt_utils + - name: dbt_utils + package: dbt-labs/dbt_utils version: 0.8.6 - git: https://github.com/elementary-data/dbt-data-reliability.git - revision: ab21363935c42490a60a779557ba99bed96b754c -sha1_hash: 661e08669f6a005c445ed631a333de316b58d57f + name: elementary + revision: 72c3f8e9d35bb71bc03e66046de360680156a293 +sha1_hash: a84c14a4032dd29a0a679e48976d600a25582a86 diff --git a/elementary/monitor/dbt_project/packages.yml b/elementary/monitor/dbt_project/packages.yml index 7e8c9cea4..803d3f314 100644 --- a/elementary/monitor/dbt_project/packages.yml +++ b/elementary/monitor/dbt_project/packages.yml @@ -2,7 +2,7 @@ packages: - package: dbt-labs/dbt_utils version: [">=0.8.0", "<0.9.0"] - git: https://github.com/elementary-data/dbt-data-reliability.git - revision: ab21363935c42490a60a779557ba99bed96b754c + revision: devin/ELE-5282-1772640713 # NOTE - for unreleased CLI versions we often need to update the package version to a commit hash (please leave this # commented, so it will be easy to access) diff --git a/pyproject.toml b/pyproject.toml index 6043c7b10..02c6359d8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ dbt-trino = {version = ">=1.5.0,<2.0.0", optional = true} dbt-clickhouse = {version = ">=0.20,<2.0.0", optional = true} dbt-duckdb = {version = ">=1.5.0,<2.0.0", optional = true} dbt-dremio = {version = ">=1.5.0,<2.0.0", optional = true} +dbt-fabric = {version = ">=1.4,<2.0.0", optional = true} [tool.poetry.extras] snowflake = ["dbt-snowflake"] bigquery = ["dbt-bigquery"] @@ -68,7 +69,8 @@ clickhouse = ["dbt-clickhouse"] trino = ["dbt-trino"] duckdb = ["dbt-duckdb"] dremio = ["dbt-dremio"] -all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-clickhouse", "dbt-athena-community", "dbt-trino", "dbt-duckdb", "dbt-dremio"] +fabric = ["dbt-fabric"] +all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-clickhouse", "dbt-athena-community", "dbt-trino", "dbt-duckdb", "dbt-dremio", "dbt-fabric"] [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/e2e_dbt_project/models/schema.yml b/tests/e2e_dbt_project/models/schema.yml index 7934fae2a..89e6aea84 100644 --- a/tests/e2e_dbt_project/models/schema.yml +++ b/tests/e2e_dbt_project/models/schema.yml @@ -347,7 +347,7 @@ models: - name: groups columns: - name: group_a - data_type: "{{ 'strIng' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb') else 'CHArACTER varying' if target.type == 'redshift' else 'teXt' }}" + data_type: "{{ 'strIng' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb' or target.type == 'fabric') else 'CHArACTER varying' if target.type == 'redshift' else 'teXt' }}" - name: group_b data_type: double - name: group_c @@ -365,7 +365,7 @@ models: - name: stats_players columns: - name: player - data_type: "{{ 'STRING' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb') else 'character varying' if target.type == 'redshift' else 'TEXT' }}" + data_type: "{{ 'STRING' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb' or target.type == 'fabric') else 'character varying' if target.type == 'redshift' else 'TEXT' }}" - name: goals data_type: BOOLEAN - name: coffee_cups_consumed From ac95204cc9074d3a6519af585acd182ba000bd54 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 13:40:38 +0000 Subject: [PATCH 02/21] test: add Fabric target to test profiles Add local Docker SQL Server profile for Fabric adapter testing. Uses same local Docker test credentials pattern as other adapters (e.g. postgres admin/admin, dremio dremio123). Co-Authored-By: Itamar Hartstein --- tests/profiles/profiles.yml.j2 | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/profiles/profiles.yml.j2 b/tests/profiles/profiles.yml.j2 index 77138bf93..cca5989dd 100644 --- a/tests/profiles/profiles.yml.j2 +++ b/tests/profiles/profiles.yml.j2 @@ -60,6 +60,18 @@ elementary_tests: file_format: delta threads: 4 + fabric: &fabric + type: fabric + driver: "ODBC Driver 18 for SQL Server" + server: 127.0.0.1 + port: 1433 + database: master + schema: {{ schema_name }} + user: sa + password: "Elementary123!" + trust_cert: true + threads: 4 + # ── Cloud targets (secrets substituted at CI time) ───────────────── snowflake: &snowflake @@ -120,7 +132,7 @@ elementary_tests: elementary: target: postgres outputs: -{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'spark', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %} +{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'spark', 'fabric', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %} {%- for t in targets %} {{ t }}: <<: *{{ t }} From 6a737e825a7c1c0f81be6584af2b876d675c722e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 13:42:43 +0000 Subject: [PATCH 03/21] feat: add SQL Server support to elementary CLI - Add dbt-sqlserver as optional dependency with sqlserver extra in pyproject.toml - Add sqlserver__get_adapter_unique_id macro (uses target.server) - Add SQL Server transient error patterns to transient_errors.py - Add sqlserver to data type mappings in test schema.yml Co-Authored-By: Itamar Hartstein --- elementary/clients/dbt/transient_errors.py | 8 ++++++++ .../dbt_project/macros/get_adapter_type_and_unique_id.sql | 4 ++++ pyproject.toml | 4 +++- tests/e2e_dbt_project/models/schema.yml | 4 ++-- 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/elementary/clients/dbt/transient_errors.py b/elementary/clients/dbt/transient_errors.py index 3874193d4..fb5b371d8 100644 --- a/elementary/clients/dbt/transient_errors.py +++ b/elementary/clients/dbt/transient_errors.py @@ -117,6 +117,14 @@ "tcp provider: an existing connection was forcibly closed", "login timeout expired", ), + "sqlserver": ( + "connection timed out", + "could not connect to the server", + "ssl syscall error", + "communication link failure", + "tcp provider: an existing connection was forcibly closed", + "login timeout expired", + ), } # Pre-computed union of all adapter-specific patterns for the fallback path diff --git a/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql b/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql index 3a9a3c186..30000b440 100644 --- a/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql +++ b/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql @@ -25,3 +25,7 @@ {% macro fabric__get_adapter_unique_id() %} {{ return(target.server) }} {% endmacro %} + +{% macro sqlserver__get_adapter_unique_id() %} + {{ return(target.server) }} +{% endmacro %} diff --git a/pyproject.toml b/pyproject.toml index 02c6359d8..057f208c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ dbt-clickhouse = {version = ">=0.20,<2.0.0", optional = true} dbt-duckdb = {version = ">=1.5.0,<2.0.0", optional = true} dbt-dremio = {version = ">=1.5.0,<2.0.0", optional = true} dbt-fabric = {version = ">=1.4,<2.0.0", optional = true} +dbt-sqlserver = {version = ">=1.4,<2.0.0", optional = true} [tool.poetry.extras] snowflake = ["dbt-snowflake"] bigquery = ["dbt-bigquery"] @@ -70,7 +71,8 @@ trino = ["dbt-trino"] duckdb = ["dbt-duckdb"] dremio = ["dbt-dremio"] fabric = ["dbt-fabric"] -all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-clickhouse", "dbt-athena-community", "dbt-trino", "dbt-duckdb", "dbt-dremio", "dbt-fabric"] +sqlserver = ["dbt-sqlserver"] +all = ["dbt-snowflake", "dbt-bigquery", "dbt-redshift", "dbt-postgres", "dbt-databricks", "dbt-spark", "dbt-clickhouse", "dbt-athena-community", "dbt-trino", "dbt-duckdb", "dbt-dremio", "dbt-fabric", "dbt-sqlserver"] [build-system] requires = ["poetry-core>=1.0.0"] diff --git a/tests/e2e_dbt_project/models/schema.yml b/tests/e2e_dbt_project/models/schema.yml index 89e6aea84..c1eeb09b6 100644 --- a/tests/e2e_dbt_project/models/schema.yml +++ b/tests/e2e_dbt_project/models/schema.yml @@ -347,7 +347,7 @@ models: - name: groups columns: - name: group_a - data_type: "{{ 'strIng' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb' or target.type == 'fabric') else 'CHArACTER varying' if target.type == 'redshift' else 'teXt' }}" + data_type: "{{ 'strIng' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb' or target.type == 'fabric' or target.type == 'sqlserver') else 'CHArACTER varying' if target.type == 'redshift' else 'teXt' }}" - name: group_b data_type: double - name: group_c @@ -365,7 +365,7 @@ models: - name: stats_players columns: - name: player - data_type: "{{ 'STRING' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb' or target.type == 'fabric') else 'character varying' if target.type == 'redshift' else 'TEXT' }}" + data_type: "{{ 'STRING' if (target.type == 'bigquery' or target.type == 'databricks' or target.type == 'athena' or target.type == 'spark') else 'varchar' if (target.type == 'trino' or target.type == 'dremio' or target.type == 'duckdb' or target.type == 'fabric' or target.type == 'sqlserver') else 'character varying' if target.type == 'redshift' else 'TEXT' }}" - name: goals data_type: BOOLEAN - name: coffee_cups_consumed From b54a7a18b59bb5791bcff90847b5b42dcdb49105 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 13:42:54 +0000 Subject: [PATCH 04/21] test: add SQL Server target to test profiles Add local Docker SQL Server profile for sqlserver adapter testing, alongside the existing fabric profile. Co-Authored-By: Itamar Hartstein --- tests/profiles/profiles.yml.j2 | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/profiles/profiles.yml.j2 b/tests/profiles/profiles.yml.j2 index cca5989dd..0ae2d6537 100644 --- a/tests/profiles/profiles.yml.j2 +++ b/tests/profiles/profiles.yml.j2 @@ -72,6 +72,19 @@ elementary_tests: trust_cert: true threads: 4 + sqlserver: &sqlserver + type: sqlserver + driver: "ODBC Driver 18 for SQL Server" + server: 127.0.0.1 + port: 1433 + database: master + schema: {{ schema_name }} + user: sa + password: "Elementary123!" + encrypt: false + trust_cert: true + threads: 4 + # ── Cloud targets (secrets substituted at CI time) ───────────────── snowflake: &snowflake @@ -132,7 +145,7 @@ elementary_tests: elementary: target: postgres outputs: -{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'spark', 'fabric', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %} +{%- set targets = ['postgres', 'clickhouse', 'trino', 'dremio', 'duckdb', 'spark', 'fabric', 'sqlserver', 'snowflake', 'bigquery', 'redshift', 'databricks_catalog', 'athena'] %} {%- for t in targets %} {{ t }}: <<: *{{ t }} From 62a36e612ccd10d6188a8ecec2936f6d6e76b4b7 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 13:48:46 +0000 Subject: [PATCH 05/21] ci: add Fabric and SQL Server to CI workflows - Add fabric and sqlserver to warehouse-type matrix in test-all-warehouses.yml - Add fabric and sqlserver to workflow_dispatch options in test-warehouse.yml - Add SQL Server Docker service to docker-compose.yml - Add SQL Server startup step and ODBC driver install to CI workflow - Mark fabric/sqlserver as Docker-based adapters for schema naming Co-Authored-By: Itamar Hartstein --- .github/workflows/test-all-warehouses.yml | 2 ++ .github/workflows/test-warehouse.yml | 19 ++++++++++++++++++- tests/e2e_dbt_project/docker-compose.yml | 16 ++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml index 7efa247f0..de6ba8f80 100644 --- a/.github/workflows/test-all-warehouses.yml +++ b/.github/workflows/test-all-warehouses.yml @@ -100,6 +100,8 @@ jobs: trino, dremio, spark, + fabric, + sqlserver, ] uses: ./.github/workflows/test-warehouse.yml with: diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 5ed0ed99b..8cfd5adcc 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -18,6 +18,8 @@ on: - duckdb - trino - dremio + - fabric + - sqlserver elementary-ref: type: string required: false @@ -159,6 +161,13 @@ jobs: run: | docker compose up -d --build --wait spark-thrift + - name: Start SQL Server + if: inputs.warehouse-type == 'fabric' || inputs.warehouse-type == 'sqlserver' + working-directory: ${{ env.E2E_DBT_PROJECT_DIR }} + run: | + docker compose up -d sqlserver + timeout 120 bash -c 'until docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "${MSSQL_SA_PASSWORD:-Elementary123!}" -C -Q "SELECT 1" -b 2>/dev/null; do sleep 5; done' + - name: Setup Python uses: actions/setup-python@v5 with: @@ -168,6 +177,14 @@ jobs: if: inputs.warehouse-type == 'spark' run: sudo apt-get install -y python3-dev libsasl2-dev gcc + - name: Install ODBC driver for SQL Server + if: inputs.warehouse-type == 'fabric' || inputs.warehouse-type == 'sqlserver' + run: | + curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | sudo tee /etc/apt/trusted.gpg.d/microsoft.asc > /dev/null + curl -fsSL https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list > /dev/null + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 unixodbc-dev + - name: Install dbt run: > pip install @@ -188,7 +205,7 @@ jobs: # This enables caching the seeded database state between runs. IS_DOCKER=false case "${{ inputs.warehouse-type }}" in - postgres|clickhouse|trino|dremio|duckdb|spark) IS_DOCKER=true ;; + postgres|clickhouse|trino|dremio|duckdb|spark|fabric|sqlserver) IS_DOCKER=true ;; esac if [ "$IS_DOCKER" = "true" ]; then diff --git a/tests/e2e_dbt_project/docker-compose.yml b/tests/e2e_dbt_project/docker-compose.yml index 47b4c4bb7..542d31537 100644 --- a/tests/e2e_dbt_project/docker-compose.yml +++ b/tests/e2e_dbt_project/docker-compose.yml @@ -290,6 +290,22 @@ services: timeout: 5s retries: 10 + # ── SQL Server (for Fabric / SQL Server adapters) ───────────────── + sqlserver: + image: mcr.microsoft.com/mssql/server:2022-latest + container_name: sqlserver + ports: + - "127.0.0.1:1433:1433" + environment: + ACCEPT_EULA: "Y" + MSSQL_SA_PASSWORD: "${MSSQL_SA_PASSWORD:-Elementary123!}" + MSSQL_PID: "Developer" + healthcheck: + test: /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "${MSSQL_SA_PASSWORD:-Elementary123!}" -C -Q "SELECT 1" -b + interval: 10s + timeout: 5s + retries: 10 + networks: dremio-lakehouse: From ba52254bb039e3dbd8575b1da0e27959bc5aadcd Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 14:03:53 +0000 Subject: [PATCH 06/21] ci: temporarily limit matrix to fabric+sqlserver, use ELE-5282 branch for dbt-data-reliability Co-Authored-By: Itamar Hartstein --- .github/workflows/test-all-warehouses.yml | 27 +++++++++++------------ 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml index de6ba8f80..3a0dd3fe8 100644 --- a/.github/workflows/test-all-warehouses.yml +++ b/.github/workflows/test-all-warehouses.yml @@ -87,19 +87,18 @@ jobs: fail-fast: false matrix: dbt-version: ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) || fromJSON('[null]') }} - warehouse-type: - [ - postgres, - snowflake, - bigquery, - redshift, - databricks_catalog, - athena, - clickhouse, - duckdb, - trino, - dremio, - spark, + warehouse-type: [ + # postgres, + # snowflake, + # bigquery, + # redshift, + # databricks_catalog, + # athena, + # clickhouse, + # duckdb, + # trino, + # dremio, + # spark, fabric, sqlserver, ] @@ -107,7 +106,7 @@ jobs: with: warehouse-type: ${{ matrix.warehouse-type }} elementary-ref: ${{ inputs.elementary-ref || ((github.event_name == 'pull_request_target' || github.event_name == 'pull_request') && github.event.pull_request.head.sha) || '' }} - dbt-data-reliability-ref: ${{ inputs.dbt-data-reliability-ref }} + dbt-data-reliability-ref: ${{ inputs.dbt-data-reliability-ref || 'devin/ELE-5282-1772640713' }} dbt-version: ${{ matrix.dbt-version }} generate-data: ${{ inputs.generate-data || false }} secrets: inherit From cfc9b6807f2b00a3802364ce5b1c4c266017ea8b Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 14:12:01 +0000 Subject: [PATCH 07/21] fix: use cloud auth (ServicePrincipal) for fabric profile, keep sqlserver as Docker target dbt-fabric only supports Azure AD authentication methods and cannot connect to a local SQL Server Docker container with SQL auth. Move fabric from Docker targets to cloud targets using ServicePrincipal authentication (matching the dbt-data-reliability configuration). Co-Authored-By: Itamar Hartstein --- .github/workflows/test-warehouse.yml | 4 ++-- tests/profiles/profiles.yml.j2 | 27 +++++++++++++++------------ 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 8cfd5adcc..8a0c526bd 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -162,7 +162,7 @@ jobs: docker compose up -d --build --wait spark-thrift - name: Start SQL Server - if: inputs.warehouse-type == 'fabric' || inputs.warehouse-type == 'sqlserver' + if: inputs.warehouse-type == 'sqlserver' working-directory: ${{ env.E2E_DBT_PROJECT_DIR }} run: | docker compose up -d sqlserver @@ -205,7 +205,7 @@ jobs: # This enables caching the seeded database state between runs. IS_DOCKER=false case "${{ inputs.warehouse-type }}" in - postgres|clickhouse|trino|dremio|duckdb|spark|fabric|sqlserver) IS_DOCKER=true ;; + postgres|clickhouse|trino|dremio|duckdb|spark|sqlserver) IS_DOCKER=true ;; esac if [ "$IS_DOCKER" = "true" ]; then diff --git a/tests/profiles/profiles.yml.j2 b/tests/profiles/profiles.yml.j2 index 0ae2d6537..3543496c6 100644 --- a/tests/profiles/profiles.yml.j2 +++ b/tests/profiles/profiles.yml.j2 @@ -60,18 +60,6 @@ elementary_tests: file_format: delta threads: 4 - fabric: &fabric - type: fabric - driver: "ODBC Driver 18 for SQL Server" - server: 127.0.0.1 - port: 1433 - database: master - schema: {{ schema_name }} - user: sa - password: "Elementary123!" - trust_cert: true - threads: 4 - sqlserver: &sqlserver type: sqlserver driver: "ODBC Driver 18 for SQL Server" @@ -87,6 +75,21 @@ elementary_tests: # ── Cloud targets (secrets substituted at CI time) ───────────────── + fabric: &fabric + type: fabric + driver: "ODBC Driver 18 for SQL Server" + server: {{ fabric_server | toyaml }} + port: 1433 + database: {{ fabric_database | toyaml }} + schema: {{ schema_name }} + authentication: ServicePrincipal + tenant_id: {{ fabric_tenant_id | toyaml }} + client_id: {{ fabric_client_id | toyaml }} + client_secret: {{ fabric_client_secret | toyaml }} + encrypt: true + trust_cert: false + threads: 4 + snowflake: &snowflake type: snowflake account: {{ snowflake_account | toyaml }} From 7b51cd76d869aa8fa93750da6472723278ee65bf Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 14:25:15 +0000 Subject: [PATCH 08/21] fix: replace || with concat() for T-SQL compatibility in CLI dbt macros SQL Server / T-SQL does not support the || operator for string concatenation. Replace all occurrences with the cross-database concat() function which is supported by all target databases. Co-Authored-By: Itamar Hartstein --- .../dbt_project/macros/alerts/population/test_alerts.sql | 4 ++-- .../macros/base_queries/current_tests_run_results_query.sql | 2 +- elementary/monitor/dbt_project/macros/get_test_results.sql | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 0ce7dc0da..0dc2da064 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -128,12 +128,12 @@ select distinct failed_tests.alert_id, {# Generate elementary unique id which is used to identify between tests, and set it as alert_class_id #} - coalesce(failed_tests.test_unique_id, 'None') || '.' || coalesce(failed_tests.column_name, 'None') || '.' || coalesce(failed_tests.sub_type, 'None') as alert_class_id, + concat(coalesce(failed_tests.test_unique_id, 'None'), '.', coalesce(failed_tests.column_name, 'None'), '.', coalesce(failed_tests.sub_type, 'None')) as alert_class_id, case when failed_tests.test_type = 'schema_change' then failed_tests.test_unique_id {# In old versions of elementary, elementary_test_results doesn't contain test_short_name, so we use dbt_test short_name. #} when tests.short_name = 'dimension_anomalies' then failed_tests.test_unique_id - else coalesce(failed_tests.test_unique_id, 'None') || '.' || coalesce(failed_tests.column_name, 'None') || '.' || coalesce(failed_tests.sub_type, 'None') + else concat(coalesce(failed_tests.test_unique_id, 'None'), '.', coalesce(failed_tests.column_name, 'None'), '.', coalesce(failed_tests.sub_type, 'None')) end as elementary_unique_id, failed_tests.data_issue_id, failed_tests.test_execution_id, diff --git a/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql b/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql index b10bc8ba6..b0b7257bf 100644 --- a/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql +++ b/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql @@ -49,7 +49,7 @@ when elementary_test_results.test_type = 'schema_change' then elementary_test_results.test_unique_id {# In old versions of elementary, elementary_test_results doesn't contain test_short_name, so we use dbt_test short_name. #} when dbt_tests.short_name = 'dimension_anomalies' then elementary_test_results.test_unique_id - else coalesce(elementary_test_results.test_unique_id, 'None') || '.' || coalesce(nullif(elementary_test_results.column_name, ''), 'None') || '.' || coalesce(elementary_test_results.test_sub_type, 'None') + else concat(coalesce(elementary_test_results.test_unique_id, 'None'), '.', coalesce(nullif(elementary_test_results.column_name, ''), 'None'), '.', coalesce(elementary_test_results.test_sub_type, 'None')) end as elementary_unique_id, elementary_test_results.invocation_id, elementary_test_results.data_issue_id, diff --git a/elementary/monitor/dbt_project/macros/get_test_results.sql b/elementary/monitor/dbt_project/macros/get_test_results.sql index 35218e678..6171be3c4 100644 --- a/elementary/monitor/dbt_project/macros/get_test_results.sql +++ b/elementary/monitor/dbt_project/macros/get_test_results.sql @@ -162,7 +162,7 @@ CASE WHEN etr.test_type = 'schema_change' THEN etr.test_unique_id WHEN dt.short_name = 'dimension_anomalies' THEN etr.test_unique_id - ELSE coalesce(etr.test_unique_id, 'None') || '.' || coalesce(nullif(etr.column_name, ''), 'None') || '.' || coalesce(etr.test_sub_type, 'None') + ELSE concat(coalesce(etr.test_unique_id, 'None'), '.', coalesce(nullif(etr.column_name, ''), 'None'), '.', coalesce(etr.test_sub_type, 'None')) END AS elementary_unique_id, etr.detected_at, etr.database_name, @@ -266,4 +266,4 @@ {%- endfor -%} {% do return(test_results) %} -{%- endmacro -%} \ No newline at end of file +{%- endmacro -%} From 1739662ebabe15aac769dd21f5f7705a00e5fa27 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 14:41:43 +0000 Subject: [PATCH 09/21] fix: replace LIMIT with TOP for T-SQL compatibility in CLI dbt macros Co-Authored-By: Itamar Hartstein --- .../monitor/dbt_project/macros/get_latest_invocation.sql | 4 ++++ .../monitor/dbt_project/macros/get_test_last_invocation.sql | 4 ++-- elementary/monitor/dbt_project/macros/test_conn.sql | 4 ++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/get_latest_invocation.sql b/elementary/monitor/dbt_project/macros/get_latest_invocation.sql index a2590e84f..c2f51aedf 100644 --- a/elementary/monitor/dbt_project/macros/get_latest_invocation.sql +++ b/elementary/monitor/dbt_project/macros/get_latest_invocation.sql @@ -7,7 +7,11 @@ {% endif %} {% set get_pkg_version_query %} + {% if target.type in ('fabric', 'sqlserver') %} + select top 1 * from {{ invocations_relation }} order by generated_at desc + {% else %} select * from {{ invocations_relation }} order by generated_at desc limit 1 + {% endif %} {% endset %} {% set result = elementary.run_query(get_pkg_version_query) %} {% if not result %} diff --git a/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql b/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql index 6b94d1ddb..e09dcc780 100644 --- a/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql +++ b/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql @@ -8,7 +8,7 @@ ), test_invocation as ( - select distinct invocation_id, detected_at + select {% if target.type in ('fabric', 'sqlserver') %}top 1{% endif %} distinct invocation_id, detected_at from elementary_test_results {% if invocation_id %} where invocation_id = {{ "'" ~ invocation_id ~ "'" }} @@ -16,7 +16,7 @@ where detected_at < {{ "'" ~ invocation_max_time ~ "'" }} {% endif %} order by detected_at desc - limit 1 + {% if target.type not in ('fabric', 'sqlserver') %}limit 1{% endif %} ) {% if invocations_relation %} diff --git a/elementary/monitor/dbt_project/macros/test_conn.sql b/elementary/monitor/dbt_project/macros/test_conn.sql index 6f1ed9b64..35bb8bbd0 100644 --- a/elementary/monitor/dbt_project/macros/test_conn.sql +++ b/elementary/monitor/dbt_project/macros/test_conn.sql @@ -6,7 +6,11 @@ {% set elementary_database, elementary_schema = elementary.get_package_database_and_schema() %} {% set elementary_model_relation = api.Relation.create(elementary_database, elementary_schema, "dbt_models") %} {% set query %} + {% if target.type in ('fabric', 'sqlserver') %} + select top 10 * from {{ elementary_model_relation }} + {% else %} select * from {{ elementary_model_relation }} limit 10 + {% endif %} {% endset %} {% do elementary.run_query(query) %} {% endmacro %} From 65d032eba466ac24f1a6cb666a8199adbb6b1f1e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 14:50:53 +0000 Subject: [PATCH 10/21] ci: trigger rebuild with updated dbt-data-reliability dispatches Co-Authored-By: Itamar Hartstein From f51dfc9b81d68304826dd56556f4313bc3bec054 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 15:08:41 +0000 Subject: [PATCH 11/21] fix: add fabric__get_test_results to avoid nested CTE issue in T-SQL T-SQL does not allow nested CTEs (WITH inside WITH). The default get_test_results macro wraps current_tests_run_results_query (which starts with WITH) inside another WITH clause, creating nested CTEs. The fabric__ dispatch materialises the inner query into a temp table first, then builds ordered_test_results on top without nesting. sqlserver inherits from fabric automatically, so no sqlserver__ override is needed. Co-Authored-By: Itamar Hartstein --- .../dbt_project/macros/get_test_results.sql | 125 ++++++++++++++++++ 1 file changed, 125 insertions(+) diff --git a/elementary/monitor/dbt_project/macros/get_test_results.sql b/elementary/monitor/dbt_project/macros/get_test_results.sql index 6171be3c4..ab23b6da2 100644 --- a/elementary/monitor/dbt_project/macros/get_test_results.sql +++ b/elementary/monitor/dbt_project/macros/get_test_results.sql @@ -107,6 +107,131 @@ {% do return(test_results) %} {%- endmacro -%} +{%- macro fabric__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} + {# + T-SQL does not allow nested CTEs (WITH inside WITH). + current_tests_run_results_query already starts with WITH, so we + cannot wrap it in another CTE. Instead we materialise it into a + temp table first, then build ordered_test_results on top. + Note: sqlserver adapter inherits from fabric, so this dispatch + covers both fabric and sqlserver targets automatically. + #} + {% set elementary_tests_allowlist_status = ['fail', 'warn'] if disable_passed_test_metrics else ['fail', 'warn', 'pass'] %} + + {# Step 1 – materialise the base test-results query into a temp table #} + {% set base_query %} + {{ elementary_cli.current_tests_run_results_query(days_back=days_back) }} + {% endset %} + + {% set elementary_database, elementary_schema = elementary.get_package_database_and_schema() %} + {% set base_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'test_results_base', base_query) %} + + {# Step 2 – build ordered_test_results from the materialised base (no nested CTE) #} + {% set select_ordered %} + select + *, + {{ elementary.edr_datediff(elementary.edr_cast_as_timestamp('detected_at'), elementary.edr_current_timestamp(), 'day') }} as days_diff, + row_number() over (partition by elementary_unique_id order by {{elementary.edr_cast_as_timestamp('detected_at')}} desc) as invocations_rank_index + from {{ base_relation }} + {% endset %} + + {% set ordered_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'ordered_test_results', select_ordered) %} + + {# Step 3 – final select with column pruning and filtering #} + {% set select_test_results %} + select + test_results.id, + test_results.invocation_id, + test_results.test_execution_id, + test_results.model_unique_id, + test_results.test_unique_id, + test_results.elementary_unique_id, + {{elementary.edr_cast_as_timestamp('test_results.detected_at')}} as detected_at, + test_results.database_name, + test_results.schema_name, + test_results.table_name, + test_results.column_name, + test_results.test_type, + test_results.test_sub_type, + test_results.test_results_description, + test_results.test_description, + test_results.original_path, + test_results.package_name, + test_results.owners, + test_results.model_owner, + test_results.tags, + test_results.test_tags, + test_results.model_tags, + test_results.meta, + test_results.model_meta, + case when test_results.invocations_rank_index = 1 then test_results.test_results_query else NULL end as test_results_query, + test_results.other, + test_results.test_name, + test_results.test_params, + test_results.severity, + test_results.status, + test_results.execution_time, + test_results.days_diff, + test_results.invocations_rank_index, + test_results.failures, + test_results.result_rows + from {{ ordered_relation }} as test_results + where test_results.invocations_rank_index <= {{ invocations_per_test }} + order by test_results.elementary_unique_id, test_results.invocations_rank_index desc + {%- endset -%} + + {% set final_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'final_test_results', select_test_results) %} + + {% set test_results = [] %} + + {% set test_results_agate_sql %} + select * from {{ final_relation }} + {% endset %} + + {% set valid_ids_query %} + select distinct id + from {{ final_relation }} + where invocations_rank_index = 1 + {% endset %} + + {% set test_results_agate = elementary.run_query(test_results_agate_sql) %} + {% set test_result_rows_agate = elementary_cli.get_result_rows_agate(days_back, valid_ids_query) %} + + {# Clean up intermediate tables #} + {% do elementary.fully_drop_relation(base_relation) %} + {% do elementary.fully_drop_relation(ordered_relation) %} + {% if not elementary.has_temp_table_support() %} + {% do elementary.fully_drop_relation(final_relation) %} + {% endif %} + + {% set tests = elementary.agate_to_dicts(test_results_agate) %} + + {% set filtered_tests = [] %} + {% for test in tests %} + {% set test_meta = fromjson(test.meta) %} + {% if test_meta.get("elementary", {}).get("include", true) %} + {% do filtered_tests.append(test) %} + {% endif %} + {% endfor %} + + {% for test in filtered_tests %} + {% set test_rows_sample = none %} + {% if test.invocations_rank_index == 1 %} + {% set test_type = test.test_type %} + {% set test_params = fromjson(test.test_params) %} + {% set status = test.status | lower %} + + {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} + {%- endif -%} + {% endif %} + {# Adding sample data to test results #} + {% do test.update({"sample_data": test_rows_sample}) %} + {% do test_results.append(test) %} + {%- endfor -%} + {% do return(test_results) %} +{%- endmacro -%} + {%- macro clickhouse__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} {% do elementary.run_query('drop table if exists ordered_test_results') %} {% set create_table_query %} From dff9bc3c255e1074b6641bcef1b9c3f265b23de0 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 15:21:02 +0000 Subject: [PATCH 12/21] fix: move ORDER BY out of create_temp_table query for T-SQL compatibility T-SQL forbids ORDER BY in views/subqueries/CTEs unless TOP or OFFSET is also specified. The fabric__create_temp_table uses a VIEW workaround, so ORDER BY must be in the final SELECT, not in the materialised query. Co-Authored-By: Itamar Hartstein --- elementary/monitor/dbt_project/macros/get_test_results.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elementary/monitor/dbt_project/macros/get_test_results.sql b/elementary/monitor/dbt_project/macros/get_test_results.sql index ab23b6da2..5629dcc61 100644 --- a/elementary/monitor/dbt_project/macros/get_test_results.sql +++ b/elementary/monitor/dbt_project/macros/get_test_results.sql @@ -177,15 +177,16 @@ test_results.result_rows from {{ ordered_relation }} as test_results where test_results.invocations_rank_index <= {{ invocations_per_test }} - order by test_results.elementary_unique_id, test_results.invocations_rank_index desc {%- endset -%} {% set final_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'final_test_results', select_test_results) %} {% set test_results = [] %} + {# ORDER BY must be here, not inside create_temp_table — T-SQL forbids ORDER BY in views/subqueries without TOP #} {% set test_results_agate_sql %} select * from {{ final_relation }} + order by elementary_unique_id, invocations_rank_index desc {% endset %} {% set valid_ids_query %} From 699bee3993a7fd07d4535b7042aec7c1d1d76dc8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 15:35:36 +0000 Subject: [PATCH 13/21] fix: quote reserved keyword aliases (schema, database) for T-SQL compatibility Co-Authored-By: Itamar Hartstein --- .../monitor/dbt_project/macros/base_queries/resources.sql | 8 ++++---- elementary/monitor/dbt_project/macros/get_models_runs.sql | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/base_queries/resources.sql b/elementary/monitor/dbt_project/macros/base_queries/resources.sql index d0d819eec..f530f8bfd 100644 --- a/elementary/monitor/dbt_project/macros/base_queries/resources.sql +++ b/elementary/monitor/dbt_project/macros/base_queries/resources.sql @@ -7,10 +7,10 @@ select unique_id, name, - schema_name as schema, + schema_name as "schema", tags, owner as owners, - database_name as database + database_name as "database" from dbt_models {% if exclude_elementary %} where package_name != 'elementary' @@ -31,10 +31,10 @@ unique_id, name, source_name, - schema_name AS schema, + schema_name AS "schema", tags, owner AS owners, - database_name as database + database_name as "database" from dbt_sources {% if exclude_elementary %} where package_name != 'elementary' diff --git a/elementary/monitor/dbt_project/macros/get_models_runs.sql b/elementary/monitor/dbt_project/macros/get_models_runs.sql index dc8851780..6b91bcb5b 100644 --- a/elementary/monitor/dbt_project/macros/get_models_runs.sql +++ b/elementary/monitor/dbt_project/macros/get_models_runs.sql @@ -11,7 +11,7 @@ unique_id, invocation_id, name, - schema_name as schema, + schema_name as "schema", status, case when status != 'success' then 0 From 1f239034bebc5ca9cde670ced27438bd471820b8 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 16:00:14 +0000 Subject: [PATCH 14/21] ci: restore all adapters in CI matrix after fabric+sqlserver pass Co-Authored-By: Itamar Hartstein --- .github/workflows/test-all-warehouses.yml | 25 ++++++++++++----------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml index 3a0dd3fe8..12e8eac46 100644 --- a/.github/workflows/test-all-warehouses.yml +++ b/.github/workflows/test-all-warehouses.yml @@ -87,18 +87,19 @@ jobs: fail-fast: false matrix: dbt-version: ${{ inputs.dbt-version && fromJSON(format('["{0}"]', inputs.dbt-version)) || fromJSON('[null]') }} - warehouse-type: [ - # postgres, - # snowflake, - # bigquery, - # redshift, - # databricks_catalog, - # athena, - # clickhouse, - # duckdb, - # trino, - # dremio, - # spark, + warehouse-type: + [ + postgres, + snowflake, + bigquery, + redshift, + databricks_catalog, + athena, + clickhouse, + duckdb, + trino, + dremio, + spark, fabric, sqlserver, ] From a86d990b0acee530d622aa722815d2c773f7918d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 16:18:50 +0000 Subject: [PATCH 15/21] fix: use adapter dispatch for reserved keyword aliases instead of double-quotes Double-quoted identifiers break Spark SQL. Use adapter dispatch with fabric__ variants that quote 'schema' and 'database' using T-SQL square brackets, while the default keeps the original unquoted aliases. Co-Authored-By: Itamar Hartstein --- .../macros/base_queries/resources.sql | 67 +++++++++++++++++-- .../dbt_project/macros/get_models_runs.sql | 43 +++++++++++- 2 files changed, 105 insertions(+), 5 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/base_queries/resources.sql b/elementary/monitor/dbt_project/macros/base_queries/resources.sql index f530f8bfd..894f7762c 100644 --- a/elementary/monitor/dbt_project/macros/base_queries/resources.sql +++ b/elementary/monitor/dbt_project/macros/base_queries/resources.sql @@ -1,4 +1,33 @@ {% macro get_model_resources(exclude_elementary=true) %} + {{ return(adapter.dispatch('get_model_resources', 'elementary_cli')(exclude_elementary)) }} +{% endmacro %} + + +{% macro default__get_model_resources(exclude_elementary=true) %} + {% set model_resources_query %} + with dbt_models as ( + select * from {{ ref('elementary', 'dbt_models') }} + ) + + select + unique_id, + name, + schema_name as schema, + tags, + owner as owners, + database_name as database + from dbt_models + {% if exclude_elementary %} + where package_name != 'elementary' + {% endif %} + {% endset %} + {% set models_agate = run_query(model_resources_query) %} + {% do return(elementary.agate_to_dicts(models_agate)) %} +{% endmacro %} + + +{% macro fabric__get_model_resources(exclude_elementary=true) %} + {# T-SQL: 'schema' and 'database' are reserved keywords — quote with square brackets #} {% set model_resources_query %} with dbt_models as ( select * from {{ ref('elementary', 'dbt_models') }} @@ -7,10 +36,10 @@ select unique_id, name, - schema_name as "schema", + schema_name as [schema], tags, owner as owners, - database_name as "database" + database_name as [database] from dbt_models {% if exclude_elementary %} where package_name != 'elementary' @@ -22,6 +51,36 @@ {% macro get_source_resources(exclude_elementary=true) %} + {{ return(adapter.dispatch('get_source_resources', 'elementary_cli')(exclude_elementary)) }} +{% endmacro %} + + +{% macro default__get_source_resources(exclude_elementary=true) %} + {% set source_resources_query %} + with dbt_sources as ( + select * from {{ ref('elementary', 'dbt_sources') }} + ) + + select + unique_id, + name, + source_name, + schema_name AS schema, + tags, + owner AS owners, + database_name as database + from dbt_sources + {% if exclude_elementary %} + where package_name != 'elementary' + {% endif %} + {% endset %} + {% set sources_agate = run_query(source_resources_query) %} + {% do return(elementary.agate_to_dicts(sources_agate)) %} +{% endmacro %} + + +{% macro fabric__get_source_resources(exclude_elementary=true) %} + {# T-SQL: 'schema' and 'database' are reserved keywords — quote with square brackets #} {% set source_resources_query %} with dbt_sources as ( select * from {{ ref('elementary', 'dbt_sources') }} @@ -31,10 +90,10 @@ unique_id, name, source_name, - schema_name AS "schema", + schema_name AS [schema], tags, owner AS owners, - database_name as "database" + database_name as [database] from dbt_sources {% if exclude_elementary %} where package_name != 'elementary' diff --git a/elementary/monitor/dbt_project/macros/get_models_runs.sql b/elementary/monitor/dbt_project/macros/get_models_runs.sql index 6b91bcb5b..6a9023670 100644 --- a/elementary/monitor/dbt_project/macros/get_models_runs.sql +++ b/elementary/monitor/dbt_project/macros/get_models_runs.sql @@ -1,4 +1,45 @@ {%- macro get_models_runs(days_back = 7, exclude_elementary=false) -%} + {{ return(adapter.dispatch('get_models_runs', 'elementary_cli')(days_back, exclude_elementary)) }} +{%- endmacro -%} + + +{%- macro default__get_models_runs(days_back = 7, exclude_elementary=false) -%} + {% set models_runs_query %} + with model_runs as ( + select + *, + row_number() over (partition by unique_id order by generated_at desc) as invocations_rank_index + from {{ ref('elementary', 'model_run_results') }} + ) + + select + unique_id, + invocation_id, + name, + schema_name as schema, + status, + case + when status != 'success' then 0 + else round({{ elementary.edr_cast_as_numeric('execution_time') }}, 1) + end as execution_time, + full_refresh, + materialization, + case when invocations_rank_index = 1 then compiled_code else NULL end as compiled_code, + generated_at + from model_runs + where {{ elementary.edr_datediff(elementary.edr_cast_as_timestamp('generated_at'), elementary.edr_current_timestamp(), 'day') }} < {{ days_back }} + {% if exclude_elementary %} + and unique_id not like 'model.elementary.%' + {% endif %} + order by generated_at + {% endset %} + {% set models_runs_agate = run_query(models_runs_query) %} + {% do return(elementary.agate_to_dicts(models_runs_agate)) %} +{%- endmacro -%} + + +{%- macro fabric__get_models_runs(days_back = 7, exclude_elementary=false) -%} + {# T-SQL: 'schema' and 'database' are reserved keywords — quote with square brackets #} {% set models_runs_query %} with model_runs as ( select @@ -11,7 +52,7 @@ unique_id, invocation_id, name, - schema_name as "schema", + schema_name as [schema], status, case when status != 'success' then 0 From 613d8ed8bf22100c4f0059d515265007d4f39a26 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 17:05:16 +0000 Subject: [PATCH 16/21] fix: use dbt.concat() cross-DB macro instead of concat() for Redshift compatibility Redshift's concat() only accepts 2 arguments. The dbt.concat() macro generates the correct SQL per adapter (|| on Postgres/Redshift/Snowflake, concat() on others). Co-Authored-By: Itamar Hartstein --- .../dbt_project/macros/alerts/population/test_alerts.sql | 4 ++-- .../macros/base_queries/current_tests_run_results_query.sql | 2 +- elementary/monitor/dbt_project/macros/get_test_results.sql | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql index 0dc2da064..bdc2a9253 100644 --- a/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/population/test_alerts.sql @@ -128,12 +128,12 @@ select distinct failed_tests.alert_id, {# Generate elementary unique id which is used to identify between tests, and set it as alert_class_id #} - concat(coalesce(failed_tests.test_unique_id, 'None'), '.', coalesce(failed_tests.column_name, 'None'), '.', coalesce(failed_tests.sub_type, 'None')) as alert_class_id, + {{ dbt.concat(["coalesce(failed_tests.test_unique_id, 'None')", "'.'", "coalesce(failed_tests.column_name, 'None')", "'.'", "coalesce(failed_tests.sub_type, 'None')"]) }} as alert_class_id, case when failed_tests.test_type = 'schema_change' then failed_tests.test_unique_id {# In old versions of elementary, elementary_test_results doesn't contain test_short_name, so we use dbt_test short_name. #} when tests.short_name = 'dimension_anomalies' then failed_tests.test_unique_id - else concat(coalesce(failed_tests.test_unique_id, 'None'), '.', coalesce(failed_tests.column_name, 'None'), '.', coalesce(failed_tests.sub_type, 'None')) + else {{ dbt.concat(["coalesce(failed_tests.test_unique_id, 'None')", "'.'", "coalesce(failed_tests.column_name, 'None')", "'.'", "coalesce(failed_tests.sub_type, 'None')"]) }} end as elementary_unique_id, failed_tests.data_issue_id, failed_tests.test_execution_id, diff --git a/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql b/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql index b0b7257bf..1ad1ccb00 100644 --- a/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql +++ b/elementary/monitor/dbt_project/macros/base_queries/current_tests_run_results_query.sql @@ -49,7 +49,7 @@ when elementary_test_results.test_type = 'schema_change' then elementary_test_results.test_unique_id {# In old versions of elementary, elementary_test_results doesn't contain test_short_name, so we use dbt_test short_name. #} when dbt_tests.short_name = 'dimension_anomalies' then elementary_test_results.test_unique_id - else concat(coalesce(elementary_test_results.test_unique_id, 'None'), '.', coalesce(nullif(elementary_test_results.column_name, ''), 'None'), '.', coalesce(elementary_test_results.test_sub_type, 'None')) + else {{ dbt.concat(["coalesce(elementary_test_results.test_unique_id, 'None')", "'.'", "coalesce(nullif(elementary_test_results.column_name, ''), 'None')", "'.'", "coalesce(elementary_test_results.test_sub_type, 'None')"]) }} end as elementary_unique_id, elementary_test_results.invocation_id, elementary_test_results.data_issue_id, diff --git a/elementary/monitor/dbt_project/macros/get_test_results.sql b/elementary/monitor/dbt_project/macros/get_test_results.sql index 5629dcc61..b1669823a 100644 --- a/elementary/monitor/dbt_project/macros/get_test_results.sql +++ b/elementary/monitor/dbt_project/macros/get_test_results.sql @@ -288,7 +288,7 @@ CASE WHEN etr.test_type = 'schema_change' THEN etr.test_unique_id WHEN dt.short_name = 'dimension_anomalies' THEN etr.test_unique_id - ELSE concat(coalesce(etr.test_unique_id, 'None'), '.', coalesce(nullif(etr.column_name, ''), 'None'), '.', coalesce(etr.test_sub_type, 'None')) + ELSE {{ dbt.concat(["coalesce(etr.test_unique_id, 'None')", "'.'", "coalesce(nullif(etr.column_name, ''), 'None')", "'.'", "coalesce(etr.test_sub_type, 'None')"]) }} END AS elementary_unique_id, etr.detected_at, etr.database_name, From d566f902fdffad1ea86baf6054b626d4d45dadfc Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 20:47:19 +0000 Subject: [PATCH 17/21] refactor: add edr_quote_identifier macro to eliminate fabric__ duplicates Replaces the adapter dispatch + fabric__ duplication pattern with a single edr_quote_identifier('schema') helper that returns [schema] on T-SQL and schema on all other adapters. Applied to get_models_runs, get_model_resources, and get_source_resources. Co-Authored-By: Itamar Hartstein --- .../macros/base_queries/resources.sql | 67 ++----------------- .../dbt_project/macros/get_models_runs.sql | 43 +----------- .../macros/utils/edr_quote_identifier.sql | 7 ++ 3 files changed, 12 insertions(+), 105 deletions(-) create mode 100644 elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql diff --git a/elementary/monitor/dbt_project/macros/base_queries/resources.sql b/elementary/monitor/dbt_project/macros/base_queries/resources.sql index 894f7762c..b95c58bd8 100644 --- a/elementary/monitor/dbt_project/macros/base_queries/resources.sql +++ b/elementary/monitor/dbt_project/macros/base_queries/resources.sql @@ -1,33 +1,4 @@ {% macro get_model_resources(exclude_elementary=true) %} - {{ return(adapter.dispatch('get_model_resources', 'elementary_cli')(exclude_elementary)) }} -{% endmacro %} - - -{% macro default__get_model_resources(exclude_elementary=true) %} - {% set model_resources_query %} - with dbt_models as ( - select * from {{ ref('elementary', 'dbt_models') }} - ) - - select - unique_id, - name, - schema_name as schema, - tags, - owner as owners, - database_name as database - from dbt_models - {% if exclude_elementary %} - where package_name != 'elementary' - {% endif %} - {% endset %} - {% set models_agate = run_query(model_resources_query) %} - {% do return(elementary.agate_to_dicts(models_agate)) %} -{% endmacro %} - - -{% macro fabric__get_model_resources(exclude_elementary=true) %} - {# T-SQL: 'schema' and 'database' are reserved keywords — quote with square brackets #} {% set model_resources_query %} with dbt_models as ( select * from {{ ref('elementary', 'dbt_models') }} @@ -36,10 +7,10 @@ select unique_id, name, - schema_name as [schema], + schema_name as {{ elementary_cli.edr_quote_identifier('schema') }}, tags, owner as owners, - database_name as [database] + database_name as {{ elementary_cli.edr_quote_identifier('database') }} from dbt_models {% if exclude_elementary %} where package_name != 'elementary' @@ -51,36 +22,6 @@ {% macro get_source_resources(exclude_elementary=true) %} - {{ return(adapter.dispatch('get_source_resources', 'elementary_cli')(exclude_elementary)) }} -{% endmacro %} - - -{% macro default__get_source_resources(exclude_elementary=true) %} - {% set source_resources_query %} - with dbt_sources as ( - select * from {{ ref('elementary', 'dbt_sources') }} - ) - - select - unique_id, - name, - source_name, - schema_name AS schema, - tags, - owner AS owners, - database_name as database - from dbt_sources - {% if exclude_elementary %} - where package_name != 'elementary' - {% endif %} - {% endset %} - {% set sources_agate = run_query(source_resources_query) %} - {% do return(elementary.agate_to_dicts(sources_agate)) %} -{% endmacro %} - - -{% macro fabric__get_source_resources(exclude_elementary=true) %} - {# T-SQL: 'schema' and 'database' are reserved keywords — quote with square brackets #} {% set source_resources_query %} with dbt_sources as ( select * from {{ ref('elementary', 'dbt_sources') }} @@ -90,10 +31,10 @@ unique_id, name, source_name, - schema_name AS [schema], + schema_name AS {{ elementary_cli.edr_quote_identifier('schema') }}, tags, owner AS owners, - database_name as [database] + database_name as {{ elementary_cli.edr_quote_identifier('database') }} from dbt_sources {% if exclude_elementary %} where package_name != 'elementary' diff --git a/elementary/monitor/dbt_project/macros/get_models_runs.sql b/elementary/monitor/dbt_project/macros/get_models_runs.sql index 6a9023670..f17aedd27 100644 --- a/elementary/monitor/dbt_project/macros/get_models_runs.sql +++ b/elementary/monitor/dbt_project/macros/get_models_runs.sql @@ -1,45 +1,4 @@ {%- macro get_models_runs(days_back = 7, exclude_elementary=false) -%} - {{ return(adapter.dispatch('get_models_runs', 'elementary_cli')(days_back, exclude_elementary)) }} -{%- endmacro -%} - - -{%- macro default__get_models_runs(days_back = 7, exclude_elementary=false) -%} - {% set models_runs_query %} - with model_runs as ( - select - *, - row_number() over (partition by unique_id order by generated_at desc) as invocations_rank_index - from {{ ref('elementary', 'model_run_results') }} - ) - - select - unique_id, - invocation_id, - name, - schema_name as schema, - status, - case - when status != 'success' then 0 - else round({{ elementary.edr_cast_as_numeric('execution_time') }}, 1) - end as execution_time, - full_refresh, - materialization, - case when invocations_rank_index = 1 then compiled_code else NULL end as compiled_code, - generated_at - from model_runs - where {{ elementary.edr_datediff(elementary.edr_cast_as_timestamp('generated_at'), elementary.edr_current_timestamp(), 'day') }} < {{ days_back }} - {% if exclude_elementary %} - and unique_id not like 'model.elementary.%' - {% endif %} - order by generated_at - {% endset %} - {% set models_runs_agate = run_query(models_runs_query) %} - {% do return(elementary.agate_to_dicts(models_runs_agate)) %} -{%- endmacro -%} - - -{%- macro fabric__get_models_runs(days_back = 7, exclude_elementary=false) -%} - {# T-SQL: 'schema' and 'database' are reserved keywords — quote with square brackets #} {% set models_runs_query %} with model_runs as ( select @@ -52,7 +11,7 @@ unique_id, invocation_id, name, - schema_name as [schema], + schema_name as {{ elementary_cli.edr_quote_identifier('schema') }}, status, case when status != 'success' then 0 diff --git a/elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql b/elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql new file mode 100644 index 000000000..0a6511f3f --- /dev/null +++ b/elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql @@ -0,0 +1,7 @@ +{%- macro edr_quote_identifier(identifier) -%} + {%- if target.type in ('fabric', 'sqlserver') -%} + [{{ identifier }}] + {%- else -%} + {{ identifier }} + {%- endif -%} +{%- endmacro -%} From 40c8044d0b7902d28f5a1b869cb41f1f0490e0fb Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 7 Mar 2026 23:10:00 +0000 Subject: [PATCH 18/21] refactor: use elementary.is_tsql() instead of target.type checks Replace all 'target.type in (fabric, sqlserver)' conditions with elementary.is_tsql() macro from dbt-data-reliability. Co-Authored-By: Itamar Hartstein --- .../monitor/dbt_project/macros/get_latest_invocation.sql | 2 +- .../monitor/dbt_project/macros/get_test_last_invocation.sql | 4 ++-- elementary/monitor/dbt_project/macros/test_conn.sql | 2 +- .../monitor/dbt_project/macros/utils/edr_quote_identifier.sql | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/get_latest_invocation.sql b/elementary/monitor/dbt_project/macros/get_latest_invocation.sql index c2f51aedf..f551182f7 100644 --- a/elementary/monitor/dbt_project/macros/get_latest_invocation.sql +++ b/elementary/monitor/dbt_project/macros/get_latest_invocation.sql @@ -7,7 +7,7 @@ {% endif %} {% set get_pkg_version_query %} - {% if target.type in ('fabric', 'sqlserver') %} + {% if elementary.is_tsql() %} select top 1 * from {{ invocations_relation }} order by generated_at desc {% else %} select * from {{ invocations_relation }} order by generated_at desc limit 1 diff --git a/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql b/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql index e09dcc780..c7e664c3b 100644 --- a/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql +++ b/elementary/monitor/dbt_project/macros/get_test_last_invocation.sql @@ -8,7 +8,7 @@ ), test_invocation as ( - select {% if target.type in ('fabric', 'sqlserver') %}top 1{% endif %} distinct invocation_id, detected_at + select {% if elementary.is_tsql() %}top 1{% endif %} distinct invocation_id, detected_at from elementary_test_results {% if invocation_id %} where invocation_id = {{ "'" ~ invocation_id ~ "'" }} @@ -16,7 +16,7 @@ where detected_at < {{ "'" ~ invocation_max_time ~ "'" }} {% endif %} order by detected_at desc - {% if target.type not in ('fabric', 'sqlserver') %}limit 1{% endif %} + {% if not elementary.is_tsql() %}limit 1{% endif %} ) {% if invocations_relation %} diff --git a/elementary/monitor/dbt_project/macros/test_conn.sql b/elementary/monitor/dbt_project/macros/test_conn.sql index 35bb8bbd0..ad3b52d38 100644 --- a/elementary/monitor/dbt_project/macros/test_conn.sql +++ b/elementary/monitor/dbt_project/macros/test_conn.sql @@ -6,7 +6,7 @@ {% set elementary_database, elementary_schema = elementary.get_package_database_and_schema() %} {% set elementary_model_relation = api.Relation.create(elementary_database, elementary_schema, "dbt_models") %} {% set query %} - {% if target.type in ('fabric', 'sqlserver') %} + {% if elementary.is_tsql() %} select top 10 * from {{ elementary_model_relation }} {% else %} select * from {{ elementary_model_relation }} limit 10 diff --git a/elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql b/elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql index 0a6511f3f..c0cef95f0 100644 --- a/elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql +++ b/elementary/monitor/dbt_project/macros/utils/edr_quote_identifier.sql @@ -1,5 +1,5 @@ {%- macro edr_quote_identifier(identifier) -%} - {%- if target.type in ('fabric', 'sqlserver') -%} + {%- if elementary.is_tsql() -%} [{{ identifier }}] {%- else -%} {{ identifier }} From 39a3c8d4a0831f8f375c0a59c0fd429767ed065d Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 8 Mar 2026 11:51:47 +0000 Subject: [PATCH 19/21] refactor: address PR review feedback - extract _TSQL_TRANSIENT, remove redundant sqlserver dispatch, deduplicate get_test_results processing Co-Authored-By: Itamar Hartstein --- elementary/clients/dbt/transient_errors.py | 27 ++- .../macros/get_adapter_type_and_unique_id.sql | 4 - .../dbt_project/macros/get_test_results.sql | 176 +++++------------- 3 files changed, 57 insertions(+), 150 deletions(-) diff --git a/elementary/clients/dbt/transient_errors.py b/elementary/clients/dbt/transient_errors.py index fb5b371d8..5f6cedaa2 100644 --- a/elementary/clients/dbt/transient_errors.py +++ b/elementary/clients/dbt/transient_errors.py @@ -45,6 +45,15 @@ "service unavailable", ) +_TSQL_TRANSIENT: Tuple[str, ...] = ( + "connection timed out", + "could not connect to the server", + "ssl syscall error", + "communication link failure", + "tcp provider: an existing connection was forcibly closed", + "login timeout expired", +) + _ADAPTER_PATTERNS: Dict[str, Tuple[str, ...]] = { "bigquery": ( # Streaming-buffer delay after a streaming insert. @@ -109,22 +118,8 @@ # DuckDB runs in-process; transient errors are rare. # Common patterns (connection reset, broken pipe) are in _COMMON. ), - "fabric": ( - "connection timed out", - "could not connect to the server", - "ssl syscall error", - "communication link failure", - "tcp provider: an existing connection was forcibly closed", - "login timeout expired", - ), - "sqlserver": ( - "connection timed out", - "could not connect to the server", - "ssl syscall error", - "communication link failure", - "tcp provider: an existing connection was forcibly closed", - "login timeout expired", - ), + "fabric": _TSQL_TRANSIENT, + "sqlserver": _TSQL_TRANSIENT, } # Pre-computed union of all adapter-specific patterns for the fallback path diff --git a/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql b/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql index 30000b440..3a9a3c186 100644 --- a/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql +++ b/elementary/monitor/dbt_project/macros/get_adapter_type_and_unique_id.sql @@ -25,7 +25,3 @@ {% macro fabric__get_adapter_unique_id() %} {{ return(target.server) }} {% endmacro %} - -{% macro sqlserver__get_adapter_unique_id() %} - {{ return(target.server) }} -{% endmacro %} diff --git a/elementary/monitor/dbt_project/macros/get_test_results.sql b/elementary/monitor/dbt_project/macros/get_test_results.sql index b1669823a..75b21d31b 100644 --- a/elementary/monitor/dbt_project/macros/get_test_results.sql +++ b/elementary/monitor/dbt_project/macros/get_test_results.sql @@ -2,6 +2,44 @@ {{ return(adapter.dispatch('get_test_results', 'elementary_cli')(days_back, invocations_per_test, disable_passed_test_metrics)) }} {%- endmacro -%} +{# + Shared post-processing helper: filters tests by meta, attaches sample data. + Called by both default__ and fabric__ dispatches to avoid duplicating the + Jinja processing loop. +#} +{%- macro _process_raw_test_results(test_results_agate, test_result_rows_agate, elementary_tests_allowlist_status) -%} + {% set test_results = [] %} + {% set tests = elementary.agate_to_dicts(test_results_agate) %} + + {% set filtered_tests = [] %} + {% for test in tests %} + {% set test_meta = fromjson(test.meta) %} + {% if test_meta.get("elementary", {}).get("include", true) %} + {% do filtered_tests.append(test) %} + {% endif %} + {% endfor %} + + {% for test in filtered_tests %} + {% set test_rows_sample = none %} + {% if test.invocations_rank_index == 1 %} + {% set test_type = test.test_type %} + {% set test_params = fromjson(test.test_params) %} + {% set status = test.status | lower %} + + {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} + {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} + {%- endif -%} + {% else %} + {# Null out test_results_query for non-latest invocations to save memory #} + {% do test.update({"test_results_query": none}) %} + {% endif %} + {# Adding sample data to test results #} + {% do test.update({"sample_data": test_rows_sample}) %} + {% do test_results.append(test) %} + {%- endfor -%} + {% do return(test_results) %} +{%- endmacro -%} + {%- macro default__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} {% set elementary_tests_allowlist_status = ['fail', 'warn'] if disable_passed_test_metrics else ['fail', 'warn', 'pass'] %} {% set select_test_results %} @@ -59,8 +97,6 @@ order by test_results.elementary_unique_id, test_results.invocations_rank_index desc {%- endset -%} - {% set test_results = [] %} - {% set elementary_database, elementary_schema = elementary.get_package_database_and_schema() %} {% set ordered_test_results_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'ordered_test_results', select_test_results) %} @@ -79,32 +115,8 @@ {% if not elementary.has_temp_table_support() %} {% do elementary.fully_drop_relation(ordered_test_results_relation) %} {% endif %} - {% set tests = elementary.agate_to_dicts(test_results_agate) %} - {% set filtered_tests = [] %} - {% for test in tests %} - {% set test_meta = fromjson(test.meta) %} - {% if test_meta.get("elementary", {}).get("include", true) %} - {% do filtered_tests.append(test) %} - {% endif %} - {% endfor %} - - {% for test in filtered_tests %} - {% set test_rows_sample = none %} - {% if test.invocations_rank_index == 1 %} - {% set test_type = test.test_type %} - {% set test_params = fromjson(test.test_params) %} - {% set status = test.status | lower %} - - {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} - {%- endif -%} - {% endif %} - {# Adding sample data to test results #} - {% do test.update({"sample_data": test_rows_sample}) %} - {% do test_results.append(test) %} - {%- endfor -%} - {% do return(test_results) %} + {% do return(elementary_cli._process_raw_test_results(test_results_agate, test_result_rows_agate, elementary_tests_allowlist_status)) %} {%- endmacro -%} {%- macro fabric__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} @@ -137,61 +149,18 @@ {% set ordered_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'ordered_test_results', select_ordered) %} - {# Step 3 – final select with column pruning and filtering #} - {% set select_test_results %} - select - test_results.id, - test_results.invocation_id, - test_results.test_execution_id, - test_results.model_unique_id, - test_results.test_unique_id, - test_results.elementary_unique_id, - {{elementary.edr_cast_as_timestamp('test_results.detected_at')}} as detected_at, - test_results.database_name, - test_results.schema_name, - test_results.table_name, - test_results.column_name, - test_results.test_type, - test_results.test_sub_type, - test_results.test_results_description, - test_results.test_description, - test_results.original_path, - test_results.package_name, - test_results.owners, - test_results.model_owner, - test_results.tags, - test_results.test_tags, - test_results.model_tags, - test_results.meta, - test_results.model_meta, - case when test_results.invocations_rank_index = 1 then test_results.test_results_query else NULL end as test_results_query, - test_results.other, - test_results.test_name, - test_results.test_params, - test_results.severity, - test_results.status, - test_results.execution_time, - test_results.days_diff, - test_results.invocations_rank_index, - test_results.failures, - test_results.result_rows - from {{ ordered_relation }} as test_results - where test_results.invocations_rank_index <= {{ invocations_per_test }} - {%- endset -%} - - {% set final_relation = elementary.create_temp_table(elementary_database, elementary_schema, 'final_test_results', select_test_results) %} - - {% set test_results = [] %} - + {# Step 3 – final query: filter by invocations_per_test #} {# ORDER BY must be here, not inside create_temp_table — T-SQL forbids ORDER BY in views/subqueries without TOP #} {% set test_results_agate_sql %} - select * from {{ final_relation }} + select * + from {{ ordered_relation }} + where invocations_rank_index <= {{ invocations_per_test }} order by elementary_unique_id, invocations_rank_index desc {% endset %} {% set valid_ids_query %} select distinct id - from {{ final_relation }} + from {{ ordered_relation }} where invocations_rank_index = 1 {% endset %} @@ -201,36 +170,8 @@ {# Clean up intermediate tables #} {% do elementary.fully_drop_relation(base_relation) %} {% do elementary.fully_drop_relation(ordered_relation) %} - {% if not elementary.has_temp_table_support() %} - {% do elementary.fully_drop_relation(final_relation) %} - {% endif %} - - {% set tests = elementary.agate_to_dicts(test_results_agate) %} - - {% set filtered_tests = [] %} - {% for test in tests %} - {% set test_meta = fromjson(test.meta) %} - {% if test_meta.get("elementary", {}).get("include", true) %} - {% do filtered_tests.append(test) %} - {% endif %} - {% endfor %} - - {% for test in filtered_tests %} - {% set test_rows_sample = none %} - {% if test.invocations_rank_index == 1 %} - {% set test_type = test.test_type %} - {% set test_params = fromjson(test.test_params) %} - {% set status = test.status | lower %} - {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} - {%- endif -%} - {% endif %} - {# Adding sample data to test results #} - {% do test.update({"sample_data": test_rows_sample}) %} - {% do test_results.append(test) %} - {%- endfor -%} - {% do return(test_results) %} + {% do return(elementary_cli._process_raw_test_results(test_results_agate, test_result_rows_agate, elementary_tests_allowlist_status)) %} {%- endmacro -%} {%- macro clickhouse__get_test_results(days_back = 7, invocations_per_test = 720, disable_passed_test_metrics = false) -%} @@ -365,31 +306,6 @@ {% if not elementary.has_temp_table_support() %} {% do elementary.fully_drop_relation(ordered_test_results_relation) %} {% endif %} - {% set tests = elementary.agate_to_dicts(test_results_agate) %} - - {% set filtered_tests = [] %} - {% for test in tests %} - {% set test_meta = fromjson(test.meta) %} - {% if test_meta.get("elementary", {}).get("include", true) %} - {% do filtered_tests.append(test) %} - {% endif %} - {% endfor %} - - {% for test in filtered_tests %} - {% set test_rows_sample = none %} - {% if test.invocations_rank_index == 1 %} - {% set test_type = test.test_type %} - {% set test_params = fromjson(test.test_params) %} - {% set status = test.status | lower %} - {%- if (test_type == 'dbt_test' and status in ['fail', 'warn']) or (test_type != 'dbt_test' and status in elementary_tests_allowlist_status) -%} - {% set test_rows_sample = elementary_cli.get_test_rows_sample(test.result_rows, test_result_rows_agate.get(test.id)) %} - {%- endif -%} - {% endif %} - {# Adding sample data to test results #} - {% do test.update({"sample_data": test_rows_sample}) %} - {% do test_results.append(test) %} - {%- endfor -%} - - {% do return(test_results) %} + {% do return(elementary_cli._process_raw_test_results(test_results_agate, test_result_rows_agate, elementary_tests_allowlist_status)) %} {%- endmacro -%} From 54614537302a0d5e2b339c2c6450649bc33aff90 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 8 Mar 2026 20:07:31 +0000 Subject: [PATCH 20/21] chore: update dbt-data-reliability to merged master commit, remove CI branch pin Co-Authored-By: Itamar Hartstein --- .github/workflows/test-all-warehouses.yml | 2 +- elementary/monitor/dbt_project/package-lock.yml | 4 ++-- elementary/monitor/dbt_project/packages.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-all-warehouses.yml b/.github/workflows/test-all-warehouses.yml index 12e8eac46..de6ba8f80 100644 --- a/.github/workflows/test-all-warehouses.yml +++ b/.github/workflows/test-all-warehouses.yml @@ -107,7 +107,7 @@ jobs: with: warehouse-type: ${{ matrix.warehouse-type }} elementary-ref: ${{ inputs.elementary-ref || ((github.event_name == 'pull_request_target' || github.event_name == 'pull_request') && github.event.pull_request.head.sha) || '' }} - dbt-data-reliability-ref: ${{ inputs.dbt-data-reliability-ref || 'devin/ELE-5282-1772640713' }} + dbt-data-reliability-ref: ${{ inputs.dbt-data-reliability-ref }} dbt-version: ${{ matrix.dbt-version }} generate-data: ${{ inputs.generate-data || false }} secrets: inherit diff --git a/elementary/monitor/dbt_project/package-lock.yml b/elementary/monitor/dbt_project/package-lock.yml index df46d6613..070158da0 100644 --- a/elementary/monitor/dbt_project/package-lock.yml +++ b/elementary/monitor/dbt_project/package-lock.yml @@ -4,5 +4,5 @@ packages: version: 0.8.6 - git: https://github.com/elementary-data/dbt-data-reliability.git name: elementary - revision: 72c3f8e9d35bb71bc03e66046de360680156a293 -sha1_hash: a84c14a4032dd29a0a679e48976d600a25582a86 + revision: 534afc63c75d28b87d7cbd3b222dd3ea9a980f7b +sha1_hash: cb18b7df65415901187dcf469dcd377e56c0dc70 diff --git a/elementary/monitor/dbt_project/packages.yml b/elementary/monitor/dbt_project/packages.yml index 803d3f314..3a77430a4 100644 --- a/elementary/monitor/dbt_project/packages.yml +++ b/elementary/monitor/dbt_project/packages.yml @@ -2,7 +2,7 @@ packages: - package: dbt-labs/dbt_utils version: [">=0.8.0", "<0.9.0"] - git: https://github.com/elementary-data/dbt-data-reliability.git - revision: devin/ELE-5282-1772640713 + revision: 534afc63c75d28b87d7cbd3b222dd3ea9a980f7b # NOTE - for unreleased CLI versions we often need to update the package version to a commit hash (please leave this # commented, so it will be easy to access) From 69898366b44704aece1ad56339da36e1fe3e54b5 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sun, 8 Mar 2026 20:22:17 +0000 Subject: [PATCH 21/21] ci: use docker compose --wait for sqlserver healthcheck instead of manual polling Co-Authored-By: Itamar Hartstein --- .github/workflows/test-warehouse.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test-warehouse.yml b/.github/workflows/test-warehouse.yml index 8a0c526bd..83fad4337 100644 --- a/.github/workflows/test-warehouse.yml +++ b/.github/workflows/test-warehouse.yml @@ -165,8 +165,7 @@ jobs: if: inputs.warehouse-type == 'sqlserver' working-directory: ${{ env.E2E_DBT_PROJECT_DIR }} run: | - docker compose up -d sqlserver - timeout 120 bash -c 'until docker exec sqlserver /opt/mssql-tools18/bin/sqlcmd -S localhost -U sa -P "${MSSQL_SA_PASSWORD:-Elementary123!}" -C -Q "SELECT 1" -b 2>/dev/null; do sleep 5; done' + docker compose up -d --wait sqlserver - name: Setup Python uses: actions/setup-python@v5