From d78e92a2b30e09c242246cf67dace990859752cf Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Sun, 17 Mar 2024 10:36:36 +0100 Subject: [PATCH 1/3] Cleanup experimental SQL template --- .../{{.project_name}}/src/orders_daily.sql.tmpl | 12 ++++++------ .../{{.project_name}}/src/orders_raw.sql.tmpl | 6 ++++-- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl index 76ecadd3ea..a96098e0b2 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl @@ -1,14 +1,14 @@ -- This query is executed using Databricks Workflows (see resources/{{.project_name}}_sql_job.yml) -{{- /* We can't use a materialized view here since they don't support 'create or refresh yet.*/}} -CREATE OR REPLACE VIEW - IDENTIFIER(CONCAT({{"{{"}}catalog{{"}}"}}, '.', {{"{{"}}schema{{"}}"}}, '.', 'orders_daily')) +USE CATALOG {{"{{"}}catalog{{"}}"}}; +USE SCHEMA {{"{{"}}schema{{"}}"}}; + +CREATE OR REPLACE MATERIALIZED VIEW orders_daily AS SELECT order_date, count(*) AS number_of_orders -FROM - IDENTIFIER(CONCAT({{"{{"}}catalog{{"}}"}}, '.', {{"{{"}}schema{{"}}"}}, '.', 'orders_raw')) +FROM orders_raw -- During development, only process a smaller range of data -WHERE {{"{{"}}bundle_target{{"}}"}} == "prod" OR (order_date >= '2019-08-01' AND order_date < '2019-09-01') +WHERE if({{"{{"}}bundle_target{{"}}"}} == "prod", true, order_date >= '2019-08-01' AND order_date < '2019-09-01') GROUP BY order_date diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl index 96769062b5..77f0afe3d3 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_raw.sql.tmpl @@ -3,8 +3,10 @@ -- The streaming table below ingests all JSON files in /databricks-datasets/retail-org/sales_orders/ -- See also https://docs.databricks.com/sql/language-manual/sql-ref-syntax-ddl-create-streaming-table.html -CREATE OR REFRESH STREAMING TABLE - IDENTIFIER(CONCAT({{"{{"}}catalog{{"}}"}}, '.', {{"{{"}}schema{{"}}"}}, '.', 'orders_raw')) +USE CATALOG {{"{{"}}catalog{{"}}"}}; +USE SCHEMA {{"{{"}}schema{{"}}"}}; + +CREATE OR REFRESH STREAMING TABLE orders_raw AS SELECT customer_name, DATE(TIMESTAMP(FROM_UNIXTIME(TRY_CAST(order_datetime AS BIGINT)))) AS order_date, From 95fc745ab4c467f4e22dcc8d02bf49397c223dd0 Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Sun, 17 Mar 2024 10:36:53 +0100 Subject: [PATCH 2/3] Refine for non-UC workspaces --- .../template/{{.project_name}}/src/orders_daily.sql.tmpl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl index a96098e0b2..eb035c8f19 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl @@ -3,7 +3,14 @@ USE CATALOG {{"{{"}}catalog{{"}}"}}; USE SCHEMA {{"{{"}}schema{{"}}"}}; +{{- if eq (default_catalog) ""}} +{{- /* This workspace might not have Unity Catalog, */}} +{{- /* so let's not show both materialized views and streaming tables. */}} +{{- /* They're not supported without Unity Catalog! */}} +CREATE OR REPLACE VIEW orders_daily +{{- else}} CREATE OR REPLACE MATERIALIZED VIEW orders_daily +{{- end}} AS SELECT order_date, count(*) AS number_of_orders FROM orders_raw From 483e26ff0045e0d0add3eda5b1a67f13dc285dcd Mon Sep 17 00:00:00 2001 From: Lennart Kats Date: Wed, 20 Mar 2024 15:42:00 +0100 Subject: [PATCH 3/3] Revert "Refine for non-UC workspaces" This reverts commit 95fc745ab4c467f4e22dcc8d02bf49397c223dd0. --- .../template/{{.project_name}}/src/orders_daily.sql.tmpl | 7 ------- 1 file changed, 7 deletions(-) diff --git a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl index eb035c8f19..a96098e0b2 100644 --- a/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl +++ b/libs/template/templates/default-sql/template/{{.project_name}}/src/orders_daily.sql.tmpl @@ -3,14 +3,7 @@ USE CATALOG {{"{{"}}catalog{{"}}"}}; USE SCHEMA {{"{{"}}schema{{"}}"}}; -{{- if eq (default_catalog) ""}} -{{- /* This workspace might not have Unity Catalog, */}} -{{- /* so let's not show both materialized views and streaming tables. */}} -{{- /* They're not supported without Unity Catalog! */}} -CREATE OR REPLACE VIEW orders_daily -{{- else}} CREATE OR REPLACE MATERIALIZED VIEW orders_daily -{{- end}} AS SELECT order_date, count(*) AS number_of_orders FROM orders_raw