From 834b124df41805767fcb9d4995705388a58075e4 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 12 Feb 2025 20:44:01 +0100 Subject: [PATCH 1/4] Use schema field for pipeline in builtin template The presence of this field implies that the pipeline is in direct publishing mode. This is can be used only with Unity Catalog, so the template only configures it if it detects Unity Catalog. --- .../my_default_python/resources/my_default_python.pipeline.yml | 2 +- .../resources/{{.project_name}}.pipeline.yml.tmpl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml index f9e083f4fb..2b2bf85a15 100644 --- a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml +++ b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.pipeline.yml @@ -4,7 +4,7 @@ resources: my_default_python_pipeline: name: my_default_python_pipeline catalog: main - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} libraries: - notebook: path: ../src/dlt_pipeline.ipynb diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl index 50f11fe2cc..87af4737ba 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl @@ -6,10 +6,11 @@ resources: {{- if or (eq default_catalog "") (eq default_catalog "hive_metastore")}} ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: # catalog: catalog_name + target: {{.project_name}}_${bundle.target} {{- else}} catalog: {{default_catalog}} + schema: {{.project_name}}_${bundle.target} {{- end}} - target: {{.project_name}}_${bundle.target} libraries: - notebook: path: ../src/dlt_pipeline.ipynb From eeb70c77950f57e6d2111934aaaf55a28de49cf0 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 14 Feb 2025 08:56:40 +0100 Subject: [PATCH 2/4] Schema field works outside UC as well --- .../resources/{{.project_name}}.pipeline.yml.tmpl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl index 87af4737ba..ff938577fd 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.pipeline.yml.tmpl @@ -6,11 +6,10 @@ resources: {{- if or (eq default_catalog "") (eq default_catalog "hive_metastore")}} ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: # catalog: catalog_name - target: {{.project_name}}_${bundle.target} {{- else}} catalog: {{default_catalog}} - schema: {{.project_name}}_${bundle.target} {{- end}} + schema: {{.project_name}}_${bundle.target} libraries: - notebook: path: ../src/dlt_pipeline.ipynb From 1b991ce979430e2701c3037997e03ac0abd349ca Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 3 Mar 2025 11:51:55 +0100 Subject: [PATCH 3/4] Update acceptance tests --- .../default-python/classic/out.compare-vs-serverless.diff | 2 +- .../my_default_python/resources/my_default_python.pipeline.yml | 2 +- .../default-python/serverless-customcatalog/output.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff index 6890badf0b..8fdcb8359f 100644 --- a/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff +++ b/acceptance/bundle/templates/default-python/classic/out.compare-vs-serverless.diff @@ -48,7 +48,7 @@ - catalog: main + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: catalog_name - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} - serverless: true libraries: - notebook: diff --git a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml index 4176f765d3..d07d4fa6ce 100644 --- a/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml +++ b/acceptance/bundle/templates/default-python/classic/output/my_default_python/resources/my_default_python.pipeline.yml @@ -5,7 +5,7 @@ resources: name: my_default_python_pipeline ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: # catalog: catalog_name - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} libraries: - notebook: path: ../src/dlt_pipeline.ipynb diff --git a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt index 30726013bf..a6a92dfd4e 100644 --- a/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt +++ b/acceptance/bundle/templates/default-python/serverless-customcatalog/output.txt @@ -18,5 +18,5 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde - ## Catalog is required for serverless compute - catalog: main + catalog: customcatalog - target: my_default_python_${bundle.target} + schema: my_default_python_${bundle.target} serverless: true From da2ae95ab7b62dc7380dfa7825d57bcd91feb650 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 5 Mar 2025 14:54:58 +0100 Subject: [PATCH 4/4] Update integration test assertion --- integration/bundle/testdata/default_python/bundle_summary.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt index 9680097598..0b4cbb1814 100644 --- a/integration/bundle/testdata/default_python/bundle_summary.txt +++ b/integration/bundle/testdata/default_python/bundle_summary.txt @@ -164,7 +164,7 @@ } ], "name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_pipeline", - "target": "project_name_$UNIQUE_PRJ_dev", + "schema": "project_name_$UNIQUE_PRJ_dev", "url": "[DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]" } }