From 94ee12051c84fd1cc5a021c603a0eb3432fbb0bf Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Fri, 26 Sep 2025 15:48:13 +0200 Subject: [PATCH 1/5] Add serverless support to dbt-sql template This mirrors the approach in default-python. --- .../my_dbt_sql/resources/my_dbt_sql.job.yml | 22 ++++++++----------- .../telemetry/dbt-sql/out.requests.txt | 2 +- .../templates/telemetry/dbt-sql/output.txt | 4 ++++ .../dbt-sql/databricks_template_schema.json | 7 ++++++ .../resources/{{.project_name}}.job.yml.tmpl | 21 ++++++++++++++++++ 5 files changed, 42 insertions(+), 14 deletions(-) diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml index 0211029c21..a8f82a5845 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml @@ -15,6 +15,7 @@ resources: tasks: - task_key: dbt + environment_key: default dbt_task: project_directory: ../ # The default schema, catalog, etc. are defined in ../dbt_profiles/profiles.yml @@ -25,17 +26,12 @@ resources: - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' - libraries: - - pypi: - package: dbt-databricks>=1.8.0,<2.0.0 + environments: + - environment_key: default - new_cluster: - spark_version: 15.4.x-scala2.12 - node_type_id: [NODE_TYPE_ID] - data_security_mode: SINGLE_USER - num_workers: 0 - spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode - custom_tags: - ResourceClass: SingleNode + # Full documentation of this spec can be found at: + # https://docs.databricks.com/api/workspace/jobs/create#environments-spec + spec: + environment_version: "2" + dependencies: + - dbt-databricks>=1.8.0,<2.0.0 diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/out.requests.txt b/acceptance/bundle/templates/telemetry/dbt-sql/out.requests.txt index b7616ccd3a..74dcc3bb40 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/out.requests.txt +++ b/acceptance/bundle/templates/telemetry/dbt-sql/out.requests.txt @@ -28,7 +28,7 @@ "uploadTime": [UNIX_TIME_MILLIS], "items": [], "protoLogs": [ - "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[CMD-EXEC-ID]\",\"version\":\"[DEV_VERSION]\",\"command\":\"bundle_init\",\"operating_system\":\"[OS]\",\"execution_time_ms\":\"SMALL_INT\",\"exit_code\":0},\"bundle_init_event\":{\"bundle_uuid\":\"[BUNDLE-UUID]\",\"template_name\":\"dbt-sql\",\"template_enum_args\":[{\"key\":\"personal_schemas\",\"value\":\"yes, use a schema based on the current user name during development\"}]}}}}" + "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[CMD-EXEC-ID]\",\"version\":\"[DEV_VERSION]\",\"command\":\"bundle_init\",\"operating_system\":\"[OS]\",\"execution_time_ms\":\"SMALL_INT\",\"exit_code\":0},\"bundle_init_event\":{\"bundle_uuid\":\"[BUNDLE-UUID]\",\"template_name\":\"dbt-sql\",\"template_enum_args\":[{\"key\":\"personal_schemas\",\"value\":\"yes, use a schema based on the current user name during development\"},{\"key\":\"serverless\",\"value\":\"yes\"}]}}}}" ] } } diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/output.txt b/acceptance/bundle/templates/telemetry/dbt-sql/output.txt index e1a5fa06ad..2455fe4442 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/output.txt +++ b/acceptance/bundle/templates/telemetry/dbt-sql/output.txt @@ -29,6 +29,10 @@ Refer to the README.md file for full "getting started" guide and production setu { "key": "personal_schemas", "value": "yes, use a schema based on the current user name during development" + }, + { + "key": "serverless", + "value": "yes" } ] } diff --git a/libs/template/templates/dbt-sql/databricks_template_schema.json b/libs/template/templates/dbt-sql/databricks_template_schema.json index cccf145dc5..962913be06 100644 --- a/libs/template/templates/dbt-sql/databricks_template_schema.json +++ b/libs/template/templates/dbt-sql/databricks_template_schema.json @@ -47,6 +47,13 @@ "pattern_match_failure_message": "Invalid schema name.", "description": "\nPlease provide an initial schema during development.\ndefault_schema", "order": 5 + }, + "serverless": { + "type": "string", + "default": "yes", + "enum": ["yes", "no"], + "description": "Use serverless compute", + "order": 6 } }, "success_message": "\nšŸ“Š Your new project has been created in the '{{.project_name}}' directory!\nIf you already have dbt installed, just type 'cd {{.project_name}}; dbt init' to get started.\nRefer to the README.md file for full \"getting started\" guide and production setup instructions.\n" diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index 5ca04580e9..02474ddd69 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -1,3 +1,5 @@ +{{$with_serverless := (eq .serverless "yes") -}} + resources: jobs: {{.project_name}}_job: @@ -15,6 +17,9 @@ resources: tasks: - task_key: dbt + {{- if $with_serverless }} + environment_key: default + {{- end }} dbt_task: project_directory: ../ # The default schema, catalog, etc. are defined in ../dbt_profiles/profiles.yml @@ -32,6 +37,8 @@ resources: - 'dbt run --target=${bundle.target}' {{- end}} + {{- if not $with_serverless }} + libraries: - pypi: package: dbt-databricks>=1.8.0,<2.0.0 @@ -46,3 +53,17 @@ resources: spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode + {{- end }} + + {{- if $with_serverless }} + + environments: + - environment_key: default + + # Full documentation of this spec can be found at: + # https://docs.databricks.com/api/workspace/jobs/create#environments-spec + spec: + environment_version: "2" + dependencies: + - dbt-databricks>=1.8.0,<2.0.0 + {{- end }} From bdcfffee4bc7353e2c5e1af813c1816a7de96202 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 29 Sep 2025 09:42:00 +0200 Subject: [PATCH 2/5] Entry in CHANGELOG --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index bdbb2dfd40..970e3519f4 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,6 +9,7 @@ ### Dependency updates ### Bundles +* Prompt for serverless compute in `dbt-sql` templat (defaults to `yes`) ([#3668](https://github.com/databricks/cli/pull/3668)) ### API Changes * Added `databricks account account-groups-v2` command group. From a5ae5f2bf1267226d377f03742e71b296a9e87f5 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 29 Sep 2025 09:42:17 +0200 Subject: [PATCH 3/5] Fix unit test --- libs/template/renderer_test.go | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index fe7c09d2bd..81dde00c62 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -135,15 +135,18 @@ func TestBuiltinDbtTemplateValid(t *testing.T) { for _, personal_schemas := range []string{"yes", "no"} { for _, target := range []string{"dev", "prod"} { for _, isServicePrincipal := range []bool{true, false} { - config := map[string]any{ - "project_name": "my_project", - "http_path": "/sql/1.0/warehouses/123", - "default_catalog": "hive_metastore", - "personal_schemas": personal_schemas, - "shared_schema": "lennart", + for _, serverless := range []string{"yes", "no"} { + config := map[string]any{ + "project_name": "my_project", + "http_path": "/sql/1.0/warehouses/123", + "default_catalog": "hive_metastore", + "personal_schemas": personal_schemas, + "shared_schema": "lennart", + "serverless": serverless, + } + build := false + assertBuiltinTemplateValid(t, "dbt-sql", config, target, isServicePrincipal, build, t.TempDir()) } - build := false - assertBuiltinTemplateValid(t, "dbt-sql", config, target, isServicePrincipal, build, t.TempDir()) } } } From 7fe2ecfa446ec88dc54c8361bfc28583c1400029 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 29 Sep 2025 09:47:13 +0200 Subject: [PATCH 4/5] Remove comment --- .../dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml | 3 --- .../{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl | 3 --- 2 files changed, 6 deletions(-) diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml index a8f82a5845..ea834cd9de 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml @@ -28,9 +28,6 @@ resources: environments: - environment_key: default - - # Full documentation of this spec can be found at: - # https://docs.databricks.com/api/workspace/jobs/create#environments-spec spec: environment_version: "2" dependencies: diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index 02474ddd69..7e2c48616c 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -59,9 +59,6 @@ resources: environments: - environment_key: default - - # Full documentation of this spec can be found at: - # https://docs.databricks.com/api/workspace/jobs/create#environments-spec spec: environment_version: "2" dependencies: From d61320227997685683909b44487ee67d11c66e94 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Mon, 29 Sep 2025 00:52:18 -0700 Subject: [PATCH 5/5] Fix typo in dbt-sql template entry --- NEXT_CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 970e3519f4..22aef9f5a6 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,7 +9,7 @@ ### Dependency updates ### Bundles -* Prompt for serverless compute in `dbt-sql` templat (defaults to `yes`) ([#3668](https://github.com/databricks/cli/pull/3668)) +* Prompt for serverless compute in `dbt-sql` template (defaults to `yes`) ([#3668](https://github.com/databricks/cli/pull/3668)) ### API Changes * Added `databricks account account-groups-v2` command group.