diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index bdbb2dfd40..22aef9f5a6 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,6 +9,7 @@ ### Dependency updates ### Bundles +* Prompt for serverless compute in `dbt-sql` template (defaults to `yes`) ([#3668](https://github.com/databricks/cli/pull/3668)) ### API Changes * Added `databricks account account-groups-v2` command group. diff --git a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml index 0211029c21..ea834cd9de 100644 --- a/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml +++ b/acceptance/bundle/templates/dbt-sql/output/my_dbt_sql/resources/my_dbt_sql.job.yml @@ -15,6 +15,7 @@ resources: tasks: - task_key: dbt + environment_key: default dbt_task: project_directory: ../ # The default schema, catalog, etc. are defined in ../dbt_profiles/profiles.yml @@ -25,17 +26,9 @@ resources: - 'dbt seed --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' - 'dbt run --target=${bundle.target} --vars "{ dev_schema: ${workspace.current_user.short_name} }"' - libraries: - - pypi: - package: dbt-databricks>=1.8.0,<2.0.0 - - new_cluster: - spark_version: 15.4.x-scala2.12 - node_type_id: [NODE_TYPE_ID] - data_security_mode: SINGLE_USER - num_workers: 0 - spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode - custom_tags: - ResourceClass: SingleNode + environments: + - environment_key: default + spec: + environment_version: "2" + dependencies: + - dbt-databricks>=1.8.0,<2.0.0 diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/out.requests.txt b/acceptance/bundle/templates/telemetry/dbt-sql/out.requests.txt index b7616ccd3a..74dcc3bb40 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/out.requests.txt +++ b/acceptance/bundle/templates/telemetry/dbt-sql/out.requests.txt @@ -28,7 +28,7 @@ "uploadTime": [UNIX_TIME_MILLIS], "items": [], "protoLogs": [ - "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[CMD-EXEC-ID]\",\"version\":\"[DEV_VERSION]\",\"command\":\"bundle_init\",\"operating_system\":\"[OS]\",\"execution_time_ms\":\"SMALL_INT\",\"exit_code\":0},\"bundle_init_event\":{\"bundle_uuid\":\"[BUNDLE-UUID]\",\"template_name\":\"dbt-sql\",\"template_enum_args\":[{\"key\":\"personal_schemas\",\"value\":\"yes, use a schema based on the current user name during development\"}]}}}}" + "{\"frontend_log_event_id\":\"[UUID]\",\"entry\":{\"databricks_cli_log\":{\"execution_context\":{\"cmd_exec_id\":\"[CMD-EXEC-ID]\",\"version\":\"[DEV_VERSION]\",\"command\":\"bundle_init\",\"operating_system\":\"[OS]\",\"execution_time_ms\":\"SMALL_INT\",\"exit_code\":0},\"bundle_init_event\":{\"bundle_uuid\":\"[BUNDLE-UUID]\",\"template_name\":\"dbt-sql\",\"template_enum_args\":[{\"key\":\"personal_schemas\",\"value\":\"yes, use a schema based on the current user name during development\"},{\"key\":\"serverless\",\"value\":\"yes\"}]}}}}" ] } } diff --git a/acceptance/bundle/templates/telemetry/dbt-sql/output.txt b/acceptance/bundle/templates/telemetry/dbt-sql/output.txt index e1a5fa06ad..2455fe4442 100644 --- a/acceptance/bundle/templates/telemetry/dbt-sql/output.txt +++ b/acceptance/bundle/templates/telemetry/dbt-sql/output.txt @@ -29,6 +29,10 @@ Refer to the README.md file for full "getting started" guide and production setu { "key": "personal_schemas", "value": "yes, use a schema based on the current user name during development" + }, + { + "key": "serverless", + "value": "yes" } ] } diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index fe7c09d2bd..81dde00c62 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -135,15 +135,18 @@ func TestBuiltinDbtTemplateValid(t *testing.T) { for _, personal_schemas := range []string{"yes", "no"} { for _, target := range []string{"dev", "prod"} { for _, isServicePrincipal := range []bool{true, false} { - config := map[string]any{ - "project_name": "my_project", - "http_path": "/sql/1.0/warehouses/123", - "default_catalog": "hive_metastore", - "personal_schemas": personal_schemas, - "shared_schema": "lennart", + for _, serverless := range []string{"yes", "no"} { + config := map[string]any{ + "project_name": "my_project", + "http_path": "/sql/1.0/warehouses/123", + "default_catalog": "hive_metastore", + "personal_schemas": personal_schemas, + "shared_schema": "lennart", + "serverless": serverless, + } + build := false + assertBuiltinTemplateValid(t, "dbt-sql", config, target, isServicePrincipal, build, t.TempDir()) } - build := false - assertBuiltinTemplateValid(t, "dbt-sql", config, target, isServicePrincipal, build, t.TempDir()) } } } diff --git a/libs/template/templates/dbt-sql/databricks_template_schema.json b/libs/template/templates/dbt-sql/databricks_template_schema.json index cccf145dc5..962913be06 100644 --- a/libs/template/templates/dbt-sql/databricks_template_schema.json +++ b/libs/template/templates/dbt-sql/databricks_template_schema.json @@ -47,6 +47,13 @@ "pattern_match_failure_message": "Invalid schema name.", "description": "\nPlease provide an initial schema during development.\ndefault_schema", "order": 5 + }, + "serverless": { + "type": "string", + "default": "yes", + "enum": ["yes", "no"], + "description": "Use serverless compute", + "order": 6 } }, "success_message": "\nšŸ“Š Your new project has been created in the '{{.project_name}}' directory!\nIf you already have dbt installed, just type 'cd {{.project_name}}; dbt init' to get started.\nRefer to the README.md file for full \"getting started\" guide and production setup instructions.\n" diff --git a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index 5ca04580e9..7e2c48616c 100644 --- a/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/libs/template/templates/dbt-sql/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -1,3 +1,5 @@ +{{$with_serverless := (eq .serverless "yes") -}} + resources: jobs: {{.project_name}}_job: @@ -15,6 +17,9 @@ resources: tasks: - task_key: dbt + {{- if $with_serverless }} + environment_key: default + {{- end }} dbt_task: project_directory: ../ # The default schema, catalog, etc. are defined in ../dbt_profiles/profiles.yml @@ -32,6 +37,8 @@ resources: - 'dbt run --target=${bundle.target}' {{- end}} + {{- if not $with_serverless }} + libraries: - pypi: package: dbt-databricks>=1.8.0,<2.0.0 @@ -46,3 +53,14 @@ resources: spark.databricks.cluster.profile: singleNode custom_tags: ResourceClass: SingleNode + {{- end }} + + {{- if $with_serverless }} + + environments: + - environment_key: default + spec: + environment_version: "2" + dependencies: + - dbt-databricks>=1.8.0,<2.0.0 + {{- end }}