From 7e9a19c02aa87f156ab7b6b48abe54f1d4a586f5 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Fri, 21 Feb 2025 16:44:03 +0100 Subject: [PATCH 1/3] Add default cluster property `data_security_mode` set to "SINGLE_USER" in default python template --- .../resources/my_default_python.job.yml | 1 + .../testdata/default_python/bundle_summary.txt | 14 +++++--------- .../resources/{{.project_name}}.job.yml.tmpl | 1 + 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml index d9e31691a6..7c11e143f4 100644 --- a/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml +++ b/acceptance/bundle/templates/default-python/output/my_default_python/resources/my_default_python.job.yml @@ -44,6 +44,7 @@ resources: new_cluster: spark_version: 15.4.x-scala2.12 node_type_id: i3.xlarge + data_security_mode: SINGLE_USER autoscale: min_workers: 1 max_workers: 4 diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt index 450f01c467..b575c3b43d 100644 --- a/integration/bundle/testdata/default_python/bundle_summary.txt +++ b/integration/bundle/testdata/default_python/bundle_summary.txt @@ -33,10 +33,11 @@ "value": "[USERNAME]" } ], + "externalId": "[UUID]", "groups": [ { "$ref": "Groups/[USERGROUP]", - "display": "team.engineering", + "display": "admins", "type": "direct", "value": "[USERGROUP]" } @@ -67,11 +68,6 @@ "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json" }, "edit_mode": "UI_LOCKED", - "email_notifications": { - "on_failure": [ - "[USERNAME]" - ] - }, "format": "MULTI_TASK", "id": "[NUMID]", "job_clusters": [ @@ -82,7 +78,8 @@ "max_workers": 4, "min_workers": 1 }, - "node_type_id": "i3.xlarge", + "data_security_mode": "SINGLE_USER", + "node_type_id": "Standard_D3_v2", "spark_version": "15.4.x-scala2.12" } } @@ -145,7 +142,6 @@ }, "pipelines": { "project_name_$UNIQUE_PRJ_pipeline": { - "catalog": "main", "configuration": { "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src" }, @@ -164,7 +160,7 @@ ], "name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_pipeline", "target": "project_name_$UNIQUE_PRJ_dev", - "url": "[DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]" + "url": "[DATABRICKS_URL]/pipelines/[UUID]" } } }, diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index 5211e3894b..735a232be2 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -69,6 +69,7 @@ resources: new_cluster: spark_version: {{template "latest_lts_dbr_version"}} node_type_id: {{smallest_node_type}} + data_security_mode: SINGLE_USER autoscale: min_workers: 1 max_workers: 4 From ad2cc898dcde78716ece89955810701de9ca70e1 Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Wed, 26 Feb 2025 09:57:55 +0100 Subject: [PATCH 2/3] revert changes to expected bundle summary for default python template (keeping newly introduced data_security_mode field) --- .../testdata/default_python/bundle_summary.txt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/integration/bundle/testdata/default_python/bundle_summary.txt b/integration/bundle/testdata/default_python/bundle_summary.txt index b575c3b43d..9680097598 100644 --- a/integration/bundle/testdata/default_python/bundle_summary.txt +++ b/integration/bundle/testdata/default_python/bundle_summary.txt @@ -33,11 +33,10 @@ "value": "[USERNAME]" } ], - "externalId": "[UUID]", "groups": [ { "$ref": "Groups/[USERGROUP]", - "display": "admins", + "display": "team.engineering", "type": "direct", "value": "[USERGROUP]" } @@ -68,6 +67,11 @@ "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/state/metadata.json" }, "edit_mode": "UI_LOCKED", + "email_notifications": { + "on_failure": [ + "[USERNAME]" + ] + }, "format": "MULTI_TASK", "id": "[NUMID]", "job_clusters": [ @@ -79,7 +83,7 @@ "min_workers": 1 }, "data_security_mode": "SINGLE_USER", - "node_type_id": "Standard_D3_v2", + "node_type_id": "i3.xlarge", "spark_version": "15.4.x-scala2.12" } } @@ -142,6 +146,7 @@ }, "pipelines": { "project_name_$UNIQUE_PRJ_pipeline": { + "catalog": "main", "configuration": { "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files/src" }, @@ -160,7 +165,7 @@ ], "name": "[dev [USERNAME]] project_name_$UNIQUE_PRJ_pipeline", "target": "project_name_$UNIQUE_PRJ_dev", - "url": "[DATABRICKS_URL]/pipelines/[UUID]" + "url": "[DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]" } } }, From b95949d3e5714599b0367e188228d23cf3ac472e Mon Sep 17 00:00:00 2001 From: Anton Nekipelov <226657+anton-107@users.noreply.github.com> Date: Wed, 26 Feb 2025 12:33:21 +0100 Subject: [PATCH 3/3] Add default cluster property data_security_mode set to "SINGLE_USER" in experimental-jobs-as-code template --- acceptance/bundle/templates/experimental-jobs-as-code/output.txt | 1 + .../output/my_jobs_as_code/resources/my_jobs_as_code_job.py | 1 + .../{{.project_name}}/resources/{{.project_name}}_job.py.tmpl | 1 + 3 files changed, 3 insertions(+) diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt index 984dad6041..2099dd4987 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt @@ -34,6 +34,7 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi "max_workers": 4, "min_workers": 1 }, + "data_security_mode": "SINGLE_USER", "node_type_id": "i3.xlarge", "spark_version": "15.4.x-scala2.12" } diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py index e8406fd7b5..be7254b808 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output/my_jobs_as_code/resources/my_jobs_as_code_job.py @@ -56,6 +56,7 @@ "new_cluster": { "spark_version": "15.4.x-scala2.12", "node_type_id": "i3.xlarge", + "data_security_mode": "SINGLE_USER", "autoscale": { "min_workers": 1, "max_workers": 4, diff --git a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl index 7c7a0d33f7..d9d2487999 100644 --- a/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl +++ b/libs/template/templates/experimental-jobs-as-code/template/{{.project_name}}/resources/{{.project_name}}_job.py.tmpl @@ -97,6 +97,7 @@ This job runs {{.project_name}}_pipeline on a schedule. "new_cluster": { "spark_version": "{{template "latest_lts_dbr_version"}}", "node_type_id": "{{smallest_node_type}}", + "data_security_mode": "SINGLE_USER", "autoscale": { "min_workers": 1, "max_workers": 4,