From 09dd6e84cca76327ec6ee9356f4ce8ed815a7300 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Tue, 29 Jul 2025 17:08:41 +0200 Subject: [PATCH 1/2] Add telemetry to track use og the legacy run_as option --- .../telemetry/deploy-compute-type/output.txt | 8 ++++ .../deploy-experimental/databricks.yml | 14 ++++++ .../deploy-experimental/out.test.toml | 5 +++ .../telemetry/deploy-experimental/output.txt | 44 +++++++++++++++++++ .../telemetry/deploy-experimental/script | 5 +++ .../deploy-name-prefix/custom/output.txt | 4 ++ .../mode-development/output.txt | 4 ++ .../bundle/telemetry/deploy/out.telemetry.txt | 4 ++ .../config/mutator/resourcemutator/run_as.go | 3 ++ 9 files changed, 91 insertions(+) create mode 100644 acceptance/bundle/telemetry/deploy-experimental/databricks.yml create mode 100644 acceptance/bundle/telemetry/deploy-experimental/out.test.toml create mode 100644 acceptance/bundle/telemetry/deploy-experimental/output.txt create mode 100644 acceptance/bundle/telemetry/deploy-experimental/script diff --git a/acceptance/bundle/telemetry/deploy-compute-type/output.txt b/acceptance/bundle/telemetry/deploy-compute-type/output.txt index bb95771e28..9cb418ea1d 100644 --- a/acceptance/bundle/telemetry/deploy-compute-type/output.txt +++ b/acceptance/bundle/telemetry/deploy-compute-type/output.txt @@ -13,6 +13,10 @@ Deployment complete! >>> cat out.requests.txt [ + { + "key": "experimental.use_legacy_run_as", + "value": false + }, { "key": "presets_name_prefix_is_set", "value": false @@ -39,6 +43,10 @@ Deployment complete! } ] [ + { + "key": "experimental.use_legacy_run_as", + "value": false + }, { "key": "presets_name_prefix_is_set", "value": false diff --git a/acceptance/bundle/telemetry/deploy-experimental/databricks.yml b/acceptance/bundle/telemetry/deploy-experimental/databricks.yml new file mode 100644 index 0000000000..80ed7a55db --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-experimental/databricks.yml @@ -0,0 +1,14 @@ +experimental: + use_legacy_run_as: true + +run_as: + user_name: abcd + +resources: + jobs: + foo: + tasks: + - task_key: task1 + spark_python_task: + python_file: /Workspace/test.py + existing_cluster_id: hjkl diff --git a/acceptance/bundle/telemetry/deploy-experimental/out.test.toml b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml new file mode 100644 index 0000000000..8f3575be7b --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-experimental/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/telemetry/deploy-experimental/output.txt b/acceptance/bundle/telemetry/deploy-experimental/output.txt new file mode 100644 index 0000000000..e5e2ea254c --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-experimental/output.txt @@ -0,0 +1,44 @@ + +>>> [CLI] bundle deploy +Warning: You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the DLT pipelines in your DAB as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC. + at experimental.use_legacy_run_as + in databricks.yml:4:22 + +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> cat out.requests.txt +{ + "bool_values": [ + { + "key": "experimental.use_legacy_run_as", + "value": true + }, + { + "key": "presets_name_prefix_is_set", + "value": false + }, + { + "key": "skip_artifact_cleanup", + "value": false + }, + { + "key": "python_wheel_wrapper_is_set", + "value": false + }, + { + "key": "has_serverless_compute", + "value": false + }, + { + "key": "has_classic_job_compute", + "value": false + }, + { + "key": "has_classic_interactive_compute", + "value": true + } + ] +} diff --git a/acceptance/bundle/telemetry/deploy-experimental/script b/acceptance/bundle/telemetry/deploy-experimental/script new file mode 100644 index 0000000000..67a3ba6299 --- /dev/null +++ b/acceptance/bundle/telemetry/deploy-experimental/script @@ -0,0 +1,5 @@ +trace $CLI bundle deploy + +trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental | {bool_values}' + +rm out.requests.txt diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt index 350ba032bf..35c1c4070e 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt @@ -8,6 +8,10 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ + { + "key": "experimental.use_legacy_run_as", + "value": false + }, { "key": "presets_name_prefix_is_set", "value": true diff --git a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt index 11222ebafd..c54466ab89 100644 --- a/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt +++ b/acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt @@ -8,6 +8,10 @@ Deployment complete! >>> cat out.requests.txt { "bool_values": [ + { + "key": "experimental.use_legacy_run_as", + "value": false + }, { "key": "presets_name_prefix_is_set", "value": true diff --git a/acceptance/bundle/telemetry/deploy/out.telemetry.txt b/acceptance/bundle/telemetry/deploy/out.telemetry.txt index 05afbaafeb..8d4afad759 100644 --- a/acceptance/bundle/telemetry/deploy/out.telemetry.txt +++ b/acceptance/bundle/telemetry/deploy/out.telemetry.txt @@ -42,6 +42,10 @@ "lookup_variable_count": 0, "target_count": 1, "bool_values": [ + { + "key": "experimental.use_legacy_run_as", + "value": false + }, { "key": "presets_name_prefix_is_set", "value": false diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index fb5408bfbd..e04b7410fa 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -181,6 +181,9 @@ func setPipelineOwnersToRunAsIdentity(b *bundle.Bundle) { } func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + // Track the use of the legacy run_as mode. + b.Metrics.AddBoolValue("experimental.use_legacy_run_as", b.Config.Experimental != nil && b.Config.Experimental.UseLegacyRunAs) + // Mutator is a no-op if run_as is not specified in the bundle if b.Config.Value().Get("run_as").Kind() == dyn.KindInvalid { return nil From 3111f151570f3304ee471cfe345f656bb6f81d59 Mon Sep 17 00:00:00 2001 From: Shreyas Goenka Date: Wed, 30 Jul 2025 13:21:22 +0200 Subject: [PATCH 2/2] fix tests --- acceptance/bundle/paths/fallback_metric/output.txt | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acceptance/bundle/paths/fallback_metric/output.txt b/acceptance/bundle/paths/fallback_metric/output.txt index 77e41b7336..6b8ef00e47 100644 --- a/acceptance/bundle/paths/fallback_metric/output.txt +++ b/acceptance/bundle/paths/fallback_metric/output.txt @@ -14,6 +14,10 @@ Deployment complete! >>> cat out.requests.txt [ + { + "key": "experimental.use_legacy_run_as", + "value": false + }, { "key": "presets_name_prefix_is_set", "value": false @@ -40,6 +44,10 @@ Deployment complete! } ] [ + { + "key": "experimental.use_legacy_run_as", + "value": false + }, { "key": "presets_name_prefix_is_set", "value": false