diff --git a/acceptance/bundle/run_as/databricks.yml b/acceptance/bundle/run_as/databricks.yml new file mode 100644 index 0000000000..88f96155af --- /dev/null +++ b/acceptance/bundle/run_as/databricks.yml @@ -0,0 +1,40 @@ +resources: + jobs: + job_one: + name: Job 1 + + job_two: + name: Job 2 + run_as: + service_principal_name: "sp_override" + + job_three: + name: Job 3 + run_as: + user_name: "user_override" + + pipelines: + pipeline_one: + name: Pipeline 1 + + pipeline_two: + name: Pipeline 2 + run_as: + service_principal_name: "sp_override" + + pipeline_three: + name: Pipeline 3 + run_as: + user_name: "user_override" + +targets: + no_run_as: + default: true + + with_user_run_as: + run_as: + user_name: "user_base" + + with_sp_run_as: + run_as: + service_principal_name: "sp_base" diff --git a/acceptance/bundle/run_as/out.test.toml b/acceptance/bundle/run_as/out.test.toml new file mode 100644 index 0000000000..8f3575be7b --- /dev/null +++ b/acceptance/bundle/run_as/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/run_as/output.txt b/acceptance/bundle/run_as/output.txt new file mode 100644 index 0000000000..ca8c1b13bc --- /dev/null +++ b/acceptance/bundle/run_as/output.txt @@ -0,0 +1,24 @@ + +>>> [CLI] bundle validate -o json -t no_run_as +jobs.job_one: null +jobs.job_three: {"user_name":"user_override"} +jobs.job_two: {"service_principal_name":"sp_override"} +pipelines.pipeline_one: null +pipelines.pipeline_three: {"user_name":"user_override"} +pipelines.pipeline_two: {"service_principal_name":"sp_override"} + +>>> [CLI] bundle validate -o json -t with_user_run_as +jobs.job_one: {"user_name":"user_base"} +jobs.job_three: {"user_name":"user_override"} +jobs.job_two: {"service_principal_name":"sp_override"} +pipelines.pipeline_one: {"user_name":"user_base"} +pipelines.pipeline_three: {"user_name":"user_override"} +pipelines.pipeline_two: {"service_principal_name":"sp_override"} + +>>> [CLI] bundle validate -o json -t with_sp_run_as +jobs.job_one: {"service_principal_name":"sp_base"} +jobs.job_three: {"user_name":"user_override"} +jobs.job_two: {"service_principal_name":"sp_override"} +pipelines.pipeline_one: {"service_principal_name":"sp_base"} +pipelines.pipeline_three: {"user_name":"user_override"} +pipelines.pipeline_two: {"service_principal_name":"sp_override"} diff --git a/acceptance/bundle/run_as/pipelines/_script b/acceptance/bundle/run_as/pipelines/_script index 75a56b6364..2396eaa88d 100644 --- a/acceptance/bundle/run_as/pipelines/_script +++ b/acceptance/bundle/run_as/pipelines/_script @@ -1,4 +1,4 @@ trace errcode $CLI bundle validate -t t_user_name -trace musterr $CLI bundle validate -t t_user_name_different +trace errcode $CLI bundle validate -t t_user_name_different trace errcode $CLI bundle validate -t t_service_principal_name -trace musterr $CLI bundle validate -t t_service_principal_name_different +trace errcode $CLI bundle validate -t t_service_principal_name_different diff --git a/acceptance/bundle/run_as/pipelines/regular_user/output.txt b/acceptance/bundle/run_as/pipelines/regular_user/output.txt index 725d7ab8a0..8f5a1a4adf 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/output.txt +++ b/acceptance/bundle/run_as/pipelines/regular_user/output.txt @@ -1,5 +1,5 @@ -=== t_user_name target must succeed, the rest must fail +=== run_as should succeed in all cases >>> errcode [CLI] bundle validate -t t_user_name Name: run_as Target: t_user_name @@ -9,50 +9,29 @@ Workspace: Validation OK! ->>> musterr [CLI] bundle validate -t t_user_name_different -Error: pipelines do not support a setting a run_as user that is different from the owner. -Current identity: [USERNAME]. Run as identity: different@databricks.com. -See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property. - in databricks.yml:20:5 - +>>> errcode [CLI] bundle validate -t t_user_name_different Name: run_as Target: t_user_name_different Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_user_name_different -Found 1 error - -Exit code (musterr): 1 +Validation OK! >>> errcode [CLI] bundle validate -t t_service_principal_name -Error: pipelines do not support a setting a run_as user that is different from the owner. -Current identity: [USERNAME]. Run as identity: [UUID]. -See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property. - in databricks.yml:20:5 - Name: run_as Target: t_service_principal_name Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_service_principal_name -Found 1 error - -Exit code: 1 - ->>> musterr [CLI] bundle validate -t t_service_principal_name_different -Error: pipelines do not support a setting a run_as user that is different from the owner. -Current identity: [USERNAME]. Run as identity: [UUID]. -See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property. - in databricks.yml:20:5 +Validation OK! +>>> errcode [CLI] bundle validate -t t_service_principal_name_different Name: run_as Target: t_service_principal_name_different Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_service_principal_name_different -Found 1 error - -Exit code (musterr): 1 +Validation OK! diff --git a/acceptance/bundle/run_as/pipelines/regular_user/script b/acceptance/bundle/run_as/pipelines/regular_user/script index 2110198b94..b02e9b33a0 100644 --- a/acceptance/bundle/run_as/pipelines/regular_user/script +++ b/acceptance/bundle/run_as/pipelines/regular_user/script @@ -1,3 +1,3 @@ cp -r $TESTDIR/../{databricks.yml,dlt} . -title "t_user_name target must succeed, the rest must fail" +title "run_as should succeed in all cases" source $TESTDIR/../_script diff --git a/acceptance/bundle/run_as/pipelines/service_principal/output.txt b/acceptance/bundle/run_as/pipelines/service_principal/output.txt index cf1adfc985..8f5a1a4adf 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/output.txt +++ b/acceptance/bundle/run_as/pipelines/service_principal/output.txt @@ -1,36 +1,22 @@ -=== t_service_principal_name must succeed, the rest must fail +=== run_as should succeed in all cases >>> errcode [CLI] bundle validate -t t_user_name -Error: pipelines do not support a setting a run_as user that is different from the owner. -Current identity: [USERNAME]. Run as identity: tester@databricks.com. -See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property. - in databricks.yml:20:5 - Name: run_as Target: t_user_name Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_user_name -Found 1 error - -Exit code: 1 - ->>> musterr [CLI] bundle validate -t t_user_name_different -Error: pipelines do not support a setting a run_as user that is different from the owner. -Current identity: [USERNAME]. Run as identity: different@databricks.com. -See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property. - in databricks.yml:20:5 +Validation OK! +>>> errcode [CLI] bundle validate -t t_user_name_different Name: run_as Target: t_user_name_different Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_user_name_different -Found 1 error - -Exit code (musterr): 1 +Validation OK! >>> errcode [CLI] bundle validate -t t_service_principal_name Name: run_as @@ -41,18 +27,11 @@ Workspace: Validation OK! ->>> musterr [CLI] bundle validate -t t_service_principal_name_different -Error: pipelines do not support a setting a run_as user that is different from the owner. -Current identity: [USERNAME]. Run as identity: [UUID]. -See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property. - in databricks.yml:20:5 - +>>> errcode [CLI] bundle validate -t t_service_principal_name_different Name: run_as Target: t_service_principal_name_different Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_service_principal_name_different -Found 1 error - -Exit code (musterr): 1 +Validation OK! diff --git a/acceptance/bundle/run_as/pipelines/service_principal/script b/acceptance/bundle/run_as/pipelines/service_principal/script index 01b5302f87..b02e9b33a0 100644 --- a/acceptance/bundle/run_as/pipelines/service_principal/script +++ b/acceptance/bundle/run_as/pipelines/service_principal/script @@ -1,3 +1,3 @@ cp -r $TESTDIR/../{databricks.yml,dlt} . -title "t_service_principal_name must succeed, the rest must fail" +title "run_as should succeed in all cases" source $TESTDIR/../_script diff --git a/acceptance/bundle/run_as/script b/acceptance/bundle/run_as/script new file mode 100644 index 0000000000..5921ec6e28 --- /dev/null +++ b/acceptance/bundle/run_as/script @@ -0,0 +1,3 @@ +trace $CLI bundle validate -o json -t no_run_as | jq -r '.resources | to_entries[] | .key as $resource_type | .value | to_entries[] | "\($resource_type).\(.key): \(.value.run_as)"' +trace $CLI bundle validate -o json -t with_user_run_as | jq -r '.resources | to_entries[] | .key as $resource_type | .value | to_entries[] | "\($resource_type).\(.key): \(.value.run_as)"' +trace $CLI bundle validate -o json -t with_sp_run_as | jq -r '.resources | to_entries[] | .key as $resource_type | .value | to_entries[] | "\($resource_type).\(.key): \(.value.run_as)"' diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index e04b7410fa..d4b8f18e1e 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/databricks-sdk-go/service/jobs" + "github.com/databricks/databricks-sdk-go/service/pipelines" ) type setRunAs struct{} @@ -79,16 +80,6 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics { return diags } - // DLT pipelines do not support run_as in the API. - if len(b.Config.Resources.Pipelines) > 0 { - diags = diags.Extend(reportRunAsNotSupported( - "pipelines", - b.Config.GetLocation("resources.pipelines"), - b.Config.Workspace.CurrentUser.UserName, - identity, - )) - } - // Model serving endpoints do not support run_as in the API. if len(b.Config.Resources.ModelServingEndpoints) > 0 { diags = diags.Extend(reportRunAsNotSupported( @@ -150,6 +141,24 @@ func setRunAsForJobs(b *bundle.Bundle) { } } +func setRunAsForPipelines(b *bundle.Bundle) { + runAs := b.Config.RunAs + if runAs == nil { + return + } + + for i := range b.Config.Resources.Pipelines { + pipeline := b.Config.Resources.Pipelines[i] + if pipeline.RunAs != nil { + continue + } + pipeline.RunAs = &pipelines.RunAs{ + ServicePrincipalName: runAs.ServicePrincipalName, + UserName: runAs.UserName, + } + } +} + // Legacy behavior of run_as for DLT pipelines. Available under the experimental.use_run_as_legacy flag. // Only available to unblock customers stuck due to breaking changes in https://github.com/databricks/cli/pull/1233 func setPipelineOwnersToRunAsIdentity(b *bundle.Bundle) { @@ -189,6 +198,8 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { return nil } + // User has opted to use the legacy behavior of run_as with the + // experimental.use_legacy_run_as flag. if b.Config.Experimental != nil && b.Config.Experimental.UseLegacyRunAs { setPipelineOwnersToRunAsIdentity(b) setRunAsForJobs(b) @@ -209,5 +220,6 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { } setRunAsForJobs(b) + setRunAsForPipelines(b) return nil } diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 96f37acc55..784d24749a 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -95,6 +95,9 @@ func TestRunAsWorksForAllowedResources(t *testing.T) { Experiments: map[string]*resources.MlflowExperiment{ "experiment_one": {}, }, + Pipelines: map[string]*resources.Pipeline{ + "pipeline_one": {}, + }, }, } @@ -146,6 +149,7 @@ var allowList = []string{ "database_instances", "synced_database_tables", "jobs", + "pipelines", "models", "registered_models", "experiments",