Add top level run_as support for DLT pipelines

acceptance/bundle/run_as/databricks.yml

@@ -0,0 +1,40 @@
+resources:
+  jobs:
+    job_one:
+      name: Job 1
+
+    job_two:
+      name: Job 2
+      run_as:
+        service_principal_name: "sp_override"
+
+    job_three:
+      name: Job 3
+      run_as:
+        user_name: "user_override"
+
+  pipelines:
+    pipeline_one:
+      name: Pipeline 1
+
+    pipeline_two:
+      name: Pipeline 2
+      run_as:
+        service_principal_name: "sp_override"
+
+    pipeline_three:
+      name: Pipeline 3
+      run_as:
+        user_name: "user_override"
+
+targets:
+  no_run_as:
+    default: true
+
+  with_user_run_as:
+    run_as:
+      user_name: "user_base"
+
+  with_sp_run_as:
+    run_as:
+      service_principal_name: "sp_base"

acceptance/bundle/run_as/out.test.toml

@@ -0,0 +1,5 @@
+Local = true
+Cloud = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"]

acceptance/bundle/run_as/output.txt

@@ -0,0 +1,24 @@
+
+>>> [CLI] bundle validate -o json -t no_run_as
+jobs.job_one: null
+jobs.job_three: {"user_name":"user_override"}
+jobs.job_two: {"service_principal_name":"sp_override"}
+pipelines.pipeline_one: null
+pipelines.pipeline_three: {"user_name":"user_override"}
+pipelines.pipeline_two: {"service_principal_name":"sp_override"}
+
+>>> [CLI] bundle validate -o json -t with_user_run_as
+jobs.job_one: {"user_name":"user_base"}
+jobs.job_three: {"user_name":"user_override"}
+jobs.job_two: {"service_principal_name":"sp_override"}
+pipelines.pipeline_one: {"user_name":"user_base"}
+pipelines.pipeline_three: {"user_name":"user_override"}
+pipelines.pipeline_two: {"service_principal_name":"sp_override"}
+
+>>> [CLI] bundle validate -o json -t with_sp_run_as
+jobs.job_one: {"service_principal_name":"sp_base"}
+jobs.job_three: {"user_name":"user_override"}
+jobs.job_two: {"service_principal_name":"sp_override"}
+pipelines.pipeline_one: {"service_principal_name":"sp_base"}
+pipelines.pipeline_three: {"user_name":"user_override"}
+pipelines.pipeline_two: {"service_principal_name":"sp_override"}

acceptance/bundle/run_as/pipelines/_script

@@ -1,4 +1,4 @@
 trace errcode $CLI bundle validate -t t_user_name
-trace musterr $CLI bundle validate -t t_user_name_different
+trace errcode $CLI bundle validate -t t_user_name_different
 trace errcode $CLI bundle validate -t t_service_principal_name
-trace musterr $CLI bundle validate -t t_service_principal_name_different
+trace errcode $CLI bundle validate -t t_service_principal_name_different

acceptance/bundle/run_as/pipelines/regular_user/output.txt

@@ -1,5 +1,5 @@
 
-=== t_user_name target must succeed, the rest must fail
+=== run_as should succeed in all cases
 >>> errcode [CLI] bundle validate -t t_user_name
 Name: run_as
 Target: t_user_name
@@ -9,50 +9,29 @@ Workspace:
 
 Validation OK!
 
->>> musterr [CLI] bundle validate -t t_user_name_different
-Error: pipelines do not support a setting a run_as user that is different from the owner.
-Current identity: [USERNAME]. Run as identity: different@databricks.com.
-See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.
-  in databricks.yml:20:5
-
+>>> errcode [CLI] bundle validate -t t_user_name_different
 Name: run_as
 Target: t_user_name_different
 Workspace:
   User: [USERNAME]
   Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_user_name_different
 
-Found 1 error
-
-Exit code (musterr): 1
+Validation OK!
 
 >>> errcode [CLI] bundle validate -t t_service_principal_name
-Error: pipelines do not support a setting a run_as user that is different from the owner.
-Current identity: [USERNAME]. Run as identity: [UUID].
-See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.
-  in databricks.yml:20:5
-
 Name: run_as
 Target: t_service_principal_name
 Workspace:
   User: [USERNAME]
   Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_service_principal_name
 
-Found 1 error
-
-Exit code: 1
-
->>> musterr [CLI] bundle validate -t t_service_principal_name_different
-Error: pipelines do not support a setting a run_as user that is different from the owner.
-Current identity: [USERNAME]. Run as identity: [UUID].
-See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.
-  in databricks.yml:20:5
+Validation OK!
 
+>>> errcode [CLI] bundle validate -t t_service_principal_name_different
 Name: run_as
 Target: t_service_principal_name_different
 Workspace:
   User: [USERNAME]
   Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_service_principal_name_different
 
-Found 1 error
-
-Exit code (musterr): 1
+Validation OK!

acceptance/bundle/run_as/pipelines/regular_user/script

@@ -1,3 +1,3 @@
 cp -r $TESTDIR/../{databricks.yml,dlt} .
-title "t_user_name target must succeed, the rest must fail"
+title "run_as should succeed in all cases"
 source $TESTDIR/../_script

acceptance/bundle/run_as/pipelines/service_principal/output.txt

@@ -1,36 +1,22 @@
 
-=== t_service_principal_name must succeed, the rest must fail
+=== run_as should succeed in all cases
 >>> errcode [CLI] bundle validate -t t_user_name
-Error: pipelines do not support a setting a run_as user that is different from the owner.
-Current identity: [USERNAME]. Run as identity: tester@databricks.com.
-See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.
-  in databricks.yml:20:5
-
 Name: run_as
 Target: t_user_name
 Workspace:
   User: [USERNAME]
   Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_user_name
 
-Found 1 error
-
-Exit code: 1
-
->>> musterr [CLI] bundle validate -t t_user_name_different
-Error: pipelines do not support a setting a run_as user that is different from the owner.
-Current identity: [USERNAME]. Run as identity: different@databricks.com.
-See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.
-  in databricks.yml:20:5
+Validation OK!
 
+>>> errcode [CLI] bundle validate -t t_user_name_different
 Name: run_as
 Target: t_user_name_different
 Workspace:
   User: [USERNAME]
   Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_user_name_different
 
-Found 1 error
-
-Exit code (musterr): 1
+Validation OK!
 
 >>> errcode [CLI] bundle validate -t t_service_principal_name
 Name: run_as
@@ -41,18 +27,11 @@ Workspace:
 
 Validation OK!
 
->>> musterr [CLI] bundle validate -t t_service_principal_name_different
-Error: pipelines do not support a setting a run_as user that is different from the owner.
-Current identity: [USERNAME]. Run as identity: [UUID].
-See https://docs.databricks.com/dev-tools/bundles/run-as.html to learn more about the run_as property.
-  in databricks.yml:20:5
-
+>>> errcode [CLI] bundle validate -t t_service_principal_name_different
 Name: run_as
 Target: t_service_principal_name_different
 Workspace:
   User: [USERNAME]
   Path: /Workspace/Users/[USERNAME]/.bundle/run_as/t_service_principal_name_different
 
-Found 1 error
-
-Exit code (musterr): 1
+Validation OK!

acceptance/bundle/run_as/pipelines/service_principal/script

@@ -1,3 +1,3 @@
 cp -r $TESTDIR/../{databricks.yml,dlt} .
-title "t_service_principal_name must succeed, the rest must fail"
+title "run_as should succeed in all cases"
 source $TESTDIR/../_script

acceptance/bundle/run_as/script

@@ -0,0 +1,3 @@
+trace $CLI bundle validate -o json -t no_run_as | jq -r '.resources | to_entries[] | .key as $resource_type | .value | to_entries[] | "\($resource_type).\(.key): \(.value.run_as)"'
+trace $CLI bundle validate -o json -t with_user_run_as | jq -r '.resources | to_entries[] | .key as $resource_type | .value | to_entries[] | "\($resource_type).\(.key): \(.value.run_as)"'
+trace $CLI bundle validate -o json -t with_sp_run_as | jq -r '.resources | to_entries[] | .key as $resource_type | .value | to_entries[] | "\($resource_type).\(.key): \(.value.run_as)"'

bundle/config/mutator/resourcemutator/run_as.go

@@ -10,6 +10,7 @@ import (
 	"github.com/databricks/cli/libs/diag"
 	"github.com/databricks/cli/libs/dyn"
 	"github.com/databricks/databricks-sdk-go/service/jobs"
+	"github.com/databricks/databricks-sdk-go/service/pipelines"
 )
 
 type setRunAs struct{}
@@ -79,16 +80,6 @@ func validateRunAs(b *bundle.Bundle) diag.Diagnostics {
 		return diags
 	}
 
-	// DLT pipelines do not support run_as in the API.
-	if len(b.Config.Resources.Pipelines) > 0 {
-		diags = diags.Extend(reportRunAsNotSupported(
-			"pipelines",
-			b.Config.GetLocation("resources.pipelines"),
-			b.Config.Workspace.CurrentUser.UserName,
-			identity,
-		))
-	}
-
 	// Model serving endpoints do not support run_as in the API.
 	if len(b.Config.Resources.ModelServingEndpoints) > 0 {
 		diags = diags.Extend(reportRunAsNotSupported(
@@ -150,6 +141,24 @@ func setRunAsForJobs(b *bundle.Bundle) {
 	}
 }
 
+func setRunAsForPipelines(b *bundle.Bundle) {
+	runAs := b.Config.RunAs
+	if runAs == nil {
+		return
+	}
+
+	for i := range b.Config.Resources.Pipelines {
+		pipeline := b.Config.Resources.Pipelines[i]
+		if pipeline.RunAs != nil {
+			continue
+		}
+		pipeline.RunAs = &pipelines.RunAs{
+			ServicePrincipalName: runAs.ServicePrincipalName,
+			UserName:             runAs.UserName,
+		}
+	}
+}
+
 // Legacy behavior of run_as for DLT pipelines. Available under the experimental.use_run_as_legacy flag.
 // Only available to unblock customers stuck due to breaking changes in https://github.com/databricks/cli/pull/1233
 func setPipelineOwnersToRunAsIdentity(b *bundle.Bundle) {
@@ -189,6 +198,8 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 		return nil
 	}
 
+	// User has opted to use the legacy behavior of run_as with the
+	// experimental.use_legacy_run_as flag.
 	if b.Config.Experimental != nil && b.Config.Experimental.UseLegacyRunAs {
 		setPipelineOwnersToRunAsIdentity(b)
 		setRunAsForJobs(b)
@@ -209,5 +220,6 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics {
 	}
 
 	setRunAsForJobs(b)
+	setRunAsForPipelines(b)
 	return nil
 }

bundle/config/mutator/resourcemutator/run_as_test.go

@@ -95,6 +95,9 @@ func TestRunAsWorksForAllowedResources(t *testing.T) {
 			Experiments: map[string]*resources.MlflowExperiment{
 				"experiment_one": {},
 			},
+			Pipelines: map[string]*resources.Pipeline{
+				"pipeline_one": {},
+			},
 		},
 	}
 
@@ -146,6 +149,7 @@ var allowList = []string{
 	"database_instances",
 	"synced_database_tables",
 	"jobs",
+	"pipelines",
 	"models",
 	"registered_models",
 	"experiments",