Fixed bundle deploy to not update permissions for unbound resources

NEXT_CHANGELOG.md

@@ -10,6 +10,7 @@
 
 ### Bundles
 * Add new Lakeflow Pipelines support for bundle generate ([#3568](https://github.com/databricks/cli/pull/3568))
+* Fix bundle deploy to not update permissions or grants for unbound resources ([#3642](https://github.com/databricks/cli/pull/3642))
 * Introduce new bundle variable: `${workspace.current_user.domain_friendly_name}` ([#3623](https://github.com/databricks/cli/pull/3623))
 
 ### API Changes

acceptance/bundle/deploy/experimental-python/output.txt

@@ -8,6 +8,7 @@ Deployment complete!
 >>> [CLI] jobs list --output json
 [
   {
+    "creator_user_name": "[USERNAME]",
     "job_id": [NUMID],
     "settings": {
       "deployment": {

acceptance/bundle/deploy/python-notebook/output.txt

@@ -8,6 +8,7 @@ Deployment complete!
 >>> [CLI] jobs list --output json
 [
   {
+    "creator_user_name": "[USERNAME]",
     "job_id": [NUMID],
     "settings": {
       "deployment": {

acceptance/bundle/deployment/unbind/grants/databricks.yml.tmpl

@@ -0,0 +1,19 @@
+bundle:
+  name: unbind_grants-$UNIQUE_NAME
+
+workspace:
+  root_path: ~/.bundle/$UNIQUE_NAME
+
+variables:
+  suffix:
+    default: ""
+    description: "Suffix for the schema name"
+
+resources:
+  schemas:
+    schema_1:
+      name: "test-schema-$UNIQUE_NAME${var.suffix}"
+      catalog_name: "main"
+      grants:
+        - principal: "account users"
+          privileges: ["CREATE_VOLUME", "SELECT"]

acceptance/bundle/deployment/unbind/grants/output.txt

@@ -0,0 +1,49 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] grants get schema main.test-schema-[UNIQUE_NAME] --output json
+{
+  "principal": "account users",
+  "privileges": [
+    "CREATE_VOLUME",
+    "SELECT"
+  ]
+}
+
+>>> [CLI] bundle deployment unbind schema_1
+Updating deployment state...
+
+>>> [CLI] bundle deploy --var suffix=another
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Grants should be the same as before unbind
+>>> [CLI] grants get schema main.test-schema-[UNIQUE_NAME] --output json
+{
+  "principal": "account users",
+  "privileges": [
+    "CREATE_VOLUME",
+    "SELECT"
+  ]
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete schema schema_1
+
+This action will result in the deletion of the following UC schemas. Any underlying data may be lost:
+  delete schema schema_1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Deleting files...
+Destroy complete!
+
+>>> [CLI] schemas delete main.test-schema-[UNIQUE_NAME]
+0

acceptance/bundle/deployment/unbind/grants/script

@@ -0,0 +1,20 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    if [[ -n "$schema_id" ]]; then
+        trace $CLI schemas delete $schema_id
+    fi
+    echo $?
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+schema_id=$($CLI bundle summary --output json | jq -r '.resources.schemas.schema_1.id')
+trace $CLI grants get schema $schema_id --output json | jq '.privilege_assignments[] | select(.principal == "account users")'
+
+trace $CLI bundle deployment unbind schema_1
+trace $CLI bundle deploy --var "suffix=another"
+
+title "Grants should be the same as before unbind"
+trace $CLI grants get schema $schema_id --output json | jq '.privilege_assignments[] | select(.principal == "account users")'

acceptance/bundle/deployment/unbind/grants/test.toml

@@ -0,0 +1,8 @@
+RequiresUnityCatalog = true
+
+[EnvMatrix]
+  DATABRICKS_BUNDLE_ENGINE = ["terraform"]
+
+Ignore = [
+    ".databricks",
+]

acceptance/bundle/deployment/unbind/job/output.txt

@@ -18,6 +18,7 @@ Deployment complete!
 
 >>> [CLI] jobs get [NUMID] --output json
 {
+  "creator_user_name":"[USERNAME]",
   "job_id":[NUMID],
   "settings": {
     "deployment": {

acceptance/bundle/deployment/unbind/permissions/databricks.yml.tmpl

@@ -0,0 +1,13 @@
+bundle:
+  name: unbind_permissions
+
+workspace:
+  root_path: "~/.bundle/$UNIQUE_NAME"
+
+resources:
+  jobs:
+    job_1:
+      name: "Job name"
+      permissions:
+        - group_name: users
+          level: CAN_MANAGE

acceptance/bundle/deployment/unbind/permissions/output.txt

@@ -0,0 +1,52 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle summary --output json
+
+>>> [CLI] jobs get-permissions [NUMID] --output json
+{
+  "all_permissions": [
+    {
+      "inherited": false,
+      "permission_level": "CAN_MANAGE"
+    }
+  ],
+  "group_name": "users"
+}
+
+>>> [CLI] bundle deployment unbind job_1
+Updating deployment state...
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+=== Permissions should be the same as before unbind
+>>> [CLI] jobs get-permissions [NUMID] --output json
+{
+  "all_permissions": [
+    {
+      "inherited": false,
+      "permission_level": "CAN_MANAGE"
+    }
+  ],
+  "group_name": "users"
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete job job_1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]
+
+Deleting files...
+Destroy complete!
+
+>>> [CLI] jobs delete [NUMID]
+0

acceptance/bundle/deployment/unbind/permissions/script

@@ -0,0 +1,18 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    trace $CLI jobs delete $job_id
+    echo $?
+}
+trap cleanup EXIT
+
+trace $CLI bundle deploy
+job_id=$(trace $CLI bundle summary --output json | jq -r '.resources.jobs.job_1.id')
+trace $CLI jobs get-permissions $job_id --output json | jq '.access_control_list[] | select(.group_name == "users")'
+
+trace $CLI bundle deployment unbind job_1
+trace $CLI bundle deploy
+
+title "Permissions should be the same as before unbind"
+trace $CLI jobs get-permissions $job_id --output json | jq '.access_control_list[] | select(.group_name == "users")'

acceptance/bundle/deployment/unbind/permissions/test.toml

@@ -0,0 +1,6 @@
+[EnvMatrix]
+  DATABRICKS_BUNDLE_ENGINE = ["terraform"]
+
+Ignore = [
+    ".databricks",
+]

acceptance/bundle/deployment/unbind/python-job/output.txt

@@ -18,6 +18,7 @@ Deployment complete!
 
 >>> [CLI] jobs get [NUMID] --output json
 {
+  "creator_user_name":"[USERNAME]",
   "job_id":[NUMID],
   "settings": {
     "deployment": {

acceptance/bundle/resource_deps/jobs_update/output.txt

@@ -153,6 +153,7 @@ Deployment complete!
 
 >>> [CLI] jobs get [BAR_ID]
 {
+  "creator_user_name":"[USERNAME]",
   "job_id":[BAR_ID],
   "settings": {
     "deployment": {

bundle/deploy/terraform/unbind.go

@@ -10,8 +10,9 @@ import (
 )
 
 type unbind struct {
-	resourceType string
-	resourceKey  string
+	bundleType     string
+	tfResourceType string
+	resourceKey    string
 }
 
 func (m *unbind) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
@@ -25,18 +26,39 @@ func (m *unbind) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
 		return diag.Errorf("terraform init: %v", err)
 	}
 
-	err = tf.StateRm(ctx, fmt.Sprintf("%s.%s", m.resourceType, m.resourceKey))
+	err = tf.StateRm(ctx, fmt.Sprintf("%s.%s", m.tfResourceType, m.resourceKey))
 	if err != nil {
 		return diag.Errorf("terraform state rm: %v", err)
 	}
 
+	// Also remove the permission if it exists
+	// First check terraform state list to see if the permission exists
+	state, err := tf.Show(ctx)
+	if err != nil {
+		return diag.Errorf("terraform show: %v", err)
+	}
+
+	if state.Values == nil || state.Values.RootModule == nil || state.Values.RootModule.Resources == nil {
+		return nil
+	}
+
+	for _, resource := range state.Values.RootModule.Resources {
+		if resource.Address == fmt.Sprintf("databricks_permissions.%s_%s", m.bundleType, m.resourceKey) ||
+			resource.Address == fmt.Sprintf("databricks_grants.%s_%s", m.bundleType, m.resourceKey) {
+			err = tf.StateRm(ctx, resource.Address)
+			if err != nil {
+				return diag.Errorf("terraform state rm: %v", err)
+			}
+		}
+	}
+
 	return nil
 }
 
 func (*unbind) Name() string {
 	return "terraform.Unbind"
 }
 
-func Unbind(resourceType, resourceKey string) bundle.Mutator {
-	return &unbind{resourceType: resourceType, resourceKey: resourceKey}
+func Unbind(bundleType, tfResourceType, resourceKey string) bundle.Mutator {
+	return &unbind{bundleType: bundleType, tfResourceType: tfResourceType, resourceKey: resourceKey}
 }

bundle/phases/bind.go

@@ -32,7 +32,7 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions) {
 	)
 }
 
-func Unbind(ctx context.Context, b *bundle.Bundle, resourceType, resourceKey string) {
+func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, resourceKey string) {
 	log.Info(ctx, "Phase: unbind")
 
 	bundle.ApplyContext(ctx, b, lock.Acquire())
@@ -48,7 +48,7 @@ func Unbind(ctx context.Context, b *bundle.Bundle, resourceType, resourceKey str
 		statemgmt.StatePull(),
 		terraform.Interpolate(),
 		terraform.Write(),
-		terraform.Unbind(resourceType, resourceKey),
+		terraform.Unbind(bundleType, tfResourceType, resourceKey),
 		statemgmt.StatePush(),
 	)
 }

cmd/bundle/deployment/unbind.go

@@ -76,8 +76,9 @@ To re-bind the resource later, use:
 			b.Config.Bundle.Deployment.Lock.Force = forceLock
 		})
 
-		tfName := terraform.GroupToTerraformName[resource.ResourceDescription().PluralName]
-		phases.Unbind(ctx, b, tfName, args[0])
+		rd := resource.ResourceDescription()
+		tfName := terraform.GroupToTerraformName[rd.PluralName]
+		phases.Unbind(ctx, b, rd.SingularName, tfName, args[0])
 		if logdiag.HasError(ctx) {
 			return root.ErrAlreadyPrinted
 		}

libs/testserver/fake_workspace.go

@@ -60,16 +60,17 @@ type FakeWorkspace struct {
 	repoIdByPath map[string]int64
 
 	// normally, ids are not sequential, but we make them sequential for deterministic diff
-	nextJobId    int64
-	nextJobRunId int64
-	Jobs         map[int64]jobs.Job
-	JobRuns      map[int64]jobs.Run
-
+	nextJobId       int64
+	nextJobRunId    int64
+	Jobs            map[int64]jobs.Job
+	JobRuns         map[int64]jobs.Run
+	JobPermissions  map[string][]jobs.JobAccessControlRequest
 	Pipelines       map[string]pipelines.GetPipelineResponse
 	PipelineUpdates map[string]bool
 	Monitors        map[string]catalog.MonitorInfo
 	Apps            map[string]apps.App
 	Schemas         map[string]catalog.SchemaInfo
+	SchemasGrants   map[string][]catalog.PrivilegeAssignment
 	Volumes         map[string]catalog.VolumeInfo
 	Dashboards      map[string]dashboards.Dashboard
 	SqlWarehouses   map[string]sql.GetWarehouseResponse
@@ -153,6 +154,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace {
 
 		Jobs:                 map[int64]jobs.Job{},
 		JobRuns:              map[int64]jobs.Run{},
+		JobPermissions:       map[string][]jobs.JobAccessControlRequest{},
+		SchemasGrants:        map[string][]catalog.PrivilegeAssignment{},
 		nextJobId:            TestJobID,
 		nextJobRunId:         TestRunID,
 		Pipelines:            map[string]pipelines.GetPipelineResponse{},