postgres: Support purge_on_delete on postgres_projects

NEXT_CHANGELOG.md

@@ -12,6 +12,7 @@
 * Preserve `.designer.ipynb` suffix when translating notebook task paths so Lakeflow Designer files referenced from a `notebook_task` resolve correctly in the workspace ([#5370](https://github.com/databricks/cli/pull/5370)).
 * Fix script output dropping last line without trailing newline ([#4995](https://github.com/databricks/cli/pull/4995)).
 * Add `--select` flag to `bundle plan` and `bundle deploy` to plan/deploy a subset of resources (e.g. `--select my_job` or `--select jobs.my_job`); resources referenced by the selection are included transitively. Direct engine only ([#5413](https://github.com/databricks/cli/pull/5413)).
+* Support `purge_on_delete: true` on `postgres_projects` so bundles can hard-delete a Lakebase project on destroy (skipping the soft-delete retention window) ([#5414](https://github.com/databricks/cli/pull/5414)).
 
 ### Dependency updates
 

acceptance/bundle/refschema/out.fields.txt

@@ -2915,6 +2915,7 @@ resources.postgres_projects.*.modified_status	string	INPUT
 resources.postgres_projects.*.name	string	REMOTE
 resources.postgres_projects.*.pg_version	int	ALL
 resources.postgres_projects.*.project_id	string	ALL
+resources.postgres_projects.*.purge_on_delete	bool	INPUT	STATE
 resources.postgres_projects.*.purge_time	*time.Time	REMOTE
 resources.postgres_projects.*.status	*postgres.ProjectStatus	REMOTE
 resources.postgres_projects.*.status.branch_logical_size_limit_bytes	int64	REMOTE

acceptance/bundle/resources/postgres_projects/purge_on_delete/databricks.yml.tmpl

@@ -0,0 +1,22 @@
+bundle:
+  name: deploy-postgres-purge-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+# Two projects side by side so the recorded destroy requests make the
+# purge_on_delete contrast obvious:
+#   - hard_delete:  purge_on_delete: true   -> DELETE …?purge=true
+#   - soft_delete:  field omitted (default) -> DELETE …    (no purge query)
+resources:
+  postgres_projects:
+    hard_delete:
+      project_id: test-pg-proj-hard-$UNIQUE_NAME
+      display_name: "purge_on_delete = true"
+      pg_version: 16
+      purge_on_delete: true
+
+    soft_delete:
+      project_id: test-pg-proj-soft-$UNIQUE_NAME
+      display_name: "purge_on_delete unset (soft delete)"
+      pg_version: 16

acceptance/bundle/resources/postgres_projects/purge_on_delete/out.destroy.txt

@@ -0,0 +1,13 @@
+The following resources will be deleted:
+  delete resources.postgres_projects.hard_delete
+  delete resources.postgres_projects.soft_delete
+
+This action will result in the deletion of the following Lakebase projects along with
+all their branches, databases, and endpoints. All data stored in them will be permanently lost:
+  delete resources.postgres_projects.hard_delete
+  delete resources.postgres_projects.soft_delete
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-purge-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.deploy.json

@@ -0,0 +1,34 @@
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-hard-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-soft-[UNIQUE_NAME]"
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects",
+  "q": {
+    "project_id": "test-pg-proj-hard-[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "display_name": "purge_on_delete = true",
+      "pg_version": 16
+    }
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/postgres/projects",
+  "q": {
+    "project_id": "test-pg-proj-soft-[UNIQUE_NAME]"
+  },
+  "body": {
+    "spec": {
+      "display_name": "purge_on_delete unset (soft delete)",
+      "pg_version": 16
+    }
+  }
+}

acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.destroy.json

@@ -0,0 +1,19 @@
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-hard-[UNIQUE_NAME]",
+  "q": {
+    "purge": "true"
+  }
+}
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-soft-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-hard-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-soft-[UNIQUE_NAME]"
+}

acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt

@@ -0,0 +1,52 @@
+
+>>> [CLI] bundle validate
+Name: deploy-postgres-purge-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-purge-[UNIQUE_NAME]/default
+
+Validation OK!
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-purge-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] postgres get-project projects/test-pg-proj-hard-[UNIQUE_NAME]
+{
+  "name": "projects/test-pg-proj-hard-[UNIQUE_NAME]",
+  "status": {
+    "branch_logical_size_limit_bytes": [NUMID],
+    "default_branch": "projects/test-pg-proj-hard-[UNIQUE_NAME]/branches/production",
+    "display_name": "purge_on_delete = true",
+    "enable_pg_native_login": false,
+    "owner": "[USERNAME]",
+    "pg_version": 16,
+    "project_id": "test-pg-proj-hard-[UNIQUE_NAME]",
+    "synthetic_storage_size_bytes": 0
+  },
+  "uid": "[UUID]"
+}
+
+>>> [CLI] postgres get-project projects/test-pg-proj-soft-[UNIQUE_NAME]
+{
+  "name": "projects/test-pg-proj-soft-[UNIQUE_NAME]",
+  "status": {
+    "branch_logical_size_limit_bytes": [NUMID],
+    "default_branch": "projects/test-pg-proj-soft-[UNIQUE_NAME]/branches/production",
+    "display_name": "purge_on_delete unset (soft delete)",
+    "enable_pg_native_login": false,
+    "owner": "[USERNAME]",
+    "pg_version": 16,
+    "project_id": "test-pg-proj-soft-[UNIQUE_NAME]",
+    "synthetic_storage_size_bytes": 0
+  },
+  "uid": "[UUID]"
+}
+
+>>> print_requests.py --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/
+
+=== bundle destroy
+>>> print_requests.py --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/

acceptance/bundle/resources/postgres_projects/purge_on_delete/script

@@ -0,0 +1,28 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+   # Belt-and-braces in case bundle destroy was skipped or partially failed.
+   # The soft-delete case leaves a record in the trash; --purge clears it.
+   $CLI postgres delete-project --purge "projects/test-pg-proj-hard-${UNIQUE_NAME}" 2>>LOG.delete-project || true
+   $CLI postgres delete-project --purge "projects/test-pg-proj-soft-${UNIQUE_NAME}" 2>>LOG.delete-project || true
+   rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+trace $CLI bundle validate
+
+rm -f out.requests.txt
+trace $CLI bundle deploy
+
+trace $CLI postgres get-project "projects/test-pg-proj-hard-${UNIQUE_NAME}" | jq 'del(.create_time, .update_time)'
+trace $CLI postgres get-project "projects/test-pg-proj-soft-${UNIQUE_NAME}" | jq 'del(.create_time, .update_time)'
+
+trace print_requests.py --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.json
+
+# bundle destroy should send ?purge=true on the hard_delete project and no
+# purge query on the soft_delete project. Both DELETEs land in the recorded
+# requests so the contrast is visible in the diff.
+title "bundle destroy"
+$CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true
+
+trace print_requests.py --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.json

acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/databricks.yml.tmpl

@@ -0,0 +1,17 @@
+bundle:
+  name: pg-purge-transitions-$UNIQUE_NAME
+
+sync:
+  paths: []
+
+# Walks purge_on_delete through unset -> true -> false -> unset, deploying
+# at each step and inspecting the persisted state so reviewers can see that
+# state tracks the user's latest intent. The final destroy is a soft delete
+# (state has purge_on_delete unset), recorded for regression coverage.
+resources:
+  postgres_projects:
+    proj:
+      project_id: test-pg-proj-$UNIQUE_NAME
+      display_name: "Transitions test"
+      pg_version: 16
+      # PURGE_ON_DELETE

acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.destroy.txt

@@ -0,0 +1,11 @@
+The following resources will be deleted:
+  delete resources.postgres_projects.proj
+
+This action will result in the deletion of the following Lakebase projects along with
+all their branches, databases, and endpoints. All data stored in them will be permanently lost:
+  delete resources.postgres_projects.proj
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.requests.destroy.json

@@ -0,0 +1,8 @@
+{
+  "method": "DELETE",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]"
+}
+{
+  "method": "GET",
+  "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]"
+}

acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt

@@ -0,0 +1,43 @@
+
+=== Step 1: deploy with purge_on_delete unset
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> get_purge
+(unset)
+
+=== Step 2: set purge_on_delete: true
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> get_purge
+json.state.resources.postgres_projects.proj.state.purge_on_delete = true;
+
+=== Step 3: flip to purge_on_delete: false
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> get_purge
+json.state.resources.postgres_projects.proj.state.purge_on_delete = false;
+
+=== Step 4: remove the line again
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> get_purge
+json.state.resources.postgres_projects.proj.state.purge_on_delete = false;
+
+=== bundle destroy (must be a plain DELETE, no ?purge=true)
+>>> print_requests.py --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/

acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script

@@ -0,0 +1,41 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+   # After this test the destroy should be a soft delete; --purge here so the
+   # trash entry doesn't linger past the test.
+   $CLI postgres delete-project --purge "projects/test-pg-proj-${UNIQUE_NAME}" 2>>LOG.delete-project || true
+   rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+# Prints the persisted purge_on_delete assignment from the direct engine's
+# resources.json. The field is omitted from JSON when unset (omitempty +
+# ForceSendFields tracking), so an absent value falls back to "(unset)".
+get_purge() {
+  gron.py < .databricks/bundle/default/resources.json | grep purge_on_delete || echo '(unset)'
+}
+
+title "Step 1: deploy with purge_on_delete unset"
+trace $CLI bundle deploy
+trace get_purge
+
+title "Step 2: set purge_on_delete: true"
+update_file.py databricks.yml "# PURGE_ON_DELETE" "purge_on_delete: true"
+trace $CLI bundle deploy
+trace get_purge
+
+title "Step 3: flip to purge_on_delete: false"
+update_file.py databricks.yml "purge_on_delete: true" "purge_on_delete: false"
+trace $CLI bundle deploy
+trace get_purge
+
+title "Step 4: remove the line again"
+update_file.py databricks.yml "purge_on_delete: false" "# PURGE_ON_DELETE"
+trace $CLI bundle deploy
+trace get_purge
+
+rm -f out.requests.txt
+
+title "bundle destroy (must be a plain DELETE, no ?purge=true)"
+$CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true
+trace print_requests.py --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.json

acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/test.toml

@@ -0,0 +1,4 @@
+# Direct engine only: this test exercises the FSF preservation in
+# PrepareState and direct's plan/diff classification when the user flips
+# purge_on_delete. Terraform has its own provider-managed state lifecycle.
+EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"]

bundle/config/resources/postgres_project.go

@@ -16,6 +16,18 @@ type PostgresProjectConfig struct {
 	// ProjectId is the user-specified ID for the project (becomes part of the hierarchical name).
 	// This is specified during creation and becomes part of Name: "projects/{project_id}"
 	ProjectId string `json:"project_id"`
+
+	// PurgeOnDelete, when true, hard-deletes the project on destroy (Purge=true on
+	// DeleteProject). When false or unset, the backend performs a soft delete that
+	// can be undone within the project's retention window. Input-only: not
+	// returned by the GET API.
+	PurgeOnDelete bool `json:"purge_on_delete,omitempty"`
+
+	// ForceSendFields shadows the embedded ProjectSpec.ForceSendFields so the
+	// SDK's marshal package tracks zero-value top-level fields (project_id,
+	// purge_on_delete) here instead of polluting ProjectSpec.ForceSendFields
+	// with names that don't exist in that struct.
+	ForceSendFields []string `json:"-" url:"-"`
 }
 
 func (c *PostgresProjectConfig) UnmarshalJSON(b []byte) error {

bundle/deploy/terraform/tfdyn/convert_postgres_project.go

@@ -16,6 +16,7 @@ func (c postgresProjectConverter) Convert(ctx context.Context, key string, vin d
 	// The bundle config has flattened ProjectSpec fields at the top level.
 	// Terraform expects them nested in a "spec" block.
 	specFields := specFieldNames(schema.ResourcePostgresProjectSpec{})
+	topLevelFields := []string{"project_id", "purge_on_delete"}
 
 	// Build the spec block from the flattened fields
 	specMap := make(map[string]dyn.Value)
@@ -25,12 +26,14 @@ func (c postgresProjectConverter) Convert(ctx context.Context, key string, vin d
 		}
 	}
 
-	// Build the output with project_id and spec
+	// Build the output with project_id, purge_on_delete and spec
 	outMap := make(map[string]dyn.Value)
 
-	// Keep project_id at top level
-	if v := vin.Get("project_id"); v.Kind() != dyn.KindInvalid {
-		outMap["project_id"] = v
+	// Keep top-level fields outside the spec block
+	for _, field := range topLevelFields {
+		if v := vin.Get(field); v.Kind() != dyn.KindInvalid {
+			outMap[field] = v
+		}
 	}
 
 	// Add spec block if we have any spec fields