From 36bd5cc980e72f195a8a7bf1b9a181ce184ad101 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Tue, 2 Jun 2026 16:29:59 +0200 Subject: [PATCH 1/2] postgres: Support `purge_on_delete` on postgres_projects MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds support for `purge_on_delete` on Lakebase `postgres_projects` so bundles can hard-delete a project on destroy. The flag is passed to the DeleteProject API call as `?purge=true`; when unset, the backend performs a soft delete that can be undone via `databricks postgres undelete-project` within the project's retention window. The field is input-only — it is not modeled by the backend resource for projects, and the GET API never returns it. We store it in state purely so DoDelete can apply it on destroy: by that point the configuration for the resource is gone, so state is the only place it can live. PrepareState preserves `input.ForceSendFields` so the structdiff comparison correctly distinguishes "explicit false" from the fictional remote zero — otherwise toggling `true -> false` would be classified as no change, state would stay `true`, and the next destroy would still emit `?purge=true`. DoUpdate strips `purge_on_delete` from the API field mask so a state-only flip doesn't fire an unnecessary remote write. Acceptance tests under `acceptance/bundle/resources/postgres_projects/`: - `purge_on_delete/`: deploys a `hard_delete` and a `soft_delete` project side by side and asserts the destroy emits `?purge=true` and a plain DELETE respectively, on both engines. - `purge_on_delete_transitions/`: direct-engine only. Walks `purge_on_delete` through unset -> true -> false -> unset and records the persisted value at each step; final destroy is a plain DELETE. Regression coverage for the FSF-preservation fix. Manually verified against dogfood with ephemeral projects on both engines (deploy -> flip -> destroy; GET on project/branches/endpoints in soft vs hard cases; native `postgres undelete-project` restoration semantics — captured in DECO-27233). Co-authored-by: Isaac --- NEXT_CHANGELOG.md | 1 + acceptance/bundle/refschema/out.fields.txt | 1 + .../purge_on_delete/databricks.yml.tmpl | 22 ++++++++ .../purge_on_delete/out.destroy.txt | 13 +++++ .../purge_on_delete/out.requests.deploy.json | 34 ++++++++++++ .../purge_on_delete/out.requests.destroy.json | 53 +++++++++++++++++++ .../purge_on_delete/out.test.toml | 6 +++ .../purge_on_delete/output.txt | 52 ++++++++++++++++++ .../postgres_projects/purge_on_delete/script | 28 ++++++++++ .../databricks.yml.tmpl | 17 ++++++ .../out.destroy.txt | 11 ++++ .../out.requests.destroy.json | 8 +++ .../purge_on_delete_transitions/out.test.toml | 6 +++ .../purge_on_delete_transitions/output.txt | 43 +++++++++++++++ .../purge_on_delete_transitions/script | 41 ++++++++++++++ .../purge_on_delete_transitions/test.toml | 4 ++ bundle/config/resources/postgres_project.go | 12 +++++ .../tfdyn/convert_postgres_project.go | 11 ++-- .../tfdyn/convert_postgres_project_test.go | 29 ++++++++++ bundle/direct/dresources/postgres_project.go | 27 ++++++++-- bundle/direct/dresources/resources.yml | 7 +++ bundle/direct/dresources/type_test.go | 3 ++ bundle/internal/schema/annotations.yml | 3 ++ bundle/schema/jsonschema.json | 3 ++ bundle/terraform_dabs_map/generated.go | 4 -- libs/testserver/postgres.go | 5 +- 26 files changed, 431 insertions(+), 13 deletions(-) create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete/out.destroy.txt create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.deploy.json create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.destroy.json create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete/script create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.destroy.txt create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.requests.destroy.json create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script create mode 100644 acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/test.toml diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index df070a324a5..3c1a43f4468 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,5 +11,6 @@ * Retry transient HTTP 5xx and 408 errors in direct deployment engine ([#5349](https://github.com/databricks/cli/pull/5349), [#5364](https://github.com/databricks/cli/pull/5364)). * Preserve `.designer.ipynb` suffix when translating notebook task paths so Lakeflow Designer files referenced from a `notebook_task` resolve correctly in the workspace ([#5370](https://github.com/databricks/cli/pull/5370)). * Fix script output dropping last line without trailing newline ([#4995](https://github.com/databricks/cli/pull/4995)). +* Support `purge_on_delete: true` on `postgres_projects` so bundles can hard-delete a Lakebase project on destroy (skipping the soft-delete retention window). ### Dependency updates diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index 96000f283e1..bb738dd29a8 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -2915,6 +2915,7 @@ resources.postgres_projects.*.modified_status string INPUT resources.postgres_projects.*.name string REMOTE resources.postgres_projects.*.pg_version int ALL resources.postgres_projects.*.project_id string ALL +resources.postgres_projects.*.purge_on_delete bool INPUT STATE resources.postgres_projects.*.purge_time *time.Time REMOTE resources.postgres_projects.*.status *postgres.ProjectStatus REMOTE resources.postgres_projects.*.status.branch_logical_size_limit_bytes int64 REMOTE diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_projects/purge_on_delete/databricks.yml.tmpl new file mode 100644 index 00000000000..44f05b0c885 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/databricks.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: deploy-postgres-purge-$UNIQUE_NAME + +sync: + paths: [] + +# Two projects side by side so the recorded destroy requests make the +# purge_on_delete contrast obvious: +# - hard_delete: purge_on_delete: true -> DELETE …?purge=true +# - soft_delete: field omitted (default) -> DELETE … (no purge query) +resources: + postgres_projects: + hard_delete: + project_id: test-pg-proj-hard-$UNIQUE_NAME + display_name: "purge_on_delete = true" + pg_version: 16 + purge_on_delete: true + + soft_delete: + project_id: test-pg-proj-soft-$UNIQUE_NAME + display_name: "purge_on_delete unset (soft delete)" + pg_version: 16 diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.destroy.txt b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.destroy.txt new file mode 100644 index 00000000000..24950b29d98 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.destroy.txt @@ -0,0 +1,13 @@ +The following resources will be deleted: + delete resources.postgres_projects.hard_delete + delete resources.postgres_projects.soft_delete + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.hard_delete + delete resources.postgres_projects.soft_delete + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-purge-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.deploy.json b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.deploy.json new file mode 100644 index 00000000000..a86773f6673 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.deploy.json @@ -0,0 +1,34 @@ +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-hard-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-soft-[UNIQUE_NAME]" +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-hard-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "purge_on_delete = true", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-soft-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "purge_on_delete unset (soft delete)", + "pg_version": 16 + } + } +} diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.destroy.json b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.destroy.json new file mode 100644 index 00000000000..39eeb050805 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.destroy.json @@ -0,0 +1,53 @@ +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-hard-[UNIQUE_NAME]", + "q": { + "purge": "true" + } +} +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-soft-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-hard-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-hard-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-soft-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-soft-[UNIQUE_NAME]" +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-hard-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "purge_on_delete = true", + "pg_version": 16 + } + } +} +{ + "method": "POST", + "path": "/api/2.0/postgres/projects", + "q": { + "project_id": "test-pg-proj-soft-[UNIQUE_NAME]" + }, + "body": { + "spec": { + "display_name": "purge_on_delete unset (soft delete)", + "pg_version": 16 + } + } +} diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml new file mode 100644 index 00000000000..110f841fa05 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct", "terraform"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt b/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt new file mode 100644 index 00000000000..030af095fb7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt @@ -0,0 +1,52 @@ + +>>> [CLI] bundle validate +Name: deploy-postgres-purge-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-purge-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-postgres-purge-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] postgres get-project projects/test-pg-proj-hard-[UNIQUE_NAME] +{ + "name": "projects/test-pg-proj-hard-[UNIQUE_NAME]", + "status": { + "branch_logical_size_limit_bytes": [NUMID], + "default_branch": "projects/test-pg-proj-hard-[UNIQUE_NAME]/branches/production", + "display_name": "purge_on_delete = true", + "enable_pg_native_login": false, + "owner": "[USERNAME]", + "pg_version": 16, + "project_id": "test-pg-proj-hard-[UNIQUE_NAME]", + "synthetic_storage_size_bytes": 0 + }, + "uid": "[UUID]" +} + +>>> [CLI] postgres get-project projects/test-pg-proj-soft-[UNIQUE_NAME] +{ + "name": "projects/test-pg-proj-soft-[UNIQUE_NAME]", + "status": { + "branch_logical_size_limit_bytes": [NUMID], + "default_branch": "projects/test-pg-proj-soft-[UNIQUE_NAME]/branches/production", + "display_name": "purge_on_delete unset (soft delete)", + "enable_pg_native_login": false, + "owner": "[USERNAME]", + "pg_version": 16, + "project_id": "test-pg-proj-soft-[UNIQUE_NAME]", + "synthetic_storage_size_bytes": 0 + }, + "uid": "[UUID]" +} + +>>> print_requests.py --keep --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ + +=== bundle destroy +>>> print_requests.py --keep --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/script b/acceptance/bundle/resources/postgres_projects/purge_on_delete/script new file mode 100644 index 00000000000..3420b294fa4 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/script @@ -0,0 +1,28 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + # Belt-and-braces in case bundle destroy was skipped or partially failed. + # The soft-delete case leaves a record in the trash; --purge clears it. + $CLI postgres delete-project --purge "projects/test-pg-proj-hard-${UNIQUE_NAME}" 2>/dev/null || true + $CLI postgres delete-project --purge "projects/test-pg-proj-soft-${UNIQUE_NAME}" 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT + +trace $CLI bundle validate + +rm -f out.requests.txt +trace $CLI bundle deploy + +trace $CLI postgres get-project "projects/test-pg-proj-hard-${UNIQUE_NAME}" | jq 'del(.create_time, .update_time)' +trace $CLI postgres get-project "projects/test-pg-proj-soft-${UNIQUE_NAME}" | jq 'del(.create_time, .update_time)' + +trace print_requests.py --keep --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.json + +# bundle destroy should send ?purge=true on the hard_delete project and no +# purge query on the soft_delete project. Both DELETEs land in the recorded +# requests so the contrast is visible in the diff. +title "bundle destroy" +$CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true + +trace print_requests.py --keep --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.json diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/databricks.yml.tmpl b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/databricks.yml.tmpl new file mode 100644 index 00000000000..7e98f348702 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/databricks.yml.tmpl @@ -0,0 +1,17 @@ +bundle: + name: pg-purge-transitions-$UNIQUE_NAME + +sync: + paths: [] + +# Walks purge_on_delete through unset -> true -> false -> unset, deploying +# at each step and inspecting the persisted state so reviewers can see that +# state tracks the user's latest intent. The final destroy is a soft delete +# (state has purge_on_delete unset), recorded for regression coverage. +resources: + postgres_projects: + proj: + project_id: test-pg-proj-$UNIQUE_NAME + display_name: "Transitions test" + pg_version: 16 + # PURGE_ON_DELETE diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.destroy.txt b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.destroy.txt new file mode 100644 index 00000000000..ca4b0f6893c --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.destroy.txt @@ -0,0 +1,11 @@ +The following resources will be deleted: + delete resources.postgres_projects.proj + +This action will result in the deletion of the following Lakebase projects along with +all their branches, databases, and endpoints. All data stored in them will be permanently lost: + delete resources.postgres_projects.proj + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.requests.destroy.json b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.requests.destroy.json new file mode 100644 index 00000000000..99384cda436 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.requests.destroy.json @@ -0,0 +1,8 @@ +{ + "method": "DELETE", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/api/2.0/postgres/projects/test-pg-proj-[UNIQUE_NAME]" +} diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml new file mode 100644 index 00000000000..4fe23e297fe --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true +CloudEnvs.azure = false +CloudEnvs.gcp = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt new file mode 100644 index 00000000000..88340b5dbe7 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt @@ -0,0 +1,43 @@ + +=== Step 1: deploy with purge_on_delete unset +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> get_purge +null + +=== Step 2: set purge_on_delete: true +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> get_purge +true + +=== Step 3: flip to purge_on_delete: false +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> get_purge +false + +=== Step 4: remove the line again +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pg-purge-transitions-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> get_purge +false + +=== bundle destroy (must be a plain DELETE, no ?purge=true) +>>> print_requests.py --keep --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script new file mode 100644 index 00000000000..9851ef948d4 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script @@ -0,0 +1,41 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + # After this test the destroy should be a soft delete; --purge here so the + # trash entry doesn't linger past the test. + $CLI postgres delete-project --purge "projects/test-pg-proj-${UNIQUE_NAME}" 2>/dev/null || true + rm -f out.requests.txt +} +trap cleanup EXIT + +# Prints the persisted purge_on_delete value from the direct engine's +# resources.json. The field is omitted from JSON when unset (omitempty + +# ForceSendFields tracking), so absent => null, explicit true/false => bool. +get_purge() { + jq -r '.state["resources.postgres_projects.proj"].state.purge_on_delete' .databricks/bundle/default/resources.json +} + +title "Step 1: deploy with purge_on_delete unset" +trace $CLI bundle deploy +trace get_purge + +title "Step 2: set purge_on_delete: true" +update_file.py databricks.yml "# PURGE_ON_DELETE" "purge_on_delete: true" +trace $CLI bundle deploy +trace get_purge + +title "Step 3: flip to purge_on_delete: false" +update_file.py databricks.yml "purge_on_delete: true" "purge_on_delete: false" +trace $CLI bundle deploy +trace get_purge + +title "Step 4: remove the line again" +update_file.py databricks.yml "purge_on_delete: false" "# PURGE_ON_DELETE" +trace $CLI bundle deploy +trace get_purge + +rm -f out.requests.txt + +title "bundle destroy (must be a plain DELETE, no ?purge=true)" +$CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true +trace print_requests.py --keep --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.json diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/test.toml b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/test.toml new file mode 100644 index 00000000000..797a2d57508 --- /dev/null +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/test.toml @@ -0,0 +1,4 @@ +# Direct engine only: this test exercises the FSF preservation in +# PrepareState and direct's plan/diff classification when the user flips +# purge_on_delete. Terraform has its own provider-managed state lifecycle. +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/resources/postgres_project.go b/bundle/config/resources/postgres_project.go index d76a6ee135c..2a61d4c5943 100644 --- a/bundle/config/resources/postgres_project.go +++ b/bundle/config/resources/postgres_project.go @@ -16,6 +16,18 @@ type PostgresProjectConfig struct { // ProjectId is the user-specified ID for the project (becomes part of the hierarchical name). // This is specified during creation and becomes part of Name: "projects/{project_id}" ProjectId string `json:"project_id"` + + // PurgeOnDelete, when true, hard-deletes the project on destroy (Purge=true on + // DeleteProject). When false or unset, the backend performs a soft delete that + // can be undone within the project's retention window. Input-only: not + // returned by the GET API. + PurgeOnDelete bool `json:"purge_on_delete,omitempty"` + + // ForceSendFields shadows the embedded ProjectSpec.ForceSendFields so the + // SDK's marshal package tracks zero-value top-level fields (project_id, + // purge_on_delete) here instead of polluting ProjectSpec.ForceSendFields + // with names that don't exist in that struct. + ForceSendFields []string `json:"-" url:"-"` } func (c *PostgresProjectConfig) UnmarshalJSON(b []byte) error { diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_project.go b/bundle/deploy/terraform/tfdyn/convert_postgres_project.go index b6b1294caf9..a9525c1cece 100644 --- a/bundle/deploy/terraform/tfdyn/convert_postgres_project.go +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_project.go @@ -16,6 +16,7 @@ func (c postgresProjectConverter) Convert(ctx context.Context, key string, vin d // The bundle config has flattened ProjectSpec fields at the top level. // Terraform expects them nested in a "spec" block. specFields := specFieldNames(schema.ResourcePostgresProjectSpec{}) + topLevelFields := []string{"project_id", "purge_on_delete"} // Build the spec block from the flattened fields specMap := make(map[string]dyn.Value) @@ -25,12 +26,14 @@ func (c postgresProjectConverter) Convert(ctx context.Context, key string, vin d } } - // Build the output with project_id and spec + // Build the output with project_id, purge_on_delete and spec outMap := make(map[string]dyn.Value) - // Keep project_id at top level - if v := vin.Get("project_id"); v.Kind() != dyn.KindInvalid { - outMap["project_id"] = v + // Keep top-level fields outside the spec block + for _, field := range topLevelFields { + if v := vin.Get(field); v.Kind() != dyn.KindInvalid { + outMap[field] = v + } } // Add spec block if we have any spec fields diff --git a/bundle/deploy/terraform/tfdyn/convert_postgres_project_test.go b/bundle/deploy/terraform/tfdyn/convert_postgres_project_test.go index 696a6e76c28..30251a4ada8 100644 --- a/bundle/deploy/terraform/tfdyn/convert_postgres_project_test.go +++ b/bundle/deploy/terraform/tfdyn/convert_postgres_project_test.go @@ -108,6 +108,35 @@ func TestConvertPostgresProjectWithPermissions(t *testing.T) { }, out.Permissions["postgres_project_my_postgres_project"]) } +func TestConvertPostgresProjectPurgeOnDelete(t *testing.T) { + src := resources.PostgresProject{ + PostgresProjectConfig: resources.PostgresProjectConfig{ + ProjectId: "my-project", + PurgeOnDelete: true, + ProjectSpec: postgres.ProjectSpec{ + DisplayName: "My Postgres Project", + }, + }, + } + + vin, err := convert.FromTyped(src, dyn.NilValue) + require.NoError(t, err) + + ctx := t.Context() + out := schema.NewResources() + err = postgresProjectConverter{}.Convert(ctx, "my_postgres_project", vin, out) + require.NoError(t, err) + + postgresProject := out.PostgresProject["my_postgres_project"] + assert.Equal(t, map[string]any{ + "project_id": "my-project", + "purge_on_delete": true, + "spec": map[string]any{ + "display_name": "My Postgres Project", + }, + }, postgresProject) +} + func TestConvertPostgresProjectMinimal(t *testing.T) { src := resources.PostgresProject{ PostgresProjectConfig: resources.PostgresProjectConfig{ diff --git a/bundle/direct/dresources/postgres_project.go b/bundle/direct/dresources/postgres_project.go index 0561cb709b3..00bd1af412e 100644 --- a/bundle/direct/dresources/postgres_project.go +++ b/bundle/direct/dresources/postgres_project.go @@ -2,6 +2,7 @@ package dresources import ( "context" + "slices" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/databricks-sdk-go" @@ -51,8 +52,10 @@ func (*ResourcePostgresProject) New(client *databricks.WorkspaceClient) *Resourc func (*ResourcePostgresProject) PrepareState(input *resources.PostgresProject) *PostgresProjectState { return &PostgresProjectState{ - ProjectId: input.ProjectId, - ProjectSpec: input.ProjectSpec, + ProjectId: input.ProjectId, + PurgeOnDelete: input.PurgeOnDelete, + ProjectSpec: input.ProjectSpec, + ForceSendFields: input.ForceSendFields, } } @@ -60,6 +63,11 @@ func (*ResourcePostgresProject) RemapState(remote *PostgresProjectRemote) *Postg return &PostgresProjectState{ ProjectId: remote.ProjectId, ProjectSpec: remote.ProjectSpec, + + // purge_on_delete is a delete-time query parameter; the GET API never + // returns it, so RemapState leaves it false. + PurgeOnDelete: false, + ForceSendFields: nil, } } @@ -137,6 +145,17 @@ func (r *ResourcePostgresProject) DoUpdate(ctx context.Context, id string, confi // not relative to our flattened state type. fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.") + // purge_on_delete is an input-only flag consulted at delete time; it is + // not a spec field. Strip it from the mask so toggling it between deploys + // becomes a state-only refresh (the framework saves newState when this + // returns nil error). + fieldPaths = slices.DeleteFunc(fieldPaths, func(p string) bool { + return p == "spec.purge_on_delete" + }) + if len(fieldPaths) == 0 { + return nil, nil + } + waiter, err := r.client.Postgres.UpdateProject(ctx, postgres.UpdateProjectRequest{ Project: postgres.Project{ Spec: &config.ProjectSpec, @@ -169,10 +188,10 @@ func (r *ResourcePostgresProject) DoUpdate(ctx context.Context, id string, confi return makePostgresProjectRemote(result), nil } -func (r *ResourcePostgresProject) DoDelete(ctx context.Context, id string, _ *PostgresProjectState) error { +func (r *ResourcePostgresProject) DoDelete(ctx context.Context, id string, state *PostgresProjectState) error { waiter, err := r.client.Postgres.DeleteProject(ctx, postgres.DeleteProjectRequest{ Name: id, - Purge: false, + Purge: state.PurgeOnDelete, ForceSendFields: nil, }) if err != nil { diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index bfb4e1f23bc..9c23ed0b413 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -485,6 +485,13 @@ resources: # project_id is immutable (part of hierarchical name, not in API spec) - field: project_id reason: immutable + ignore_remote_changes: + # purge_on_delete is a delete-time query parameter; not returned by GET. + # When the user changes it locally we still want the new value to land + # in state (so the next destroy honors it); DoUpdate strips it from the + # API field mask, leaving toggling as a state-only refresh. + - field: purge_on_delete + reason: input_only postgres_branches: recreate_on_changes: diff --git a/bundle/direct/dresources/type_test.go b/bundle/direct/dresources/type_test.go index 86f739cb2ca..46aaa6b7cbd 100644 --- a/bundle/direct/dresources/type_test.go +++ b/bundle/direct/dresources/type_test.go @@ -49,6 +49,9 @@ var knownMissingInRemoteType = map[string][]string{ "postgres_endpoints": { "replace_existing", }, + "postgres_projects": { + "purge_on_delete", + }, "vector_search_endpoints": { "target_qps", "usage_policy_id", diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 09cb8e2c1e2..6cdb3643c61 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -919,6 +919,9 @@ github.com/databricks/cli/bundle/config/resources.PostgresProject: "project_id": "description": |- PLACEHOLDER + "purge_on_delete": + "description": |- + PLACEHOLDER "spec": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 04d4e803a78..355d7e5aba3 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1541,6 +1541,9 @@ }, "project_id": { "$ref": "#/$defs/string" + }, + "purge_on_delete": { + "$ref": "#/$defs/bool" } }, "additionalProperties": false, diff --git a/bundle/terraform_dabs_map/generated.go b/bundle/terraform_dabs_map/generated.go index 536176f90d4..c623481c914 100644 --- a/bundle/terraform_dabs_map/generated.go +++ b/bundle/terraform_dabs_map/generated.go @@ -21,7 +21,6 @@ package terraform_dabs_map // postgres_branches / databricks_postgres_branch: 1 unwraps // postgres_catalogs / databricks_postgres_catalog: 1 unwraps // postgres_endpoints / databricks_postgres_endpoint: 1 unwraps -// postgres_projects / databricks_postgres_project: 1 tf-only // postgres_projects / databricks_postgres_project: 1 unwraps // postgres_synced_tables / databricks_postgres_synced_table: 1 unwraps // schemas / databricks_schema: 1 tf-only @@ -655,9 +654,6 @@ var TerraformOnlyFields = map[string]FieldSet{ "expected_last_modified": {}, "url": {}, }, - "postgres_projects": { - "purge_on_delete": {}, - }, "schemas": { "force_destroy": {}, }, diff --git a/libs/testserver/postgres.go b/libs/testserver/postgres.go index 65f187a83bd..c53c6f260f1 100644 --- a/libs/testserver/postgres.go +++ b/libs/testserver/postgres.go @@ -214,7 +214,10 @@ func (s *FakeWorkspace) PostgresProjectUpdate(req Request, name string) Response } } -// PostgresProjectDelete deletes a postgres project. +// PostgresProjectDelete deletes a postgres project. The `purge` query parameter +// is ignored: acceptance tests assert on the recorded HTTP request rather than +// on retention semantics, so a single "remove from map" action serves both +// hard- and soft-delete paths. func (s *FakeWorkspace) PostgresProjectDelete(name string) Response { defer s.LockUnlock()() From 243d15a7827e0cb6da00ab81c3f00fc860adab84 Mon Sep 17 00:00:00 2001 From: Pieter Noordhuis Date: Wed, 3 Jun 2026 09:49:18 +0200 Subject: [PATCH 2/2] Address PR review comments - script: drop --keep flags on print_requests.py (was double-printing deploy requests in destroy.json) and redirect cleanup-noise to LOG.delete-project so it's visible under `go test -v`. - purge_on_delete_transitions/script: replace inline jq with the `gron.py | grep` idiom used elsewhere; unset surfaces as "(unset)". - DoUpdate: keep the change-list approach. Tried a static spec mask but the API rejects "*" (expands to nested attrs the body must populate) and rejects fields in the mask that aren't also in the body, so the mask has to mirror what the user actually set. Confirmed on dogfood. Co-authored-by: Isaac --- .../purge_on_delete/out.requests.destroy.json | 34 ------------------- .../purge_on_delete/output.txt | 4 +-- .../postgres_projects/purge_on_delete/script | 8 ++--- .../purge_on_delete_transitions/output.txt | 10 +++--- .../purge_on_delete_transitions/script | 10 +++--- bundle/direct/dresources/postgres_project.go | 10 +++--- bundle/direct/dresources/resources.yml | 3 +- 7 files changed, 23 insertions(+), 56 deletions(-) diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.destroy.json b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.destroy.json index 39eeb050805..dac7481daa0 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.destroy.json +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/out.requests.destroy.json @@ -13,41 +13,7 @@ "method": "GET", "path": "/api/2.0/postgres/projects/test-pg-proj-hard-[UNIQUE_NAME]" } -{ - "method": "GET", - "path": "/api/2.0/postgres/projects/test-pg-proj-hard-[UNIQUE_NAME]" -} -{ - "method": "GET", - "path": "/api/2.0/postgres/projects/test-pg-proj-soft-[UNIQUE_NAME]" -} { "method": "GET", "path": "/api/2.0/postgres/projects/test-pg-proj-soft-[UNIQUE_NAME]" } -{ - "method": "POST", - "path": "/api/2.0/postgres/projects", - "q": { - "project_id": "test-pg-proj-hard-[UNIQUE_NAME]" - }, - "body": { - "spec": { - "display_name": "purge_on_delete = true", - "pg_version": 16 - } - } -} -{ - "method": "POST", - "path": "/api/2.0/postgres/projects", - "q": { - "project_id": "test-pg-proj-soft-[UNIQUE_NAME]" - }, - "body": { - "spec": { - "display_name": "purge_on_delete unset (soft delete)", - "pg_version": 16 - } - } -} diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt b/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt index 030af095fb7..ca49f906b6b 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/output.txt @@ -46,7 +46,7 @@ Deployment complete! "uid": "[UUID]" } ->>> print_requests.py --keep --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ +>>> print_requests.py --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ === bundle destroy ->>> print_requests.py --keep --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ +>>> print_requests.py --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete/script b/acceptance/bundle/resources/postgres_projects/purge_on_delete/script index 3420b294fa4..c093ccb48d3 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete/script +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete/script @@ -3,8 +3,8 @@ envsubst < databricks.yml.tmpl > databricks.yml cleanup() { # Belt-and-braces in case bundle destroy was skipped or partially failed. # The soft-delete case leaves a record in the trash; --purge clears it. - $CLI postgres delete-project --purge "projects/test-pg-proj-hard-${UNIQUE_NAME}" 2>/dev/null || true - $CLI postgres delete-project --purge "projects/test-pg-proj-soft-${UNIQUE_NAME}" 2>/dev/null || true + $CLI postgres delete-project --purge "projects/test-pg-proj-hard-${UNIQUE_NAME}" 2>>LOG.delete-project || true + $CLI postgres delete-project --purge "projects/test-pg-proj-soft-${UNIQUE_NAME}" 2>>LOG.delete-project || true rm -f out.requests.txt } trap cleanup EXIT @@ -17,7 +17,7 @@ trace $CLI bundle deploy trace $CLI postgres get-project "projects/test-pg-proj-hard-${UNIQUE_NAME}" | jq 'del(.create_time, .update_time)' trace $CLI postgres get-project "projects/test-pg-proj-soft-${UNIQUE_NAME}" | jq 'del(.create_time, .update_time)' -trace print_requests.py --keep --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.json +trace print_requests.py --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.deploy.json # bundle destroy should send ?purge=true on the hard_delete project and no # purge query on the soft_delete project. Both DELETEs land in the recorded @@ -25,4 +25,4 @@ trace print_requests.py --keep --sort --get '//postgres' '^//workspace-files/' ' title "bundle destroy" $CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true -trace print_requests.py --keep --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.json +trace print_requests.py --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.json diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt index 88340b5dbe7..8094f11d175 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/output.txt @@ -7,7 +7,7 @@ Updating deployment state... Deployment complete! >>> get_purge -null +(unset) === Step 2: set purge_on_delete: true >>> [CLI] bundle deploy @@ -17,7 +17,7 @@ Updating deployment state... Deployment complete! >>> get_purge -true +json.state.resources.postgres_projects.proj.state.purge_on_delete = true; === Step 3: flip to purge_on_delete: false >>> [CLI] bundle deploy @@ -27,7 +27,7 @@ Updating deployment state... Deployment complete! >>> get_purge -false +json.state.resources.postgres_projects.proj.state.purge_on_delete = false; === Step 4: remove the line again >>> [CLI] bundle deploy @@ -37,7 +37,7 @@ Updating deployment state... Deployment complete! >>> get_purge -false +json.state.resources.postgres_projects.proj.state.purge_on_delete = false; === bundle destroy (must be a plain DELETE, no ?purge=true) ->>> print_requests.py --keep --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ +>>> print_requests.py --sort --get //postgres ^//workspace-files/ ^//workspace/ ^//telemetry-ext ^//operations/ diff --git a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script index 9851ef948d4..df85e1de8d5 100644 --- a/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script +++ b/acceptance/bundle/resources/postgres_projects/purge_on_delete_transitions/script @@ -3,16 +3,16 @@ envsubst < databricks.yml.tmpl > databricks.yml cleanup() { # After this test the destroy should be a soft delete; --purge here so the # trash entry doesn't linger past the test. - $CLI postgres delete-project --purge "projects/test-pg-proj-${UNIQUE_NAME}" 2>/dev/null || true + $CLI postgres delete-project --purge "projects/test-pg-proj-${UNIQUE_NAME}" 2>>LOG.delete-project || true rm -f out.requests.txt } trap cleanup EXIT -# Prints the persisted purge_on_delete value from the direct engine's +# Prints the persisted purge_on_delete assignment from the direct engine's # resources.json. The field is omitted from JSON when unset (omitempty + -# ForceSendFields tracking), so absent => null, explicit true/false => bool. +# ForceSendFields tracking), so an absent value falls back to "(unset)". get_purge() { - jq -r '.state["resources.postgres_projects.proj"].state.purge_on_delete' .databricks/bundle/default/resources.json + gron.py < .databricks/bundle/default/resources.json | grep purge_on_delete || echo '(unset)' } title "Step 1: deploy with purge_on_delete unset" @@ -38,4 +38,4 @@ rm -f out.requests.txt title "bundle destroy (must be a plain DELETE, no ?purge=true)" $CLI bundle destroy --auto-approve > out.destroy.txt 2>&1 || true -trace print_requests.py --keep --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.json +trace print_requests.py --sort --get '//postgres' '^//workspace-files/' '^//workspace/' '^//telemetry-ext' '^//operations/' > out.requests.destroy.json diff --git a/bundle/direct/dresources/postgres_project.go b/bundle/direct/dresources/postgres_project.go index 00bd1af412e..fc2ef631e38 100644 --- a/bundle/direct/dresources/postgres_project.go +++ b/bundle/direct/dresources/postgres_project.go @@ -139,10 +139,12 @@ func (r *ResourcePostgresProject) DoCreate(ctx context.Context, config *Postgres } func (r *ResourcePostgresProject) DoUpdate(ctx context.Context, id string, config *PostgresProjectState, entry *PlanEntry) (*PostgresProjectRemote, error) { - // Build update mask from fields that have action="update" in the changes map. - // This excludes immutable fields and fields that haven't changed. - // Prefix with "spec." because the API expects paths relative to the Project object, - // not relative to our flattened state type. + // Build the mask from the plan's change list and prefix with "spec." (the + // API expects paths relative to Project). The API rejects mask entries + // that aren't also populated in the request body, and a wildcard "*" + // expands to nested attributes the body would have to set too — so we + // can't use a static all-fields mask. The change list naturally tracks + // what the user actually set, so the body and mask stay consistent. fieldPaths := collectUpdatePathsWithPrefix(entry.Changes, "spec.") // purge_on_delete is an input-only flag consulted at delete time; it is diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 9c23ed0b413..ec47d50e709 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -488,8 +488,7 @@ resources: ignore_remote_changes: # purge_on_delete is a delete-time query parameter; not returned by GET. # When the user changes it locally we still want the new value to land - # in state (so the next destroy honors it); DoUpdate strips it from the - # API field mask, leaving toggling as a state-only refresh. + # in state so the next destroy honors it. - field: purge_on_delete reason: input_only