From d0361fed1b322b5542a670b974a21b1a25de036a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 23 Mar 2026 15:48:08 +0100 Subject: [PATCH 1/8] direct: catalog: add isolation_mode, enable_predictive_optimization, owner support Adds three update-only fields to the catalog resource: - isolation_mode (ISOLATED/OPEN) - enable_predictive_optimization (ENABLE/DISABLE/INHERIT) - owner Since these fields cannot be set during creation (not in CreateCatalog API), DoCreate now calls a post-create update when any of them are non-empty. DoUpdate/DoUpdateWithID are refactored to share an applyUpdate helper. Adds acceptance tests: - isolation-mode: verifies fields are applied on create and can be updated - no-update-on-create: verifies no PATCH is called when these fields are absent Co-authored-by: Isaac --- .../isolation-mode/databricks.yml.tmpl | 21 +++ .../catalogs/isolation-mode/out.test.toml | 6 + .../catalogs/isolation-mode/output.txt | 41 ++++++ .../resources/catalogs/isolation-mode/script | 24 ++++ .../catalogs/isolation-mode/test.toml | 12 ++ .../no-update-on-create/databricks.yml.tmpl | 19 +++ .../out.requests.direct.txt | 8 ++ .../no-update-on-create/out.requests.txt | 95 ++++++++++++ .../no-update-on-create/out.test.toml | 6 + .../catalogs/no-update-on-create/output.txt | 25 ++++ .../catalogs/no-update-on-create/script | 18 +++ .../catalogs/no-update-on-create/test.toml | 12 ++ bundle/config/resources/catalog.go | 5 + bundle/direct/dresources/all_test.go | 4 +- bundle/direct/dresources/catalog.go | 136 +++++++++++------- bundle/internal/schema/annotations.yml | 9 ++ bundle/schema/jsonschema.json | 15 ++ libs/testserver/catalogs.go | 6 + 18 files changed, 405 insertions(+), 57 deletions(-) create mode 100644 acceptance/bundle/resources/catalogs/isolation-mode/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs/isolation-mode/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs/isolation-mode/output.txt create mode 100644 acceptance/bundle/resources/catalogs/isolation-mode/script create mode 100644 acceptance/bundle/resources/catalogs/isolation-mode/test.toml create mode 100644 acceptance/bundle/resources/catalogs/no-update-on-create/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.direct.txt create mode 100644 acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.txt create mode 100644 acceptance/bundle/resources/catalogs/no-update-on-create/out.test.toml create mode 100644 acceptance/bundle/resources/catalogs/no-update-on-create/output.txt create mode 100644 acceptance/bundle/resources/catalogs/no-update-on-create/script create mode 100644 acceptance/bundle/resources/catalogs/no-update-on-create/test.toml diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/isolation-mode/databricks.yml.tmpl new file mode 100644 index 0000000000..601f371c50 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation-mode/databricks.yml.tmpl @@ -0,0 +1,21 @@ +bundle: + name: uc-catalog-isolation-mode-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +sync: + exclude: + - "out.*" + +resources: + catalogs: + test_catalog: + name: test_catalog_$UNIQUE_NAME + comment: "Catalog with isolation mode" + isolation_mode: ISOLATED + enable_predictive_optimization: ENABLE + +targets: + development: + default: true diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/out.test.toml b/acceptance/bundle/resources/catalogs/isolation-mode/out.test.toml new file mode 100644 index 0000000000..f1d40380d0 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation-mode/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/output.txt b/acceptance/bundle/resources/catalogs/isolation-mode/output.txt new file mode 100644 index 0000000000..9be284d991 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation-mode/output.txt @@ -0,0 +1,41 @@ + +=== Deploy catalog with isolation_mode and enable_predictive_optimization +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Assert update-only fields were applied +>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] +{ + "name": "test_catalog_[UNIQUE_NAME]", + "comment": "Catalog with isolation mode", + "isolation_mode": "ISOLATED", + "enable_predictive_optimization": "ENABLE" +} + +=== Update isolation_mode to OPEN +=== Redeploy with updated isolation_mode +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Assert isolation_mode is updated +>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] +{ + "name": "test_catalog_[UNIQUE_NAME]", + "isolation_mode": "OPEN" +} + +=== Test cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.test_catalog + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/script b/acceptance/bundle/resources/catalogs/isolation-mode/script new file mode 100644 index 0000000000..1394f02149 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation-mode/script @@ -0,0 +1,24 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +CATALOG_NAME="test_catalog_${UNIQUE_NAME}" + +cleanup() { + title "Test cleanup" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy catalog with isolation_mode and enable_predictive_optimization" +trace $CLI bundle deploy + +title "Assert update-only fields were applied" +trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment, isolation_mode, enable_predictive_optimization}" + +title "Update isolation_mode to OPEN" +update_file.py databricks.yml "isolation_mode: ISOLATED" "isolation_mode: OPEN" + +title "Redeploy with updated isolation_mode" +trace $CLI bundle deploy + +title "Assert isolation_mode is updated" +trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, isolation_mode}" diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/test.toml b/acceptance/bundle/resources/catalogs/isolation-mode/test.toml new file mode 100644 index 0000000000..d2b122411f --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation-mode/test.toml @@ -0,0 +1,12 @@ +Local = true +Cloud = true +RecordRequests = false +RequiresUnityCatalog = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/no-update-on-create/databricks.yml.tmpl new file mode 100644 index 0000000000..da3040d5b6 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: uc-catalog-no-update-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +sync: + exclude: + - "out.*" + +resources: + catalogs: + test_catalog: + name: test_catalog_$UNIQUE_NAME + comment: "Catalog without update-only fields" + +targets: + development: + default: true diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.direct.txt b/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.direct.txt new file mode 100644 index 0000000000..80e3755080 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.direct.txt @@ -0,0 +1,8 @@ +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "comment": "Catalog without update-only fields", + "name": "test_catalog_[UNIQUE_NAME]" + } +} diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.txt b/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.txt new file mode 100644 index 0000000000..e1e22907d4 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.txt @@ -0,0 +1,95 @@ +{ + "method": "GET", + "path": "/.well-known/databricks-config" +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]" +} +{ + "method": "GET", + "path": "/.well-known/databricks-config" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json" +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", + "q": { + "overwrite": "false" + }, + "body": { + "ID": "[UUID]", + "AcquisitionTime": "[TIMESTAMP]", + "IsForced": false, + "User": "[USERNAME]" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", + "return_export_info": "true" + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/export", + "q": { + "direct_download": "true", + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock" + } +} +{ + "method": "GET", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]" +} +{ + "method": "DELETE", + "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", + "q": { + "force": "true" + } +} +{ + "method": "POST", + "path": "/api/2.0/workspace/delete", + "body": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]", + "recursive": true + } +} +{ + "method": "GET", + "path": "/api/2.0/workspace/get-status", + "q": { + "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", + "return_export_info": "true" + } +} diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/out.test.toml b/acceptance/bundle/resources/catalogs/no-update-on-create/out.test.toml new file mode 100644 index 0000000000..f1d40380d0 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt b/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt new file mode 100644 index 0000000000..649702bd12 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt @@ -0,0 +1,25 @@ + +=== Deploy catalog without update-only fields +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Assert no PATCH was called during create +=== Assert catalog is created correctly +>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] +{ + "name": "test_catalog_[UNIQUE_NAME]", + "comment": "Catalog without update-only fields" +} + +=== Test cleanup +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.test_catalog + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/script b/acceptance/bundle/resources/catalogs/no-update-on-create/script new file mode 100644 index 0000000000..7b406c7a65 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +CATALOG_NAME="test_catalog_${UNIQUE_NAME}" + +cleanup() { + title "Test cleanup" + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy catalog without update-only fields" +trace $CLI bundle deploy + +title "Assert no PATCH was called during create" +print_requests.py //unity-catalog/catalogs --sort > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt + +title "Assert catalog is created correctly" +trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment}" diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/test.toml b/acceptance/bundle/resources/catalogs/no-update-on-create/test.toml new file mode 100644 index 0000000000..2b1a008639 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/test.toml @@ -0,0 +1,12 @@ +Local = true +Cloud = true +RecordRequests = true +RequiresUnityCatalog = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/config/resources/catalog.go b/bundle/config/resources/catalog.go index d558f127f5..44a7024cbc 100644 --- a/bundle/config/resources/catalog.go +++ b/bundle/config/resources/catalog.go @@ -17,6 +17,11 @@ type Catalog struct { BaseResource catalog.CreateCatalog + // Fields that can only be set via update, not during creation. + EnablePredictiveOptimization catalog.EnablePredictiveOptimization `json:"enable_predictive_optimization,omitempty"` + IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` + Owner string `json:"owner,omitempty"` + // List of grants to apply on this catalog. Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` } diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index da59fedbe0..c923588bab 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -44,8 +44,8 @@ var testConfig map[string]any = map[string]any{ Name: "mycatalog", Comment: "Test catalog", }, - // Note: EnablePredictiveOptimization and IsolationMode cannot be set during creation, - // only during updates. They are not included in the test config. + IsolationMode: catalog.CatalogIsolationModeIsolated, + EnablePredictiveOptimization: catalog.EnablePredictiveOptimizationEnable, }, "external_locations": &resources.ExternalLocation{ diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index bb1fb82490..dc545eaa58 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -6,9 +6,28 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// catalogState extends CreateCatalog with fields that can only be set via update. +type catalogState struct { + catalog.CreateCatalog + + EnablePredictiveOptimization catalog.EnablePredictiveOptimization `json:"enable_predictive_optimization,omitempty"` + IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` + Owner string `json:"owner,omitempty"` +} + +// Custom marshaling is required because CreateCatalog has custom marshaling that would otherwise swallow the extra fields. +func (s *catalogState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s catalogState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceCatalog struct { client *databricks.WorkspaceClient } @@ -17,21 +36,31 @@ func (*ResourceCatalog) New(client *databricks.WorkspaceClient) *ResourceCatalog return &ResourceCatalog{client: client} } -func (*ResourceCatalog) PrepareState(input *resources.Catalog) *catalog.CreateCatalog { - return &input.CreateCatalog +func (*ResourceCatalog) PrepareState(input *resources.Catalog) *catalogState { + return &catalogState{ + CreateCatalog: input.CreateCatalog, + EnablePredictiveOptimization: input.EnablePredictiveOptimization, + IsolationMode: input.IsolationMode, + Owner: input.Owner, + } } -func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *catalog.CreateCatalog { - return &catalog.CreateCatalog{ - Comment: info.Comment, - ConnectionName: info.ConnectionName, - Name: info.Name, - Options: info.Options, - Properties: info.Properties, - ProviderName: info.ProviderName, - ShareName: info.ShareName, - StorageRoot: info.StorageRoot, - ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), +func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *catalogState { + return &catalogState{ + CreateCatalog: catalog.CreateCatalog{ + Comment: info.Comment, + ConnectionName: info.ConnectionName, + Name: info.Name, + Options: info.Options, + Properties: info.Properties, + ProviderName: info.ProviderName, + ShareName: info.ShareName, + StorageRoot: info.StorageRoot, + ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), + }, + EnablePredictiveOptimization: info.EnablePredictiveOptimization, + IsolationMode: info.IsolationMode, + Owner: info.Owner, } } @@ -39,72 +68,69 @@ func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*catalog.Catal return r.client.Catalogs.GetByName(ctx, id) } -func (r *ResourceCatalog) DoCreate(ctx context.Context, config *catalog.CreateCatalog) (string, *catalog.CatalogInfo, error) { - response, err := r.client.Catalogs.Create(ctx, *config) +// DoCreate creates the catalog and applies update-only fields if set. +func (r *ResourceCatalog) DoCreate(ctx context.Context, config *catalogState) (string, *catalog.CatalogInfo, error) { + response, err := r.client.Catalogs.Create(ctx, config.CreateCatalog) if err != nil || response == nil { return "", nil, err } - return response.Name, response, nil -} -// DoUpdate updates the catalog in place and returns remote state. -func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *catalog.CreateCatalog, _ Changes) (*catalog.CatalogInfo, error) { - updateRequest := catalog.UpdateCatalog{ - Comment: config.Comment, - EnablePredictiveOptimization: "", // Not supported by DABs - IsolationMode: "", // Not supported by DABs - Name: id, - NewName: "", // Only set if name actually changes (see DoUpdateWithID) - Options: config.Options, - Owner: "", // Not supported by DABs - Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](config.ForceSendFields, "EnablePredictiveOptimization", "IsolationMode", "Owner"), + // IsolationMode, EnablePredictiveOptimization, and Owner cannot be set during creation; apply them via update. + if config.IsolationMode != "" || config.EnablePredictiveOptimization != "" || config.Owner != "" { + response, err = r.applyUpdate(ctx, response.Name, "", config) + if err != nil { + return "", nil, err + } } - response, err := r.client.Catalogs.Update(ctx, updateRequest) - if err != nil { - return nil, err - } + return response.Name, response, nil +} - return response, nil +// DoUpdate updates the catalog in place and returns remote state. +func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *catalogState, _ Changes) (*catalog.CatalogInfo, error) { + return r.applyUpdate(ctx, id, "", config) } // DoUpdateWithID updates the catalog and returns the new ID if the name changes. -func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateCatalog) (string, *catalog.CatalogInfo, error) { - updateRequest := catalog.UpdateCatalog{ - Comment: config.Comment, - EnablePredictiveOptimization: "", // Not supported by DABs - IsolationMode: "", // Not supported by DABs - Name: id, - NewName: "", // Initialized below if needed - Options: config.Options, - Owner: "", // Not supported by DABs - Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](config.ForceSendFields, "EnablePredictiveOptimization", "IsolationMode", "Owner"), - } - +func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *catalogState) (string, *catalog.CatalogInfo, error) { + newName := "" if config.Name != id { - updateRequest.NewName = config.Name + newName = config.Name } - response, err := r.client.Catalogs.Update(ctx, updateRequest) + response, err := r.applyUpdate(ctx, id, newName, config) if err != nil { return "", nil, err } - // Return the new name as the ID if it changed, otherwise return the old ID newID := id - if updateRequest.NewName != "" { - newID = updateRequest.NewName + if newName != "" { + newID = newName } return newID, response, nil } +// applyUpdate builds and sends an UpdateCatalog request. newName is set only when renaming. +func (r *ResourceCatalog) applyUpdate(ctx context.Context, id, newName string, config *catalogState) (*catalog.CatalogInfo, error) { + updateRequest := catalog.UpdateCatalog{ + Comment: config.Comment, + EnablePredictiveOptimization: config.EnablePredictiveOptimization, + IsolationMode: config.IsolationMode, + Name: id, + NewName: newName, + Options: config.Options, + Owner: config.Owner, + Properties: config.Properties, + ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](config.ForceSendFields), + } + + return r.client.Catalogs.Update(ctx, updateRequest) +} + func (r *ResourceCatalog) DoDelete(ctx context.Context, id string) error { return r.client.Catalogs.Delete(ctx, catalog.DeleteCatalogRequest{ - Name: id, - Force: true, - ForceSendFields: nil, + Name: id, + Force: true, }) } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 8dcf37f7e8..f873f7babf 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -562,12 +562,21 @@ github.com/databricks/cli/bundle/config/resources.Catalog: "connection_name": "description": |- PLACEHOLDER + "enable_predictive_optimization": + "description": |- + PLACEHOLDER + "isolation_mode": + "description": |- + PLACEHOLDER "name": "description": |- PLACEHOLDER "options": "description": |- PLACEHOLDER + "owner": + "description": |- + PLACEHOLDER "properties": "description": |- PLACEHOLDER diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index abfdb55233..8813390c3b 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -295,9 +295,15 @@ "connection_name": { "$ref": "#/$defs/string" }, + "enable_predictive_optimization": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.EnablePredictiveOptimization" + }, "grants": { "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment" }, + "isolation_mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.CatalogIsolationMode" + }, "lifecycle": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" }, @@ -307,6 +313,9 @@ "options": { "$ref": "#/$defs/map/string" }, + "owner": { + "$ref": "#/$defs/string" + }, "properties": { "$ref": "#/$defs/map/string" }, @@ -3570,6 +3579,12 @@ } ] }, + "catalog.CatalogIsolationMode": { + "type": "string" + }, + "catalog.EnablePredictiveOptimization": { + "type": "string" + }, "catalog.EncryptionDetails": { "oneOf": [ { diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 859721ee73..0358acfbae 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -69,6 +69,12 @@ func (s *FakeWorkspace) CatalogsUpdate(req Request, name string) Response { if updateRequest.Owner != "" { existing.Owner = updateRequest.Owner } + if updateRequest.IsolationMode != "" { + existing.IsolationMode = updateRequest.IsolationMode + } + if updateRequest.EnablePredictiveOptimization != "" { + existing.EnablePredictiveOptimization = updateRequest.EnablePredictiveOptimization + } if updateRequest.NewName != "" { existing.Name = updateRequest.NewName existing.FullName = updateRequest.NewName From 3f7fa4e48ff9befb8705ba2b303dc8e0250ec334 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 23 Mar 2026 15:56:27 +0100 Subject: [PATCH 2/8] direct: catalog: ignore owner drift; add isolation_mode invariant test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add owner to ignore_remote_changes for catalogs — the API automatically sets it to the creator's username, causing drift when the user doesn't specify it. This also fixes the pre-existing no_drift failure for catalog.yml.tmpl. Add catalog_isolation_mode.yml.tmpl to the invariant test matrix to verify that isolation_mode and enable_predictive_optimization produce no drift after a successful deploy. Co-authored-by: Isaac --- .../invariant/configs/catalog_isolation_mode.yml.tmpl | 10 ++++++++++ acceptance/bundle/invariant/test.toml | 1 + bundle/direct/dresources/resources.yml | 5 +++++ 3 files changed, 16 insertions(+) create mode 100644 acceptance/bundle/invariant/configs/catalog_isolation_mode.yml.tmpl diff --git a/acceptance/bundle/invariant/configs/catalog_isolation_mode.yml.tmpl b/acceptance/bundle/invariant/configs/catalog_isolation_mode.yml.tmpl new file mode 100644 index 0000000000..421ee6e367 --- /dev/null +++ b/acceptance/bundle/invariant/configs/catalog_isolation_mode.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + foo: + name: test-catalog-$UNIQUE_NAME + comment: This is a test catalog with isolation mode + isolation_mode: ISOLATED + enable_predictive_optimization: ENABLE diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 36badf07e5..0311acc165 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -24,6 +24,7 @@ EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", + "catalog_isolation_mode.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 733973ecf9..395f25e0c5 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -314,6 +314,11 @@ resources: update_id_on_changes: - field: name reason: id_changes + ignore_remote_changes: + # Owner is set to the creator by the API and changes to ownership are not + # managed by DABs unless explicitly set by the user. + - field: owner + reason: managed schemas: recreate_on_changes: From a136b29cfda40e27facfe80e59cc47f8a9a1cb9a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 23 Mar 2026 16:42:01 +0100 Subject: [PATCH 3/8] catalogs: Use backend_defaults for owner instead of ignore_remote_changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Consistent with registered_models.owner — backend_defaults only suppresses drift when the user has never set the field (old/new are nil), while ignore_remote_changes suppresses drift even when user has explicitly set owner and it gets changed externally. Also exclude catalog_isolation_mode.yml.tmpl from migrate invariant test since catalog resources only work with direct deployment mode (same reason catalog.yml.tmpl is already excluded). Co-authored-by: Isaac --- acceptance/bundle/invariant/continue_293/out.test.toml | 2 +- acceptance/bundle/invariant/migrate/out.test.toml | 2 +- acceptance/bundle/invariant/migrate/test.toml | 1 + acceptance/bundle/invariant/no_drift/out.test.toml | 2 +- bundle/direct/dresources/resources.yml | 7 +++---- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index ba32b1f599..1d21b64042 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "catalog_isolation_mode.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 725019505d..57e782e212 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "catalog_isolation_mode.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index a29f9dbb5a..0c92bee32f 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -1,5 +1,6 @@ # Error: Catalog resources are only supported with direct deployment mode EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"] +EnvMatrixExclude.no_catalog_isolation_mode = ["INPUT_CONFIG=catalog_isolation_mode.yml.tmpl"] EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"] # Unexpected action='create' for resources.secret_scopes.foo.permissions diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 725019505d..57e782e212 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "catalog_isolation_mode.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 395f25e0c5..52f4d25f32 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -314,11 +314,10 @@ resources: update_id_on_changes: - field: name reason: id_changes - ignore_remote_changes: - # Owner is set to the creator by the API and changes to ownership are not - # managed by DABs unless explicitly set by the user. + backend_defaults: + # Owner is set to the creator by the API when not specified by the user. + # Same as registered_models.owner — backend-set output field. - field: owner - reason: managed schemas: recreate_on_changes: From a451d66c27a1fb37e732f81e1f1653c747f73c8a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 23 Mar 2026 16:55:12 +0100 Subject: [PATCH 4/8] catalogs: no-update-on-create: embed requests in output.txt Since this test only runs on direct mode, there's no need for a per-engine filename. Use trace to embed the filtered catalog requests inline in output.txt. Clean up out.requests.txt in the cleanup trap in case the test fails before print_requests.py runs. Co-authored-by: Isaac --- .../out.requests.direct.txt | 8 -- .../no-update-on-create/out.requests.txt | 95 ------------------- .../catalogs/no-update-on-create/output.txt | 10 ++ .../catalogs/no-update-on-create/script | 3 +- 4 files changed, 12 insertions(+), 104 deletions(-) delete mode 100644 acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.direct.txt delete mode 100644 acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.txt diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.direct.txt b/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.direct.txt deleted file mode 100644 index 80e3755080..0000000000 --- a/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.direct.txt +++ /dev/null @@ -1,8 +0,0 @@ -{ - "method": "POST", - "path": "/api/2.1/unity-catalog/catalogs", - "body": { - "comment": "Catalog without update-only fields", - "name": "test_catalog_[UNIQUE_NAME]" - } -} diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.txt b/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.txt deleted file mode 100644 index e1e22907d4..0000000000 --- a/acceptance/bundle/resources/catalogs/no-update-on-create/out.requests.txt +++ /dev/null @@ -1,95 +0,0 @@ -{ - "method": "GET", - "path": "/.well-known/databricks-config" -} -{ - "method": "GET", - "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]" -} -{ - "method": "GET", - "path": "/.well-known/databricks-config" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/terraform.tfstate", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace-files/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/resources.json" -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", - "q": { - "overwrite": "false" - }, - "body": { - "ID": "[UUID]", - "AcquisitionTime": "[TIMESTAMP]", - "IsForced": false, - "User": "[USERNAME]" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", - "return_export_info": "true" - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/export", - "q": { - "direct_download": "true", - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock" - } -} -{ - "method": "GET", - "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]" -} -{ - "method": "DELETE", - "path": "/api/2.1/unity-catalog/catalogs/test_catalog_[UNIQUE_NAME]", - "q": { - "force": "true" - } -} -{ - "method": "POST", - "path": "/api/2.0/workspace/delete", - "body": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]", - "recursive": true - } -} -{ - "method": "GET", - "path": "/api/2.0/workspace/get-status", - "q": { - "path": "/Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/state/deploy.lock", - "return_export_info": "true" - } -} diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt b/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt index 649702bd12..51591b7c27 100644 --- a/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt @@ -7,6 +7,16 @@ Updating deployment state... Deployment complete! === Assert no PATCH was called during create +>>> print_requests.py //unity-catalog/catalogs --sort +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "comment": "Catalog without update-only fields", + "name": "test_catalog_[UNIQUE_NAME]" + } +} + === Assert catalog is created correctly >>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] { diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/script b/acceptance/bundle/resources/catalogs/no-update-on-create/script index 7b406c7a65..ff733eb3e8 100644 --- a/acceptance/bundle/resources/catalogs/no-update-on-create/script +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/script @@ -5,6 +5,7 @@ CATALOG_NAME="test_catalog_${UNIQUE_NAME}" cleanup() { title "Test cleanup" trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt } trap cleanup EXIT @@ -12,7 +13,7 @@ title "Deploy catalog without update-only fields" trace $CLI bundle deploy title "Assert no PATCH was called during create" -print_requests.py //unity-catalog/catalogs --sort > out.requests.$DATABRICKS_BUNDLE_ENGINE.txt +trace print_requests.py //unity-catalog/catalogs --sort title "Assert catalog is created correctly" trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment}" From e798af7e19373684b73434a4e9884e797df6871a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 23 Mar 2026 17:01:20 +0100 Subject: [PATCH 5/8] catalogs: remove unnecessary title from cleanup functions Co-authored-by: Isaac --- acceptance/bundle/resources/catalogs/isolation-mode/output.txt | 1 - acceptance/bundle/resources/catalogs/isolation-mode/script | 1 - .../bundle/resources/catalogs/no-update-on-create/output.txt | 1 - acceptance/bundle/resources/catalogs/no-update-on-create/script | 1 - 4 files changed, 4 deletions(-) diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/output.txt b/acceptance/bundle/resources/catalogs/isolation-mode/output.txt index 9be284d991..e6fad3f51f 100644 --- a/acceptance/bundle/resources/catalogs/isolation-mode/output.txt +++ b/acceptance/bundle/resources/catalogs/isolation-mode/output.txt @@ -30,7 +30,6 @@ Deployment complete! "isolation_mode": "OPEN" } -=== Test cleanup >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.catalogs.test_catalog diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/script b/acceptance/bundle/resources/catalogs/isolation-mode/script index 1394f02149..276de1384f 100644 --- a/acceptance/bundle/resources/catalogs/isolation-mode/script +++ b/acceptance/bundle/resources/catalogs/isolation-mode/script @@ -3,7 +3,6 @@ envsubst < databricks.yml.tmpl > databricks.yml CATALOG_NAME="test_catalog_${UNIQUE_NAME}" cleanup() { - title "Test cleanup" trace $CLI bundle destroy --auto-approve } trap cleanup EXIT diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt b/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt index 51591b7c27..427e79ec66 100644 --- a/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt @@ -24,7 +24,6 @@ Deployment complete! "comment": "Catalog without update-only fields" } -=== Test cleanup >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete resources.catalogs.test_catalog diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/script b/acceptance/bundle/resources/catalogs/no-update-on-create/script index ff733eb3e8..5246365ff6 100644 --- a/acceptance/bundle/resources/catalogs/no-update-on-create/script +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/script @@ -3,7 +3,6 @@ envsubst < databricks.yml.tmpl > databricks.yml CATALOG_NAME="test_catalog_${UNIQUE_NAME}" cleanup() { - title "Test cleanup" trace $CLI bundle destroy --auto-approve rm -f out.requests.txt } From 2c5ef018fa2ebc61f5dc6772a5412361fc83f5d6 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 23 Mar 2026 17:08:49 +0100 Subject: [PATCH 6/8] direct: catalog: add ForceSendFields to DeleteCatalogRequest (exhaustruct) Co-authored-by: Isaac --- bundle/direct/dresources/catalog.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index dc545eaa58..03f1c166f7 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -130,7 +130,8 @@ func (r *ResourceCatalog) applyUpdate(ctx context.Context, id, newName string, c func (r *ResourceCatalog) DoDelete(ctx context.Context, id string) error { return r.client.Catalogs.Delete(ctx, catalog.DeleteCatalogRequest{ - Name: id, - Force: true, + Name: id, + Force: true, + ForceSendFields: nil, }) } From 3f8c004748382e6a2d558c0d22c0c0287109529f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 23 Mar 2026 20:58:41 +0100 Subject: [PATCH 7/8] test-update --- acceptance/bundle/refschema/out.fields.txt | 6 +++--- .../bundle/resources/grants/catalogs/out.plan2.direct.json | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d032aa48f8..5a51ea644b 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -227,10 +227,10 @@ resources.catalogs.*.effective_predictive_optimization_flag *catalog.EffectivePr resources.catalogs.*.effective_predictive_optimization_flag.inherited_from_name string REMOTE resources.catalogs.*.effective_predictive_optimization_flag.inherited_from_type catalog.EffectivePredictiveOptimizationFlagInheritedFromType REMOTE resources.catalogs.*.effective_predictive_optimization_flag.value catalog.EnablePredictiveOptimization REMOTE -resources.catalogs.*.enable_predictive_optimization catalog.EnablePredictiveOptimization REMOTE +resources.catalogs.*.enable_predictive_optimization catalog.EnablePredictiveOptimization ALL resources.catalogs.*.full_name string REMOTE resources.catalogs.*.id string INPUT -resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode REMOTE +resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode ALL resources.catalogs.*.lifecycle resources.Lifecycle INPUT resources.catalogs.*.lifecycle.prevent_destroy bool INPUT resources.catalogs.*.metastore_id string REMOTE @@ -238,7 +238,7 @@ resources.catalogs.*.modified_status string INPUT resources.catalogs.*.name string ALL resources.catalogs.*.options map[string]string ALL resources.catalogs.*.options.* string ALL -resources.catalogs.*.owner string REMOTE +resources.catalogs.*.owner string ALL resources.catalogs.*.properties map[string]string ALL resources.catalogs.*.properties.* string ALL resources.catalogs.*.provider_name string ALL diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json index 33644c3f88..b5c2089881 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json @@ -12,6 +12,13 @@ "full_name": "catalog_grants_[UNIQUE_NAME]", "name": "catalog_grants_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "owner": { + "action": "skip", + "reason": "backend_default", + "remote": "[USERNAME]" + } } }, "resources.catalogs.grants_catalog.grants": { From 20a57534d43f57c38fda865341d5ae2a455cd5e7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 24 Mar 2026 08:55:13 +0100 Subject: [PATCH 8/8] regenerate --- .../direct/dresources/apitypes.generated.yml | 2 +- .../internal/schema/annotations_openapi.yml | 16 ++++++++++ .../schema/annotations_openapi_overrides.yml | 9 ++++++ .../validation/generated/enum_fields.go | 4 ++- bundle/schema/jsonschema.json | 29 +++++++++++++++++-- bundle/schema/jsonschema_for_docs.json | 24 +++++++++++++++ .../databricks/bundles/catalogs/__init__.py | 12 ++++++++ .../bundles/catalogs/_models/catalog.py | 24 +++++++++++++++ .../_models/catalog_isolation_mode.py | 10 +++++++ .../_models/enable_predictive_optimization.py | 13 +++++++++ 10 files changed, 139 insertions(+), 4 deletions(-) create mode 100644 python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py create mode 100644 python/databricks/bundles/catalogs/_models/enable_predictive_optimization.py diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 8dfabd1098..82c2f863ea 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -4,7 +4,7 @@ alerts: sql.AlertV2 apps: apps.App -catalogs: catalog.CreateCatalog +catalogs: catalog.CatalogInfo clusters: compute.ClusterSpec diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index c196c7799a..6b8e95dcb5 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -1575,6 +1575,22 @@ github.com/databricks/databricks-sdk-go/service/catalog.AzureQueueStorage: Optional subscription id for the queue, event grid subscription, and external location storage account. Required for locations with a service principal storage credential +github.com/databricks/databricks-sdk-go/service/catalog.CatalogIsolationMode: + "_": + "enum": + - |- + OPEN + - |- + ISOLATED +github.com/databricks/databricks-sdk-go/service/catalog.EnablePredictiveOptimization: + "_": + "enum": + - |- + DISABLE + - |- + ENABLE + - |- + INHERIT github.com/databricks/databricks-sdk-go/service/catalog.EncryptionDetails: "_": "description": |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 611289083e..bb6fd09fe1 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -73,12 +73,21 @@ github.com/databricks/cli/bundle/config/resources.App: "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Catalog: + "enable_predictive_optimization": + "description": |- + PLACEHOLDER "grants": "description": |- PLACEHOLDER + "isolation_mode": + "description": |- + PLACEHOLDER "lifecycle": "description": |- PLACEHOLDER + "owner": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Cluster: "_": "markdown_description": |- diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index c1e098ed80..f02c6eebaa 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -38,7 +38,9 @@ var EnumFields = map[string][]string{ "resources.apps.*.resources[*].uc_securable.permission": {"EXECUTE", "MODIFY", "READ_VOLUME", "SELECT", "USE_CONNECTION", "WRITE_VOLUME"}, "resources.apps.*.resources[*].uc_securable.securable_type": {"CONNECTION", "FUNCTION", "TABLE", "VOLUME"}, - "resources.catalogs.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, + "resources.catalogs.*.enable_predictive_optimization": {"DISABLE", "ENABLE", "INHERIT"}, + "resources.catalogs.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, + "resources.catalogs.*.isolation_mode": {"ISOLATED", "OPEN"}, "resources.clusters.*.aws_attributes.availability": {"ON_DEMAND", "SPOT", "SPOT_WITH_FALLBACK"}, "resources.clusters.*.aws_attributes.ebs_volume_type": {"GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 8813390c3b..8e49a1453f 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -3580,10 +3580,35 @@ ] }, "catalog.CatalogIsolationMode": { - "type": "string" + "oneOf": [ + { + "type": "string", + "enum": [ + "OPEN", + "ISOLATED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] }, "catalog.EnablePredictiveOptimization": { - "type": "string" + "oneOf": [ + { + "type": "string", + "enum": [ + "DISABLE", + "ENABLE", + "INHERIT" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] }, "catalog.EncryptionDetails": { "oneOf": [ diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index bcb6866296..04350fb08f 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -236,10 +236,16 @@ "$ref": "#/$defs/string", "x-since-version": "v0.287.0" }, + "enable_predictive_optimization": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.EnablePredictiveOptimization" + }, "grants": { "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment", "x-since-version": "v0.287.0" }, + "isolation_mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.CatalogIsolationMode" + }, "lifecycle": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle", "x-since-version": "v0.287.0" @@ -252,6 +258,9 @@ "$ref": "#/$defs/map/string", "x-since-version": "v0.287.0" }, + "owner": { + "$ref": "#/$defs/string" + }, "properties": { "$ref": "#/$defs/map/string", "x-since-version": "v0.287.0" @@ -3286,6 +3295,21 @@ }, "additionalProperties": false }, + "catalog.CatalogIsolationMode": { + "type": "string", + "enum": [ + "OPEN", + "ISOLATED" + ] + }, + "catalog.EnablePredictiveOptimization": { + "type": "string", + "enum": [ + "DISABLE", + "ENABLE", + "INHERIT" + ] + }, "catalog.EncryptionDetails": { "type": "object", "description": "Encryption options that apply to clients connecting to cloud storage.", diff --git a/python/databricks/bundles/catalogs/__init__.py b/python/databricks/bundles/catalogs/__init__.py index fc92f42e22..b4b5ca6e00 100644 --- a/python/databricks/bundles/catalogs/__init__.py +++ b/python/databricks/bundles/catalogs/__init__.py @@ -6,7 +6,11 @@ "CatalogGrantParam", "CatalogGrantPrivilege", "CatalogGrantPrivilegeParam", + "CatalogIsolationMode", + "CatalogIsolationModeParam", "CatalogParam", + "EnablePredictiveOptimization", + "EnablePredictiveOptimizationParam", "Lifecycle", "LifecycleDict", "LifecycleParam", @@ -23,6 +27,14 @@ CatalogDict, CatalogParam, ) +from databricks.bundles.catalogs._models.catalog_isolation_mode import ( + CatalogIsolationMode, + CatalogIsolationModeParam, +) +from databricks.bundles.catalogs._models.enable_predictive_optimization import ( + EnablePredictiveOptimization, + EnablePredictiveOptimizationParam, +) from databricks.bundles.catalogs._models.lifecycle import ( Lifecycle, LifecycleDict, diff --git a/python/databricks/bundles/catalogs/_models/catalog.py b/python/databricks/bundles/catalogs/_models/catalog.py index 368e96a3b2..b1926c8f1d 100644 --- a/python/databricks/bundles/catalogs/_models/catalog.py +++ b/python/databricks/bundles/catalogs/_models/catalog.py @@ -1,6 +1,14 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, TypedDict +from databricks.bundles.catalogs._models.catalog_isolation_mode import ( + CatalogIsolationMode, + CatalogIsolationModeParam, +) +from databricks.bundles.catalogs._models.enable_predictive_optimization import ( + EnablePredictiveOptimization, + EnablePredictiveOptimizationParam, +) from databricks.bundles.catalogs._models.lifecycle import Lifecycle, LifecycleParam from databricks.bundles.catalogs._models.privilege_assignment import ( PrivilegeAssignment, @@ -30,12 +38,20 @@ class Catalog(Resource): connection_name: VariableOrOptional[str] = None + enable_predictive_optimization: VariableOrOptional[EnablePredictiveOptimization] = ( + None + ) + grants: VariableOrList[PrivilegeAssignment] = field(default_factory=list) + isolation_mode: VariableOrOptional[CatalogIsolationMode] = None + lifecycle: VariableOrOptional[Lifecycle] = None options: VariableOrDict[str] = field(default_factory=dict) + owner: VariableOrOptional[str] = None + properties: VariableOrDict[str] = field(default_factory=dict) provider_name: VariableOrOptional[str] = None @@ -61,12 +77,20 @@ class CatalogDict(TypedDict, total=False): connection_name: VariableOrOptional[str] + enable_predictive_optimization: VariableOrOptional[ + EnablePredictiveOptimizationParam + ] + grants: VariableOrList[PrivilegeAssignmentParam] + isolation_mode: VariableOrOptional[CatalogIsolationModeParam] + lifecycle: VariableOrOptional[LifecycleParam] options: VariableOrDict[str] + owner: VariableOrOptional[str] + properties: VariableOrDict[str] provider_name: VariableOrOptional[str] diff --git a/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py b/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py new file mode 100644 index 0000000000..b41cf7de69 --- /dev/null +++ b/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py @@ -0,0 +1,10 @@ +from enum import Enum +from typing import Literal + + +class CatalogIsolationMode(Enum): + OPEN = "OPEN" + ISOLATED = "ISOLATED" + + +CatalogIsolationModeParam = Literal["OPEN", "ISOLATED"] | CatalogIsolationMode diff --git a/python/databricks/bundles/catalogs/_models/enable_predictive_optimization.py b/python/databricks/bundles/catalogs/_models/enable_predictive_optimization.py new file mode 100644 index 0000000000..f4ca1e5b58 --- /dev/null +++ b/python/databricks/bundles/catalogs/_models/enable_predictive_optimization.py @@ -0,0 +1,13 @@ +from enum import Enum +from typing import Literal + + +class EnablePredictiveOptimization(Enum): + DISABLE = "DISABLE" + ENABLE = "ENABLE" + INHERIT = "INHERIT" + + +EnablePredictiveOptimizationParam = ( + Literal["DISABLE", "ENABLE", "INHERIT"] | EnablePredictiveOptimization +)