diff --git a/acceptance/bundle/invariant/configs/catalog_isolation_mode.yml.tmpl b/acceptance/bundle/invariant/configs/catalog_isolation_mode.yml.tmpl new file mode 100644 index 0000000000..421ee6e367 --- /dev/null +++ b/acceptance/bundle/invariant/configs/catalog_isolation_mode.yml.tmpl @@ -0,0 +1,10 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + catalogs: + foo: + name: test-catalog-$UNIQUE_NAME + comment: This is a test catalog with isolation mode + isolation_mode: ISOLATED + enable_predictive_optimization: ENABLE diff --git a/acceptance/bundle/invariant/continue_293/out.test.toml b/acceptance/bundle/invariant/continue_293/out.test.toml index ba32b1f599..1d21b64042 100644 --- a/acceptance/bundle/invariant/continue_293/out.test.toml +++ b/acceptance/bundle/invariant/continue_293/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "catalog_isolation_mode.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/out.test.toml b/acceptance/bundle/invariant/migrate/out.test.toml index 725019505d..57e782e212 100644 --- a/acceptance/bundle/invariant/migrate/out.test.toml +++ b/acceptance/bundle/invariant/migrate/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "catalog_isolation_mode.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/migrate/test.toml b/acceptance/bundle/invariant/migrate/test.toml index a29f9dbb5a..0c92bee32f 100644 --- a/acceptance/bundle/invariant/migrate/test.toml +++ b/acceptance/bundle/invariant/migrate/test.toml @@ -1,5 +1,6 @@ # Error: Catalog resources are only supported with direct deployment mode EnvMatrixExclude.no_catalog = ["INPUT_CONFIG=catalog.yml.tmpl"] +EnvMatrixExclude.no_catalog_isolation_mode = ["INPUT_CONFIG=catalog_isolation_mode.yml.tmpl"] EnvMatrixExclude.no_external_location = ["INPUT_CONFIG=external_location.yml.tmpl"] # Unexpected action='create' for resources.secret_scopes.foo.permissions diff --git a/acceptance/bundle/invariant/no_drift/out.test.toml b/acceptance/bundle/invariant/no_drift/out.test.toml index 725019505d..57e782e212 100644 --- a/acceptance/bundle/invariant/no_drift/out.test.toml +++ b/acceptance/bundle/invariant/no_drift/out.test.toml @@ -4,4 +4,4 @@ RequiresUnityCatalog = true [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["direct"] - INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] + INPUT_CONFIG = ["alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", "catalog_isolation_mode.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", "database_instance.yml.tmpl", "experiment.yml.tmpl", "external_location.yml.tmpl", "job.yml.tmpl", "job_pydabs_10_tasks.yml.tmpl", "job_pydabs_1000_tasks.yml.tmpl", "job_cross_resource_ref.yml.tmpl", "job_permission_ref.yml.tmpl", "job_with_permissions.yml.tmpl", "job_with_task.yml.tmpl", "model.yml.tmpl", "model_serving_endpoint.yml.tmpl", "pipeline.yml.tmpl", "postgres_branch.yml.tmpl", "postgres_endpoint.yml.tmpl", "postgres_project.yml.tmpl", "registered_model.yml.tmpl", "schema.yml.tmpl", "schema_grant_ref.yml.tmpl", "schema_with_grants.yml.tmpl", "secret_scope.yml.tmpl", "synced_database_table.yml.tmpl", "volume.yml.tmpl"] diff --git a/acceptance/bundle/invariant/test.toml b/acceptance/bundle/invariant/test.toml index 36badf07e5..0311acc165 100644 --- a/acceptance/bundle/invariant/test.toml +++ b/acceptance/bundle/invariant/test.toml @@ -24,6 +24,7 @@ EnvMatrix.INPUT_CONFIG = [ "alert.yml.tmpl", "app.yml.tmpl", "catalog.yml.tmpl", + "catalog_isolation_mode.yml.tmpl", "cluster.yml.tmpl", "dashboard.yml.tmpl", "database_catalog.yml.tmpl", diff --git a/acceptance/bundle/refschema/out.fields.txt b/acceptance/bundle/refschema/out.fields.txt index d032aa48f8..5a51ea644b 100644 --- a/acceptance/bundle/refschema/out.fields.txt +++ b/acceptance/bundle/refschema/out.fields.txt @@ -227,10 +227,10 @@ resources.catalogs.*.effective_predictive_optimization_flag *catalog.EffectivePr resources.catalogs.*.effective_predictive_optimization_flag.inherited_from_name string REMOTE resources.catalogs.*.effective_predictive_optimization_flag.inherited_from_type catalog.EffectivePredictiveOptimizationFlagInheritedFromType REMOTE resources.catalogs.*.effective_predictive_optimization_flag.value catalog.EnablePredictiveOptimization REMOTE -resources.catalogs.*.enable_predictive_optimization catalog.EnablePredictiveOptimization REMOTE +resources.catalogs.*.enable_predictive_optimization catalog.EnablePredictiveOptimization ALL resources.catalogs.*.full_name string REMOTE resources.catalogs.*.id string INPUT -resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode REMOTE +resources.catalogs.*.isolation_mode catalog.CatalogIsolationMode ALL resources.catalogs.*.lifecycle resources.Lifecycle INPUT resources.catalogs.*.lifecycle.prevent_destroy bool INPUT resources.catalogs.*.metastore_id string REMOTE @@ -238,7 +238,7 @@ resources.catalogs.*.modified_status string INPUT resources.catalogs.*.name string ALL resources.catalogs.*.options map[string]string ALL resources.catalogs.*.options.* string ALL -resources.catalogs.*.owner string REMOTE +resources.catalogs.*.owner string ALL resources.catalogs.*.properties map[string]string ALL resources.catalogs.*.properties.* string ALL resources.catalogs.*.provider_name string ALL diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/isolation-mode/databricks.yml.tmpl new file mode 100644 index 0000000000..601f371c50 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation-mode/databricks.yml.tmpl @@ -0,0 +1,21 @@ +bundle: + name: uc-catalog-isolation-mode-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +sync: + exclude: + - "out.*" + +resources: + catalogs: + test_catalog: + name: test_catalog_$UNIQUE_NAME + comment: "Catalog with isolation mode" + isolation_mode: ISOLATED + enable_predictive_optimization: ENABLE + +targets: + development: + default: true diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/out.test.toml b/acceptance/bundle/resources/catalogs/isolation-mode/out.test.toml new file mode 100644 index 0000000000..f1d40380d0 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation-mode/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/output.txt b/acceptance/bundle/resources/catalogs/isolation-mode/output.txt new file mode 100644 index 0000000000..e6fad3f51f --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation-mode/output.txt @@ -0,0 +1,40 @@ + +=== Deploy catalog with isolation_mode and enable_predictive_optimization +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Assert update-only fields were applied +>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] +{ + "name": "test_catalog_[UNIQUE_NAME]", + "comment": "Catalog with isolation mode", + "isolation_mode": "ISOLATED", + "enable_predictive_optimization": "ENABLE" +} + +=== Update isolation_mode to OPEN +=== Redeploy with updated isolation_mode +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Assert isolation_mode is updated +>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] +{ + "name": "test_catalog_[UNIQUE_NAME]", + "isolation_mode": "OPEN" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.test_catalog + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/script b/acceptance/bundle/resources/catalogs/isolation-mode/script new file mode 100644 index 0000000000..276de1384f --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation-mode/script @@ -0,0 +1,23 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +CATALOG_NAME="test_catalog_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve +} +trap cleanup EXIT + +title "Deploy catalog with isolation_mode and enable_predictive_optimization" +trace $CLI bundle deploy + +title "Assert update-only fields were applied" +trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment, isolation_mode, enable_predictive_optimization}" + +title "Update isolation_mode to OPEN" +update_file.py databricks.yml "isolation_mode: ISOLATED" "isolation_mode: OPEN" + +title "Redeploy with updated isolation_mode" +trace $CLI bundle deploy + +title "Assert isolation_mode is updated" +trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, isolation_mode}" diff --git a/acceptance/bundle/resources/catalogs/isolation-mode/test.toml b/acceptance/bundle/resources/catalogs/isolation-mode/test.toml new file mode 100644 index 0000000000..d2b122411f --- /dev/null +++ b/acceptance/bundle/resources/catalogs/isolation-mode/test.toml @@ -0,0 +1,12 @@ +Local = true +Cloud = true +RecordRequests = false +RequiresUnityCatalog = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/databricks.yml.tmpl b/acceptance/bundle/resources/catalogs/no-update-on-create/databricks.yml.tmpl new file mode 100644 index 0000000000..da3040d5b6 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/databricks.yml.tmpl @@ -0,0 +1,19 @@ +bundle: + name: uc-catalog-no-update-$UNIQUE_NAME + +workspace: + root_path: ~/.bundle/$UNIQUE_NAME + +sync: + exclude: + - "out.*" + +resources: + catalogs: + test_catalog: + name: test_catalog_$UNIQUE_NAME + comment: "Catalog without update-only fields" + +targets: + development: + default: true diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/out.test.toml b/acceptance/bundle/resources/catalogs/no-update-on-create/out.test.toml new file mode 100644 index 0000000000..f1d40380d0 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt b/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt new file mode 100644 index 0000000000..427e79ec66 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/output.txt @@ -0,0 +1,34 @@ + +=== Deploy catalog without update-only fields +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Assert no PATCH was called during create +>>> print_requests.py //unity-catalog/catalogs --sort +{ + "method": "POST", + "path": "/api/2.1/unity-catalog/catalogs", + "body": { + "comment": "Catalog without update-only fields", + "name": "test_catalog_[UNIQUE_NAME]" + } +} + +=== Assert catalog is created correctly +>>> [CLI] catalogs get test_catalog_[UNIQUE_NAME] +{ + "name": "test_catalog_[UNIQUE_NAME]", + "comment": "Catalog without update-only fields" +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.catalogs.test_catalog + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/script b/acceptance/bundle/resources/catalogs/no-update-on-create/script new file mode 100644 index 0000000000..5246365ff6 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/script @@ -0,0 +1,18 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +CATALOG_NAME="test_catalog_${UNIQUE_NAME}" + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT + +title "Deploy catalog without update-only fields" +trace $CLI bundle deploy + +title "Assert no PATCH was called during create" +trace print_requests.py //unity-catalog/catalogs --sort + +title "Assert catalog is created correctly" +trace $CLI catalogs get "${CATALOG_NAME}" | jq "{name, comment}" diff --git a/acceptance/bundle/resources/catalogs/no-update-on-create/test.toml b/acceptance/bundle/resources/catalogs/no-update-on-create/test.toml new file mode 100644 index 0000000000..2b1a008639 --- /dev/null +++ b/acceptance/bundle/resources/catalogs/no-update-on-create/test.toml @@ -0,0 +1,12 @@ +Local = true +Cloud = true +RecordRequests = true +RequiresUnityCatalog = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json index 33644c3f88..b5c2089881 100644 --- a/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json +++ b/acceptance/bundle/resources/grants/catalogs/out.plan2.direct.json @@ -12,6 +12,13 @@ "full_name": "catalog_grants_[UNIQUE_NAME]", "name": "catalog_grants_[UNIQUE_NAME]", "owner": "[USERNAME]" + }, + "changes": { + "owner": { + "action": "skip", + "reason": "backend_default", + "remote": "[USERNAME]" + } } }, "resources.catalogs.grants_catalog.grants": { diff --git a/bundle/config/resources/catalog.go b/bundle/config/resources/catalog.go index d558f127f5..44a7024cbc 100644 --- a/bundle/config/resources/catalog.go +++ b/bundle/config/resources/catalog.go @@ -17,6 +17,11 @@ type Catalog struct { BaseResource catalog.CreateCatalog + // Fields that can only be set via update, not during creation. + EnablePredictiveOptimization catalog.EnablePredictiveOptimization `json:"enable_predictive_optimization,omitempty"` + IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` + Owner string `json:"owner,omitempty"` + // List of grants to apply on this catalog. Grants []catalog.PrivilegeAssignment `json:"grants,omitempty"` } diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index da59fedbe0..c923588bab 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -44,8 +44,8 @@ var testConfig map[string]any = map[string]any{ Name: "mycatalog", Comment: "Test catalog", }, - // Note: EnablePredictiveOptimization and IsolationMode cannot be set during creation, - // only during updates. They are not included in the test config. + IsolationMode: catalog.CatalogIsolationModeIsolated, + EnablePredictiveOptimization: catalog.EnablePredictiveOptimizationEnable, }, "external_locations": &resources.ExternalLocation{ diff --git a/bundle/direct/dresources/apitypes.generated.yml b/bundle/direct/dresources/apitypes.generated.yml index 8dfabd1098..82c2f863ea 100644 --- a/bundle/direct/dresources/apitypes.generated.yml +++ b/bundle/direct/dresources/apitypes.generated.yml @@ -4,7 +4,7 @@ alerts: sql.AlertV2 apps: apps.App -catalogs: catalog.CreateCatalog +catalogs: catalog.CatalogInfo clusters: compute.ClusterSpec diff --git a/bundle/direct/dresources/catalog.go b/bundle/direct/dresources/catalog.go index bb1fb82490..03f1c166f7 100644 --- a/bundle/direct/dresources/catalog.go +++ b/bundle/direct/dresources/catalog.go @@ -6,9 +6,28 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/marshal" "github.com/databricks/databricks-sdk-go/service/catalog" ) +// catalogState extends CreateCatalog with fields that can only be set via update. +type catalogState struct { + catalog.CreateCatalog + + EnablePredictiveOptimization catalog.EnablePredictiveOptimization `json:"enable_predictive_optimization,omitempty"` + IsolationMode catalog.CatalogIsolationMode `json:"isolation_mode,omitempty"` + Owner string `json:"owner,omitempty"` +} + +// Custom marshaling is required because CreateCatalog has custom marshaling that would otherwise swallow the extra fields. +func (s *catalogState) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s catalogState) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + type ResourceCatalog struct { client *databricks.WorkspaceClient } @@ -17,21 +36,31 @@ func (*ResourceCatalog) New(client *databricks.WorkspaceClient) *ResourceCatalog return &ResourceCatalog{client: client} } -func (*ResourceCatalog) PrepareState(input *resources.Catalog) *catalog.CreateCatalog { - return &input.CreateCatalog +func (*ResourceCatalog) PrepareState(input *resources.Catalog) *catalogState { + return &catalogState{ + CreateCatalog: input.CreateCatalog, + EnablePredictiveOptimization: input.EnablePredictiveOptimization, + IsolationMode: input.IsolationMode, + Owner: input.Owner, + } } -func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *catalog.CreateCatalog { - return &catalog.CreateCatalog{ - Comment: info.Comment, - ConnectionName: info.ConnectionName, - Name: info.Name, - Options: info.Options, - Properties: info.Properties, - ProviderName: info.ProviderName, - ShareName: info.ShareName, - StorageRoot: info.StorageRoot, - ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), +func (*ResourceCatalog) RemapState(info *catalog.CatalogInfo) *catalogState { + return &catalogState{ + CreateCatalog: catalog.CreateCatalog{ + Comment: info.Comment, + ConnectionName: info.ConnectionName, + Name: info.Name, + Options: info.Options, + Properties: info.Properties, + ProviderName: info.ProviderName, + ShareName: info.ShareName, + StorageRoot: info.StorageRoot, + ForceSendFields: utils.FilterFields[catalog.CreateCatalog](info.ForceSendFields), + }, + EnablePredictiveOptimization: info.EnablePredictiveOptimization, + IsolationMode: info.IsolationMode, + Owner: info.Owner, } } @@ -39,68 +68,66 @@ func (r *ResourceCatalog) DoRead(ctx context.Context, id string) (*catalog.Catal return r.client.Catalogs.GetByName(ctx, id) } -func (r *ResourceCatalog) DoCreate(ctx context.Context, config *catalog.CreateCatalog) (string, *catalog.CatalogInfo, error) { - response, err := r.client.Catalogs.Create(ctx, *config) +// DoCreate creates the catalog and applies update-only fields if set. +func (r *ResourceCatalog) DoCreate(ctx context.Context, config *catalogState) (string, *catalog.CatalogInfo, error) { + response, err := r.client.Catalogs.Create(ctx, config.CreateCatalog) if err != nil || response == nil { return "", nil, err } - return response.Name, response, nil -} -// DoUpdate updates the catalog in place and returns remote state. -func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *catalog.CreateCatalog, _ Changes) (*catalog.CatalogInfo, error) { - updateRequest := catalog.UpdateCatalog{ - Comment: config.Comment, - EnablePredictiveOptimization: "", // Not supported by DABs - IsolationMode: "", // Not supported by DABs - Name: id, - NewName: "", // Only set if name actually changes (see DoUpdateWithID) - Options: config.Options, - Owner: "", // Not supported by DABs - Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](config.ForceSendFields, "EnablePredictiveOptimization", "IsolationMode", "Owner"), + // IsolationMode, EnablePredictiveOptimization, and Owner cannot be set during creation; apply them via update. + if config.IsolationMode != "" || config.EnablePredictiveOptimization != "" || config.Owner != "" { + response, err = r.applyUpdate(ctx, response.Name, "", config) + if err != nil { + return "", nil, err + } } - response, err := r.client.Catalogs.Update(ctx, updateRequest) - if err != nil { - return nil, err - } + return response.Name, response, nil +} - return response, nil +// DoUpdate updates the catalog in place and returns remote state. +func (r *ResourceCatalog) DoUpdate(ctx context.Context, id string, config *catalogState, _ Changes) (*catalog.CatalogInfo, error) { + return r.applyUpdate(ctx, id, "", config) } // DoUpdateWithID updates the catalog and returns the new ID if the name changes. -func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *catalog.CreateCatalog) (string, *catalog.CatalogInfo, error) { - updateRequest := catalog.UpdateCatalog{ - Comment: config.Comment, - EnablePredictiveOptimization: "", // Not supported by DABs - IsolationMode: "", // Not supported by DABs - Name: id, - NewName: "", // Initialized below if needed - Options: config.Options, - Owner: "", // Not supported by DABs - Properties: config.Properties, - ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](config.ForceSendFields, "EnablePredictiveOptimization", "IsolationMode", "Owner"), - } - +func (r *ResourceCatalog) DoUpdateWithID(ctx context.Context, id string, config *catalogState) (string, *catalog.CatalogInfo, error) { + newName := "" if config.Name != id { - updateRequest.NewName = config.Name + newName = config.Name } - response, err := r.client.Catalogs.Update(ctx, updateRequest) + response, err := r.applyUpdate(ctx, id, newName, config) if err != nil { return "", nil, err } - // Return the new name as the ID if it changed, otherwise return the old ID newID := id - if updateRequest.NewName != "" { - newID = updateRequest.NewName + if newName != "" { + newID = newName } return newID, response, nil } +// applyUpdate builds and sends an UpdateCatalog request. newName is set only when renaming. +func (r *ResourceCatalog) applyUpdate(ctx context.Context, id, newName string, config *catalogState) (*catalog.CatalogInfo, error) { + updateRequest := catalog.UpdateCatalog{ + Comment: config.Comment, + EnablePredictiveOptimization: config.EnablePredictiveOptimization, + IsolationMode: config.IsolationMode, + Name: id, + NewName: newName, + Options: config.Options, + Owner: config.Owner, + Properties: config.Properties, + ForceSendFields: utils.FilterFields[catalog.UpdateCatalog](config.ForceSendFields), + } + + return r.client.Catalogs.Update(ctx, updateRequest) +} + func (r *ResourceCatalog) DoDelete(ctx context.Context, id string) error { return r.client.Catalogs.Delete(ctx, catalog.DeleteCatalogRequest{ Name: id, diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index 733973ecf9..52f4d25f32 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -314,6 +314,10 @@ resources: update_id_on_changes: - field: name reason: id_changes + backend_defaults: + # Owner is set to the creator by the API when not specified by the user. + # Same as registered_models.owner — backend-set output field. + - field: owner schemas: recreate_on_changes: diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 8dcf37f7e8..f873f7babf 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -562,12 +562,21 @@ github.com/databricks/cli/bundle/config/resources.Catalog: "connection_name": "description": |- PLACEHOLDER + "enable_predictive_optimization": + "description": |- + PLACEHOLDER + "isolation_mode": + "description": |- + PLACEHOLDER "name": "description": |- PLACEHOLDER "options": "description": |- PLACEHOLDER + "owner": + "description": |- + PLACEHOLDER "properties": "description": |- PLACEHOLDER diff --git a/bundle/internal/schema/annotations_openapi.yml b/bundle/internal/schema/annotations_openapi.yml index c196c7799a..6b8e95dcb5 100644 --- a/bundle/internal/schema/annotations_openapi.yml +++ b/bundle/internal/schema/annotations_openapi.yml @@ -1575,6 +1575,22 @@ github.com/databricks/databricks-sdk-go/service/catalog.AzureQueueStorage: Optional subscription id for the queue, event grid subscription, and external location storage account. Required for locations with a service principal storage credential +github.com/databricks/databricks-sdk-go/service/catalog.CatalogIsolationMode: + "_": + "enum": + - |- + OPEN + - |- + ISOLATED +github.com/databricks/databricks-sdk-go/service/catalog.EnablePredictiveOptimization: + "_": + "enum": + - |- + DISABLE + - |- + ENABLE + - |- + INHERIT github.com/databricks/databricks-sdk-go/service/catalog.EncryptionDetails: "_": "description": |- diff --git a/bundle/internal/schema/annotations_openapi_overrides.yml b/bundle/internal/schema/annotations_openapi_overrides.yml index 611289083e..bb6fd09fe1 100644 --- a/bundle/internal/schema/annotations_openapi_overrides.yml +++ b/bundle/internal/schema/annotations_openapi_overrides.yml @@ -73,12 +73,21 @@ github.com/databricks/cli/bundle/config/resources.App: "description": |- PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Catalog: + "enable_predictive_optimization": + "description": |- + PLACEHOLDER "grants": "description": |- PLACEHOLDER + "isolation_mode": + "description": |- + PLACEHOLDER "lifecycle": "description": |- PLACEHOLDER + "owner": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.Cluster: "_": "markdown_description": |- diff --git a/bundle/internal/validation/generated/enum_fields.go b/bundle/internal/validation/generated/enum_fields.go index c1e098ed80..f02c6eebaa 100644 --- a/bundle/internal/validation/generated/enum_fields.go +++ b/bundle/internal/validation/generated/enum_fields.go @@ -38,7 +38,9 @@ var EnumFields = map[string][]string{ "resources.apps.*.resources[*].uc_securable.permission": {"EXECUTE", "MODIFY", "READ_VOLUME", "SELECT", "USE_CONNECTION", "WRITE_VOLUME"}, "resources.apps.*.resources[*].uc_securable.securable_type": {"CONNECTION", "FUNCTION", "TABLE", "VOLUME"}, - "resources.catalogs.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, + "resources.catalogs.*.enable_predictive_optimization": {"DISABLE", "ENABLE", "INHERIT"}, + "resources.catalogs.*.grants[*].privileges[*]": {"ACCESS", "ALL_PRIVILEGES", "APPLY_TAG", "BROWSE", "CREATE", "CREATE_CATALOG", "CREATE_CLEAN_ROOM", "CREATE_CONNECTION", "CREATE_EXTERNAL_LOCATION", "CREATE_EXTERNAL_TABLE", "CREATE_EXTERNAL_VOLUME", "CREATE_FOREIGN_CATALOG", "CREATE_FOREIGN_SECURABLE", "CREATE_FUNCTION", "CREATE_MANAGED_STORAGE", "CREATE_MATERIALIZED_VIEW", "CREATE_MODEL", "CREATE_PROVIDER", "CREATE_RECIPIENT", "CREATE_SCHEMA", "CREATE_SERVICE_CREDENTIAL", "CREATE_SHARE", "CREATE_STORAGE_CREDENTIAL", "CREATE_TABLE", "CREATE_VIEW", "CREATE_VOLUME", "EXECUTE", "EXECUTE_CLEAN_ROOM_TASK", "EXTERNAL_USE_SCHEMA", "MANAGE", "MANAGE_ALLOWLIST", "MODIFY", "MODIFY_CLEAN_ROOM", "READ_FILES", "READ_PRIVATE_FILES", "READ_VOLUME", "REFRESH", "SELECT", "SET_SHARE_PERMISSION", "USAGE", "USE_CATALOG", "USE_CONNECTION", "USE_MARKETPLACE_ASSETS", "USE_PROVIDER", "USE_RECIPIENT", "USE_SCHEMA", "USE_SHARE", "WRITE_FILES", "WRITE_PRIVATE_FILES", "WRITE_VOLUME"}, + "resources.catalogs.*.isolation_mode": {"ISOLATED", "OPEN"}, "resources.clusters.*.aws_attributes.availability": {"ON_DEMAND", "SPOT", "SPOT_WITH_FALLBACK"}, "resources.clusters.*.aws_attributes.ebs_volume_type": {"GENERAL_PURPOSE_SSD", "THROUGHPUT_OPTIMIZED_HDD"}, diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index abfdb55233..8e49a1453f 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -295,9 +295,15 @@ "connection_name": { "$ref": "#/$defs/string" }, + "enable_predictive_optimization": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.EnablePredictiveOptimization" + }, "grants": { "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment" }, + "isolation_mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.CatalogIsolationMode" + }, "lifecycle": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle" }, @@ -307,6 +313,9 @@ "options": { "$ref": "#/$defs/map/string" }, + "owner": { + "$ref": "#/$defs/string" + }, "properties": { "$ref": "#/$defs/map/string" }, @@ -3570,6 +3579,37 @@ } ] }, + "catalog.CatalogIsolationMode": { + "oneOf": [ + { + "type": "string", + "enum": [ + "OPEN", + "ISOLATED" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "catalog.EnablePredictiveOptimization": { + "oneOf": [ + { + "type": "string", + "enum": [ + "DISABLE", + "ENABLE", + "INHERIT" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "catalog.EncryptionDetails": { "oneOf": [ { diff --git a/bundle/schema/jsonschema_for_docs.json b/bundle/schema/jsonschema_for_docs.json index bcb6866296..04350fb08f 100644 --- a/bundle/schema/jsonschema_for_docs.json +++ b/bundle/schema/jsonschema_for_docs.json @@ -236,10 +236,16 @@ "$ref": "#/$defs/string", "x-since-version": "v0.287.0" }, + "enable_predictive_optimization": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.EnablePredictiveOptimization" + }, "grants": { "$ref": "#/$defs/slice/github.com/databricks/databricks-sdk-go/service/catalog.PrivilegeAssignment", "x-since-version": "v0.287.0" }, + "isolation_mode": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/catalog.CatalogIsolationMode" + }, "lifecycle": { "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.Lifecycle", "x-since-version": "v0.287.0" @@ -252,6 +258,9 @@ "$ref": "#/$defs/map/string", "x-since-version": "v0.287.0" }, + "owner": { + "$ref": "#/$defs/string" + }, "properties": { "$ref": "#/$defs/map/string", "x-since-version": "v0.287.0" @@ -3286,6 +3295,21 @@ }, "additionalProperties": false }, + "catalog.CatalogIsolationMode": { + "type": "string", + "enum": [ + "OPEN", + "ISOLATED" + ] + }, + "catalog.EnablePredictiveOptimization": { + "type": "string", + "enum": [ + "DISABLE", + "ENABLE", + "INHERIT" + ] + }, "catalog.EncryptionDetails": { "type": "object", "description": "Encryption options that apply to clients connecting to cloud storage.", diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 859721ee73..0358acfbae 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -69,6 +69,12 @@ func (s *FakeWorkspace) CatalogsUpdate(req Request, name string) Response { if updateRequest.Owner != "" { existing.Owner = updateRequest.Owner } + if updateRequest.IsolationMode != "" { + existing.IsolationMode = updateRequest.IsolationMode + } + if updateRequest.EnablePredictiveOptimization != "" { + existing.EnablePredictiveOptimization = updateRequest.EnablePredictiveOptimization + } if updateRequest.NewName != "" { existing.Name = updateRequest.NewName existing.FullName = updateRequest.NewName diff --git a/python/databricks/bundles/catalogs/__init__.py b/python/databricks/bundles/catalogs/__init__.py index fc92f42e22..b4b5ca6e00 100644 --- a/python/databricks/bundles/catalogs/__init__.py +++ b/python/databricks/bundles/catalogs/__init__.py @@ -6,7 +6,11 @@ "CatalogGrantParam", "CatalogGrantPrivilege", "CatalogGrantPrivilegeParam", + "CatalogIsolationMode", + "CatalogIsolationModeParam", "CatalogParam", + "EnablePredictiveOptimization", + "EnablePredictiveOptimizationParam", "Lifecycle", "LifecycleDict", "LifecycleParam", @@ -23,6 +27,14 @@ CatalogDict, CatalogParam, ) +from databricks.bundles.catalogs._models.catalog_isolation_mode import ( + CatalogIsolationMode, + CatalogIsolationModeParam, +) +from databricks.bundles.catalogs._models.enable_predictive_optimization import ( + EnablePredictiveOptimization, + EnablePredictiveOptimizationParam, +) from databricks.bundles.catalogs._models.lifecycle import ( Lifecycle, LifecycleDict, diff --git a/python/databricks/bundles/catalogs/_models/catalog.py b/python/databricks/bundles/catalogs/_models/catalog.py index 368e96a3b2..b1926c8f1d 100644 --- a/python/databricks/bundles/catalogs/_models/catalog.py +++ b/python/databricks/bundles/catalogs/_models/catalog.py @@ -1,6 +1,14 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, TypedDict +from databricks.bundles.catalogs._models.catalog_isolation_mode import ( + CatalogIsolationMode, + CatalogIsolationModeParam, +) +from databricks.bundles.catalogs._models.enable_predictive_optimization import ( + EnablePredictiveOptimization, + EnablePredictiveOptimizationParam, +) from databricks.bundles.catalogs._models.lifecycle import Lifecycle, LifecycleParam from databricks.bundles.catalogs._models.privilege_assignment import ( PrivilegeAssignment, @@ -30,12 +38,20 @@ class Catalog(Resource): connection_name: VariableOrOptional[str] = None + enable_predictive_optimization: VariableOrOptional[EnablePredictiveOptimization] = ( + None + ) + grants: VariableOrList[PrivilegeAssignment] = field(default_factory=list) + isolation_mode: VariableOrOptional[CatalogIsolationMode] = None + lifecycle: VariableOrOptional[Lifecycle] = None options: VariableOrDict[str] = field(default_factory=dict) + owner: VariableOrOptional[str] = None + properties: VariableOrDict[str] = field(default_factory=dict) provider_name: VariableOrOptional[str] = None @@ -61,12 +77,20 @@ class CatalogDict(TypedDict, total=False): connection_name: VariableOrOptional[str] + enable_predictive_optimization: VariableOrOptional[ + EnablePredictiveOptimizationParam + ] + grants: VariableOrList[PrivilegeAssignmentParam] + isolation_mode: VariableOrOptional[CatalogIsolationModeParam] + lifecycle: VariableOrOptional[LifecycleParam] options: VariableOrDict[str] + owner: VariableOrOptional[str] + properties: VariableOrDict[str] provider_name: VariableOrOptional[str] diff --git a/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py b/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py new file mode 100644 index 0000000000..b41cf7de69 --- /dev/null +++ b/python/databricks/bundles/catalogs/_models/catalog_isolation_mode.py @@ -0,0 +1,10 @@ +from enum import Enum +from typing import Literal + + +class CatalogIsolationMode(Enum): + OPEN = "OPEN" + ISOLATED = "ISOLATED" + + +CatalogIsolationModeParam = Literal["OPEN", "ISOLATED"] | CatalogIsolationMode diff --git a/python/databricks/bundles/catalogs/_models/enable_predictive_optimization.py b/python/databricks/bundles/catalogs/_models/enable_predictive_optimization.py new file mode 100644 index 0000000000..f4ca1e5b58 --- /dev/null +++ b/python/databricks/bundles/catalogs/_models/enable_predictive_optimization.py @@ -0,0 +1,13 @@ +from enum import Enum +from typing import Literal + + +class EnablePredictiveOptimization(Enum): + DISABLE = "DISABLE" + ENABLE = "ENABLE" + INHERIT = "INHERIT" + + +EnablePredictiveOptimizationParam = ( + Literal["DISABLE", "ENABLE", "INHERIT"] | EnablePredictiveOptimization +)