diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 23715f17da..d6b5dde4f9 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -16,6 +16,7 @@ See more details here: ([#3225](https://github.com/databricks/cli/pull/3225)) ### Bundles * [Breaking Change] Remove deprecated path fallback mechanism for jobs and pipelines ([#3225](https://github.com/databricks/cli/pull/3225)) +* Add support for Lakebase synced database tables in DABs ([#3467](https://github.com/databricks/cli/pull/3467)) * Rename Delta Live Tables to Lakeflow Declarative Pipelines in the default-python template ([#3476](https://github.com/databricks/cli/pull/3476)). ### API Changes diff --git a/acceptance/bundle/deploy/lakebase/synced-database-table/databricks.yml.tmpl b/acceptance/bundle/deploy/lakebase/synced-database-table/databricks.yml.tmpl new file mode 100644 index 0000000000..a15129f700 --- /dev/null +++ b/acceptance/bundle/deploy/lakebase/synced-database-table/databricks.yml.tmpl @@ -0,0 +1,24 @@ +bundle: + name: deploy-lakebase-synced-table-$UNIQUE_NAME + +resources: + database_instances: + my_instance: + name: test-database-instance-$UNIQUE_NAME + capacity: CU_1 + database_catalogs: + my_catalog: + database_instance_name: ${resources.database_instances.my_instance.name} + database_name: my_database + name: my_catalog_$UNIQUE_NAME + create_database_if_not_exists: true + synced_database_tables: + my_synced_table: + name: ${resources.database_catalogs.my_catalog.name}.${resources.database_catalogs.my_catalog.database_name}.my_synced_table + database_instance_name: ${resources.database_instances.my_instance.name} + logical_database_name: ${resources.database_catalogs.my_catalog.database_name} + spec: + source_table_full_name: "samples.nyctaxi.trips" + scheduling_policy: SNAPSHOT + primary_key_columns: + - tpep_pickup_datetime diff --git a/acceptance/bundle/deploy/lakebase/synced-database-table/out.test.toml b/acceptance/bundle/deploy/lakebase/synced-database-table/out.test.toml new file mode 100644 index 0000000000..496bc95f78 --- /dev/null +++ b/acceptance/bundle/deploy/lakebase/synced-database-table/out.test.toml @@ -0,0 +1,9 @@ +Local = true +Cloud = true +RequiresUnityCatalog = true + +[CloudEnvs] + gcp = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform"] diff --git a/acceptance/bundle/deploy/lakebase/synced-database-table/output.txt b/acceptance/bundle/deploy/lakebase/synced-database-table/output.txt new file mode 100644 index 0000000000..5963563abc --- /dev/null +++ b/acceptance/bundle/deploy/lakebase/synced-database-table/output.txt @@ -0,0 +1,66 @@ + +>>> [CLI] bundle validate +Name: deploy-lakebase-synced-table-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default + +Validation OK! + +>>> [CLI] bundle summary +Name: deploy-lakebase-synced-table-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default +Resources: + Database catalogs: + my_catalog: + Name: my_catalog_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/my_catalog_[UNIQUE_NAME] + Database instances: + my_instance: + Name: test-database-instance-[UNIQUE_NAME] + URL: (not deployed) + Synced database tables: + my_synced_table: + Name: ${databricks_database_database_catalog.my_catalog.name}.${databricks_database_database_catalog.my_catalog.database_name}.my_synced_table + URL: [DATABRICKS_URL]/explore/data/$%7Bdatabricks_database_database_catalog.my_catalog.name%7D.$%7Bdatabricks_database_database_catalog.my_catalog.database_name%7D.my_synced_table + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle summary +Name: deploy-lakebase-synced-table-[UNIQUE_NAME] +Target: default +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default +Resources: + Database catalogs: + my_catalog: + Name: my_catalog_[UNIQUE_NAME] + URL: [DATABRICKS_URL]/explore/data/my_catalog_[UNIQUE_NAME] + Database instances: + my_instance: + Name: test-database-instance-[UNIQUE_NAME] + URL: [DATABRICKS_URL]/compute/database-instances/test-database-instance-[UNIQUE_NAME] + Synced database tables: + my_synced_table: + Name: ${resources.database_catalogs.my_catalog.name}.${resources.database_catalogs.my_catalog.database_name}.my_synced_table + URL: [DATABRICKS_URL]/explore/data/$%7Bresources.database_catalogs.my_catalog.name%7D.$%7Bresources.database_catalogs.my_catalog.database_name%7D.my_synced_table + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete database_catalog my_catalog + delete database_instance my_instance + delete synced_database_table my_synced_table + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/deploy/lakebase/synced-database-table/script b/acceptance/bundle/deploy/lakebase/synced-database-table/script new file mode 100644 index 0000000000..10dfe0d5b7 --- /dev/null +++ b/acceptance/bundle/deploy/lakebase/synced-database-table/script @@ -0,0 +1,11 @@ +envsubst < databricks.yml.tmpl > databricks.yml + +cleanup() { + trace $CLI bundle destroy --auto-approve +} + trap cleanup EXIT + +trace $CLI bundle validate +trace $CLI bundle summary +trace $CLI bundle deploy +trace $CLI bundle summary diff --git a/acceptance/bundle/deploy/lakebase/synced-database-table/test.toml b/acceptance/bundle/deploy/lakebase/synced-database-table/test.toml new file mode 100644 index 0000000000..78603c280f --- /dev/null +++ b/acceptance/bundle/deploy/lakebase/synced-database-table/test.toml @@ -0,0 +1,13 @@ +Local = true +Cloud = true + +RecordRequests = false + +[EnvMatrix] +DATABRICKS_CLI_DEPLOYMENT = ["terraform"] + +[[Repls]] +# clean up ?o= suffix after URL since not all workspaces have that +Old = '\?o=\[(NUMID|ALPHANUMID)\]' +New = '' +Order = 1000 diff --git a/acceptance/internal/handlers.go b/acceptance/internal/handlers.go index 5f7f9f0f0b..4bfae2486d 100644 --- a/acceptance/internal/handlers.go +++ b/acceptance/internal/handlers.go @@ -526,4 +526,16 @@ func addDefaultHandlers(server *testserver.Server) { server.Handle("DELETE", "/api/2.0/database/catalogs/{name}", func(req testserver.Request) any { return testserver.MapDelete(req.Workspace, req.Workspace.DatabaseCatalogs, req.Vars["name"]) }) + + server.Handle("POST", "/api/2.0/database/synced_tables", func(req testserver.Request) any { + return req.Workspace.SyncedDatabaseTableCreate(req) + }) + + server.Handle("GET", "/api/2.0/database/synced_tables/{name}", func(req testserver.Request) any { + return testserver.MapGet(req.Workspace, req.Workspace.SyncedDatabaseTables, req.Vars["name"]) + }) + + server.Handle("DELETE", "/api/2.0/database/synced_tables/{name}", func(req testserver.Request) any { + return testserver.MapDelete(req.Workspace, req.Workspace.SyncedDatabaseTables, req.Vars["name"]) + }) } diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index 3b2e20883a..e83959228a 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -15,7 +15,7 @@ import ( "github.com/databricks/cli/libs/dyn/convert" ) -var unsupportedResources = []string{"clusters", "volumes", "schemas", "quality_monitors", "registered_models", "database_catalogs"} +var unsupportedResources = []string{"clusters", "volumes", "schemas", "quality_monitors", "registered_models", "database_catalogs", "synced_database_tables"} var ( allowedLevels = []string{permissions.CAN_MANAGE, permissions.CAN_VIEW, permissions.CAN_RUN} diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index 91daafaeae..54da819d75 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -183,6 +183,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { }, }, }, + SyncedDatabaseTables: map[string]*resources.SyncedDatabaseTable{ + "synced_database_table1": { + SyncedDatabaseTable: database.SyncedDatabaseTable{ + Name: "synced_database_table1", + }, + }, + }, }, }, SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"), @@ -351,7 +358,7 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) { resourceType := resources.Type().Field(i).Name // Skip resources that are not renamed - if resourceType == "Apps" || resourceType == "SecretScopes" || resourceType == "DatabaseInstances" || resourceType == "DatabaseCatalogs" { + if resourceType == "Apps" || resourceType == "SecretScopes" || resourceType == "DatabaseInstances" || resourceType == "DatabaseCatalogs" || resourceType == "SyncedDatabaseTables" { continue } diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 19e9d174af..96f37acc55 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -47,6 +47,7 @@ func allResourceTypes(t *testing.T) []string { "schemas", "secret_scopes", "sql_warehouses", + "synced_database_tables", "volumes", }, resourceTypes, @@ -143,6 +144,7 @@ var allowList = []string{ "clusters", "database_catalogs", "database_instances", + "synced_database_tables", "jobs", "models", "registered_models", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 6187ca663f..9f95aa0b08 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -28,6 +28,7 @@ type Resources struct { SqlWarehouses map[string]*resources.SqlWarehouse `json:"sql_warehouses,omitempty"` DatabaseInstances map[string]*resources.DatabaseInstance `json:"database_instances,omitempty"` DatabaseCatalogs map[string]*resources.DatabaseCatalog `json:"database_catalogs,omitempty"` + SyncedDatabaseTables map[string]*resources.SyncedDatabaseTable `json:"synced_database_tables,omitempty"` } type ConfigResource interface { @@ -94,6 +95,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["sql_warehouses"], r.SqlWarehouses), collectResourceMap(descriptions["database_instances"], r.DatabaseInstances), collectResourceMap(descriptions["database_catalogs"], r.DatabaseCatalogs), + collectResourceMap(descriptions["synced_database_tables"], r.SyncedDatabaseTables), } } @@ -189,6 +191,12 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) } } + for k := range r.SyncedDatabaseTables { + if k == key { + found = append(found, r.SyncedDatabaseTables[k]) + } + } + if len(found) == 0 { return nil, fmt.Errorf("no such resource: %s", key) } @@ -223,5 +231,6 @@ func SupportedResources() map[string]resources.ResourceDescription { "sql_warehouses": (&resources.SqlWarehouse{}).ResourceDescription(), "database_instances": (&resources.DatabaseInstance{}).ResourceDescription(), "database_catalogs": (&resources.DatabaseCatalog{}).ResourceDescription(), + "synced_database_tables": (&resources.SyncedDatabaseTable{}).ResourceDescription(), } } diff --git a/bundle/config/resources/synced_database_table.go b/bundle/config/resources/synced_database_table.go new file mode 100644 index 0000000000..0c433daf51 --- /dev/null +++ b/bundle/config/resources/synced_database_table.go @@ -0,0 +1,53 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/service/database" +) + +type SyncedDatabaseTable struct { + ID string `json:"id,omitempty" bundle:"readonly"` + URL string `json:"url,omitempty" bundle:"internal"` + ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"` + + database.SyncedDatabaseTable +} + +func (s *SyncedDatabaseTable) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) { + _, err := w.Database.GetSyncedDatabaseTable(ctx, database.GetSyncedDatabaseTableRequest{Name: name}) + if err != nil { + log.Debugf(ctx, "synced database table %s does not exist", name) + return false, err + } + return true, nil +} + +func (s *SyncedDatabaseTable) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "synced_database_table", + PluralName: "synced_database_tables", + SingularTitle: "Synced database table", + PluralTitle: "Synced database tables", + } +} + +func (s *SyncedDatabaseTable) GetName() string { + return s.Name +} + +func (s *SyncedDatabaseTable) GetURL() string { + return s.URL +} + +func (s *SyncedDatabaseTable) InitializeURL(baseURL url.URL) { + if s.Name == "" { + return + } + baseURL.Path = "explore/data/" + s.Name + s.URL = baseURL.String() +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 27ed583a87..83478ce33e 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -190,6 +190,11 @@ func TestResourcesBindSupport(t *testing.T) { DatabaseCatalog: database.DatabaseCatalog{}, }, }, + SyncedDatabaseTables: map[string]*resources.SyncedDatabaseTable{ + "my_synced_database_table": { + SyncedDatabaseTable: database.SyncedDatabaseTable{}, + }, + }, } unbindableResources := map[string]bool{"model": true} @@ -212,6 +217,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockWarehousesAPI().EXPECT().GetById(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockDatabaseAPI().EXPECT().GetDatabaseInstance(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockDatabaseAPI().EXPECT().GetDatabaseCatalog(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockDatabaseAPI().EXPECT().GetSyncedDatabaseTable(mock.Anything, mock.Anything).Return(nil, nil) allResources := supportedResources.AllResources() for _, group := range allResources { diff --git a/bundle/deploy/terraform/pkg.go b/bundle/deploy/terraform/pkg.go index f2aad7a481..e79c82f8c4 100644 --- a/bundle/deploy/terraform/pkg.go +++ b/bundle/deploy/terraform/pkg.go @@ -118,6 +118,7 @@ var GroupToTerraformName = map[string]string{ "sql_warehouses": "databricks_sql_endpoint", "database_instances": "databricks_database_instance", "database_catalogs": "databricks_database_database_catalog", + "synced_database_tables": "databricks_database_synced_database_table", } var TerraformToGroupName = func() map[string]string { diff --git a/bundle/deploy/terraform/tfdyn/convert_synced_database_table.go b/bundle/deploy/terraform/tfdyn/convert_synced_database_table.go new file mode 100644 index 0000000000..8d485e7fdf --- /dev/null +++ b/bundle/deploy/terraform/tfdyn/convert_synced_database_table.go @@ -0,0 +1,28 @@ +package tfdyn + +import ( + "context" + + "github.com/databricks/cli/bundle/internal/tf/schema" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/convert" + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go/service/database" +) + +type syncedDatabaseTableConverter struct{} + +func (s syncedDatabaseTableConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error { + // Normalize the output value to the target schema. + vout, diags := convert.Normalize(database.SyncedDatabaseTable{}, vin) + for _, diag := range diags { + log.Debugf(ctx, "synced database table normalization diagnostic: %s", diag.Summary) + } + out.DatabaseSyncedDatabaseTable[key] = vout.AsAny() + + return nil +} + +func init() { + registerConverter("synced_database_tables", syncedDatabaseTableConverter{}) +} diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 608528df57..de233f1421 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -225,6 +225,9 @@ github.com/databricks/cli/bundle/config.Resources: The SQL warehouse definitions for the bundle, where each key is the name of the warehouse. "markdown_description": |- The SQL warehouse definitions for the bundle, where each key is the name of the warehouse. See [\_](/dev-tools/bundles/resources.md#sql_warehouses). + "synced_database_tables": + "description": |- + PLACEHOLDER "volumes": "description": |- The volume definitions for the bundle, where each key is the name of the volume. @@ -700,6 +703,31 @@ github.com/databricks/cli/bundle/config/resources.SqlWarehousePermission: "user_name": "description": |- PLACEHOLDER +github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable: + "data_synchronization_status": + "description": |- + PLACEHOLDER + "database_instance_name": + "description": |- + PLACEHOLDER + "effective_database_instance_name": + "description": |- + PLACEHOLDER + "effective_logical_database_name": + "description": |- + PLACEHOLDER + "logical_database_name": + "description": |- + PLACEHOLDER + "name": + "description": |- + PLACEHOLDER + "spec": + "description": |- + PLACEHOLDER + "unity_catalog_provisioning_state": + "description": |- + PLACEHOLDER github.com/databricks/cli/bundle/config/resources.VolumeGrant: "principal": "description": |- diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index c0aec94eb8..0db03c39ec 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1698,6 +1698,47 @@ } ] }, + "resources.SyncedDatabaseTable": { + "oneOf": [ + { + "type": "object", + "properties": { + "data_synchronization_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableStatus" + }, + "database_instance_name": { + "$ref": "#/$defs/string" + }, + "effective_database_instance_name": { + "$ref": "#/$defs/string" + }, + "effective_logical_database_name": { + "$ref": "#/$defs/string" + }, + "logical_database_name": { + "$ref": "#/$defs/string" + }, + "name": { + "$ref": "#/$defs/string" + }, + "spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableSpec" + }, + "unity_catalog_provisioning_state": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.ProvisioningInfoState" + } + }, + "additionalProperties": false, + "required": [ + "name" + ] + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Volume": { "oneOf": [ { @@ -2296,6 +2337,9 @@ "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SqlWarehouse", "markdownDescription": "The SQL warehouse definitions for the bundle, where each key is the name of the warehouse. See [sql_warehouses](https://docs.databricks.com/dev-tools/bundles/resources.html#sql_warehouses)." }, + "synced_database_tables": { + "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable" + }, "volumes": { "description": "The volume definitions for the bundle, where each key is the name of the volume.", "$ref": "#/$defs/map/github.com/databricks/cli/bundle/config/resources.Volume", @@ -4279,6 +4323,269 @@ } ] }, + "database.DeltaTableSyncInfo": { + "oneOf": [ + { + "type": "object", + "properties": { + "delta_commit_timestamp": { + "$ref": "#/$defs/string" + }, + "delta_commit_version": { + "$ref": "#/$defs/int64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.NewPipelineSpec": { + "oneOf": [ + { + "type": "object", + "properties": { + "storage_catalog": { + "$ref": "#/$defs/string" + }, + "storage_schema": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.ProvisioningInfoState": { + "type": "string" + }, + "database.ProvisioningPhase": { + "type": "string" + }, + "database.SyncedTableContinuousUpdateStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "initial_pipeline_sync_progress": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTablePipelineProgress" + }, + "last_processed_commit_version": { + "$ref": "#/$defs/int64" + }, + "timestamp": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTableFailedStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "last_processed_commit_version": { + "$ref": "#/$defs/int64" + }, + "timestamp": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTablePipelineProgress": { + "oneOf": [ + { + "type": "object", + "properties": { + "estimated_completion_time_seconds": { + "$ref": "#/$defs/float64" + }, + "latest_version_currently_processing": { + "$ref": "#/$defs/int64" + }, + "provisioning_phase": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.ProvisioningPhase" + }, + "sync_progress_completion": { + "$ref": "#/$defs/float64" + }, + "synced_row_count": { + "$ref": "#/$defs/int64" + }, + "total_row_count": { + "$ref": "#/$defs/int64" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTablePosition": { + "oneOf": [ + { + "type": "object", + "properties": { + "delta_table_sync_info": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.DeltaTableSyncInfo" + }, + "sync_end_timestamp": { + "$ref": "#/$defs/string" + }, + "sync_start_timestamp": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTableProvisioningStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "initial_pipeline_sync_progress": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTablePipelineProgress" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTableSchedulingPolicy": { + "type": "string" + }, + "database.SyncedTableSpec": { + "oneOf": [ + { + "type": "object", + "properties": { + "create_database_objects_if_missing": { + "$ref": "#/$defs/bool" + }, + "existing_pipeline_id": { + "$ref": "#/$defs/string" + }, + "new_pipeline_spec": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.NewPipelineSpec" + }, + "primary_key_columns": { + "$ref": "#/$defs/slice/string" + }, + "scheduling_policy": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableSchedulingPolicy" + }, + "source_table_full_name": { + "$ref": "#/$defs/string" + }, + "timeseries_key": { + "$ref": "#/$defs/string" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTableState": { + "type": "string" + }, + "database.SyncedTableStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "continuous_update_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableContinuousUpdateStatus" + }, + "detailed_state": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableState" + }, + "failed_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableFailedStatus" + }, + "last_sync": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTablePosition" + }, + "message": { + "$ref": "#/$defs/string" + }, + "pipeline_id": { + "$ref": "#/$defs/string" + }, + "provisioning_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableProvisioningStatus" + }, + "triggered_update_status": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTableTriggeredUpdateStatus" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, + "database.SyncedTableTriggeredUpdateStatus": { + "oneOf": [ + { + "type": "object", + "properties": { + "last_processed_commit_version": { + "$ref": "#/$defs/int64" + }, + "timestamp": { + "$ref": "#/$defs/string" + }, + "triggered_update_progress": { + "$ref": "#/$defs/github.com/databricks/databricks-sdk-go/service/database.SyncedTablePipelineProgress" + } + }, + "additionalProperties": false + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "jobs.AuthenticationMethod": { "oneOf": [ { @@ -8656,6 +8963,20 @@ } ] }, + "resources.SyncedDatabaseTable": { + "oneOf": [ + { + "type": "object", + "additionalProperties": { + "$ref": "#/$defs/github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable" + } + }, + { + "type": "string", + "pattern": "\\$\\{(var(\\.[a-zA-Z]+([-_]?[a-zA-Z0-9]+)*(\\[[0-9]+\\])*)+)\\}" + } + ] + }, "resources.Volume": { "oneOf": [ { diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 60fc7e5388..ba27fe788e 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -74,6 +74,9 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "database_catalogs": map[string]ResourceState{ "test_database_catalog": {ID: "1"}, }, + "synced_database_tables": map[string]ResourceState{ + "test_synced_database_table": {ID: "1"}, + }, } err := StateToBundle(context.Background(), state, &config) assert.NoError(t, err) @@ -242,6 +245,13 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + SyncedDatabaseTables: map[string]*resources.SyncedDatabaseTable{ + "test_synced_database_table": { + SyncedDatabaseTable: database.SyncedDatabaseTable{ + Name: "test_synced_database_table", + }, + }, + }, }, } @@ -296,6 +306,9 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { assert.Equal(t, "", config.Resources.DatabaseCatalogs["test_database_catalog"].ID) assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.DatabaseCatalogs["test_database_catalog"].ModifiedStatus) + assert.Equal(t, "", config.Resources.SyncedDatabaseTables["test_synced_database_table"].ID) + assert.Equal(t, resources.ModifiedStatusCreated, config.Resources.SyncedDatabaseTables["test_synced_database_table"].ModifiedStatus) + AssertFullResourceCoverage(t, &config) } @@ -494,6 +507,18 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + SyncedDatabaseTables: map[string]*resources.SyncedDatabaseTable{ + "test_synced_database_table": { + SyncedDatabaseTable: database.SyncedDatabaseTable{ + Name: "test_synced_database_table", + }, + }, + "test_synced_database_table_new": { + SyncedDatabaseTable: database.SyncedDatabaseTable{ + Name: "test_synced_database_table_new", + }, + }, + }, }, } state := ExportedResourcesMap{ diff --git a/libs/structwalk/walktype_test.go b/libs/structwalk/walktype_test.go index ce0f8852bf..c84b5b810f 100644 --- a/libs/structwalk/walktype_test.go +++ b/libs/structwalk/walktype_test.go @@ -123,7 +123,7 @@ func TestTypeJobSettings(t *testing.T) { func TestTypeRoot(t *testing.T) { testStruct(t, reflect.TypeOf(config.Root{}), - 3600, 3850, // 3794 at the time of the update + 3600, 4000, // 3980 at the time of the update map[string]any{ ".bundle.target": "", `.variables[*].lookup.dashboard`: "", diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index cefbabbe28..3aaef3b841 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -81,8 +81,9 @@ type FakeWorkspace struct { nextRepoId int64 Repos map[string]workspace.RepoInfo - DatabaseInstances map[string]database.DatabaseInstance - DatabaseCatalogs map[string]database.DatabaseCatalog + DatabaseInstances map[string]database.DatabaseInstance + DatabaseCatalogs map[string]database.DatabaseCatalog + SyncedDatabaseTables map[string]database.SyncedDatabaseTable } func (s *FakeWorkspace) LockUnlock() func() { @@ -151,22 +152,23 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { files: make(map[string]FileEntry), repoIdByPath: make(map[string]int64), - Jobs: map[int64]jobs.Job{}, - JobRuns: map[int64]jobs.Run{}, - nextJobId: TestJobID, - nextJobRunId: TestRunID, - Pipelines: map[string]pipelines.GetPipelineResponse{}, - PipelineUpdates: map[string]bool{}, - Monitors: map[string]catalog.MonitorInfo{}, - Apps: map[string]apps.App{}, - Schemas: map[string]catalog.SchemaInfo{}, - Volumes: map[string]catalog.VolumeInfo{}, - Dashboards: map[string]dashboards.Dashboard{}, - SqlWarehouses: map[string]sql.GetWarehouseResponse{}, - Repos: map[string]workspace.RepoInfo{}, - Acls: map[string][]workspace.AclItem{}, - DatabaseInstances: map[string]database.DatabaseInstance{}, - DatabaseCatalogs: map[string]database.DatabaseCatalog{}, + Jobs: map[int64]jobs.Job{}, + JobRuns: map[int64]jobs.Run{}, + nextJobId: TestJobID, + nextJobRunId: TestRunID, + Pipelines: map[string]pipelines.GetPipelineResponse{}, + PipelineUpdates: map[string]bool{}, + Monitors: map[string]catalog.MonitorInfo{}, + Apps: map[string]apps.App{}, + Schemas: map[string]catalog.SchemaInfo{}, + Volumes: map[string]catalog.VolumeInfo{}, + Dashboards: map[string]dashboards.Dashboard{}, + SqlWarehouses: map[string]sql.GetWarehouseResponse{}, + Repos: map[string]workspace.RepoInfo{}, + Acls: map[string][]workspace.AclItem{}, + DatabaseInstances: map[string]database.DatabaseInstance{}, + DatabaseCatalogs: map[string]database.DatabaseCatalog{}, + SyncedDatabaseTables: map[string]database.SyncedDatabaseTable{}, } } diff --git a/libs/testserver/synced_database_tables.go b/libs/testserver/synced_database_tables.go new file mode 100644 index 0000000000..e0297a68d7 --- /dev/null +++ b/libs/testserver/synced_database_tables.go @@ -0,0 +1,45 @@ +package testserver + +import ( + "encoding/json" + "fmt" + + "github.com/databricks/databricks-sdk-go/service/database" +) + +func (s *FakeWorkspace) SyncedDatabaseTableCreate(req Request) Response { + defer s.LockUnlock()() + + syncedDatabaseTable := database.SyncedDatabaseTable{} + err := json.Unmarshal(req.Body, &syncedDatabaseTable) + if err != nil { + return Response{ + Body: fmt.Sprintf("cannot unmarshal request body: %v", err), + StatusCode: 400, + } + } + + // check that the database instance exists if specified: + if syncedDatabaseTable.DatabaseInstanceName != "" { + found := false + for _, instance := range s.DatabaseInstances { + if instance.Name == syncedDatabaseTable.DatabaseInstanceName { + fmt.Printf("Found database instance: %s\n", instance.Name) + found = true + break + } + } + if !found { + return Response{ + Body: fmt.Sprintf("database instance with name '%s' not found", syncedDatabaseTable.DatabaseInstanceName), + StatusCode: 404, + } + } + } + + s.SyncedDatabaseTables[syncedDatabaseTable.Name] = syncedDatabaseTable + + return Response{ + Body: syncedDatabaseTable, + } +}