Add support for Lakebase synced database tables

NEXT_CHANGELOG.md

@@ -16,6 +16,7 @@ See more details here: ([#3225](https://github.com/databricks/cli/pull/3225))
 
 ### Bundles
 * [Breaking Change] Remove deprecated path fallback mechanism for jobs and pipelines ([#3225](https://github.com/databricks/cli/pull/3225))
+* Add support for Lakebase synced database tables in DABs ([#3467](https://github.com/databricks/cli/pull/3467))
 * Rename Delta Live Tables to Lakeflow Declarative Pipelines in the default-python template ([#3476](https://github.com/databricks/cli/pull/3476)).
 
 ### API Changes

acceptance/bundle/deploy/lakebase/synced-database-table/databricks.yml.tmpl

@@ -0,0 +1,24 @@
+bundle:
+  name: deploy-lakebase-synced-table-$UNIQUE_NAME
+
+resources:
+  database_instances:
+    my_instance:
+      name: test-database-instance-$UNIQUE_NAME
+      capacity: CU_1
+  database_catalogs:
+    my_catalog:
+      database_instance_name: ${resources.database_instances.my_instance.name}
+      database_name: my_database
+      name: my_catalog_$UNIQUE_NAME
+      create_database_if_not_exists: true
+  synced_database_tables:
+    my_synced_table:
+      name: ${resources.database_catalogs.my_catalog.name}.${resources.database_catalogs.my_catalog.database_name}.my_synced_table
+      database_instance_name: ${resources.database_instances.my_instance.name}
+      logical_database_name: ${resources.database_catalogs.my_catalog.database_name}
+      spec:
+        source_table_full_name: "samples.nyctaxi.trips"
+        scheduling_policy: SNAPSHOT
+        primary_key_columns:
+          - tpep_pickup_datetime

acceptance/bundle/deploy/lakebase/synced-database-table/out.test.toml

@@ -0,0 +1,9 @@
+Local = true
+Cloud = true
+RequiresUnityCatalog = true
+
+[CloudEnvs]
+  gcp = false
+
+[EnvMatrix]
+  DATABRICKS_CLI_DEPLOYMENT = ["terraform"]

acceptance/bundle/deploy/lakebase/synced-database-table/output.txt

@@ -0,0 +1,66 @@
+
+>>> [CLI] bundle validate
+Name: deploy-lakebase-synced-table-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default
+
+Validation OK!
+
+>>> [CLI] bundle summary
+Name: deploy-lakebase-synced-table-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default
+Resources:
+  Database catalogs:
+    my_catalog:
+      Name: my_catalog_[UNIQUE_NAME]
+      URL:  [DATABRICKS_URL]/explore/data/my_catalog_[UNIQUE_NAME]
+  Database instances:
+    my_instance:
+      Name: test-database-instance-[UNIQUE_NAME]
+      URL:  (not deployed)
+  Synced database tables:
+    my_synced_table:
+      Name: ${databricks_database_database_catalog.my_catalog.name}.${databricks_database_database_catalog.my_catalog.database_name}.my_synced_table
+      URL:  [DATABRICKS_URL]/explore/data/$%7Bdatabricks_database_database_catalog.my_catalog.name%7D.$%7Bdatabricks_database_database_catalog.my_catalog.database_name%7D.my_synced_table
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+
+>>> [CLI] bundle summary
+Name: deploy-lakebase-synced-table-[UNIQUE_NAME]
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default
+Resources:
+  Database catalogs:
+    my_catalog:
+      Name: my_catalog_[UNIQUE_NAME]
+      URL:  [DATABRICKS_URL]/explore/data/my_catalog_[UNIQUE_NAME]
+  Database instances:
+    my_instance:
+      Name: test-database-instance-[UNIQUE_NAME]
+      URL:  [DATABRICKS_URL]/compute/database-instances/test-database-instance-[UNIQUE_NAME]
+  Synced database tables:
+    my_synced_table:
+      Name: ${resources.database_catalogs.my_catalog.name}.${resources.database_catalogs.my_catalog.database_name}.my_synced_table
+      URL:  [DATABRICKS_URL]/explore/data/$%7Bresources.database_catalogs.my_catalog.name%7D.$%7Bresources.database_catalogs.my_catalog.database_name%7D.my_synced_table
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete database_catalog my_catalog
+  delete database_instance my_instance
+  delete synced_database_table my_synced_table
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/deploy-lakebase-synced-table-[UNIQUE_NAME]/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/deploy/lakebase/synced-database-table/script

@@ -0,0 +1,11 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+  trace $CLI bundle destroy --auto-approve
+}
+ trap cleanup EXIT
+
+trace $CLI bundle validate
+trace $CLI bundle summary
+trace $CLI bundle deploy
+trace $CLI bundle summary

acceptance/bundle/deploy/lakebase/synced-database-table/test.toml

@@ -0,0 +1,13 @@
+Local = true
+Cloud = true
+
+RecordRequests = false
+
+[EnvMatrix]
+DATABRICKS_CLI_DEPLOYMENT = ["terraform"]
+
+[[Repls]]
+# clean up ?o=<num> suffix after URL since not all workspaces have that
+Old = '\?o=\[(NUMID|ALPHANUMID)\]'
+New = ''
+Order = 1000

acceptance/internal/handlers.go

@@ -526,4 +526,16 @@ func addDefaultHandlers(server *testserver.Server) {
 	server.Handle("DELETE", "/api/2.0/database/catalogs/{name}", func(req testserver.Request) any {
 		return testserver.MapDelete(req.Workspace, req.Workspace.DatabaseCatalogs, req.Vars["name"])
 	})
+
+	server.Handle("POST", "/api/2.0/database/synced_tables", func(req testserver.Request) any {
+		return req.Workspace.SyncedDatabaseTableCreate(req)
+	})
+
+	server.Handle("GET", "/api/2.0/database/synced_tables/{name}", func(req testserver.Request) any {
+		return testserver.MapGet(req.Workspace, req.Workspace.SyncedDatabaseTables, req.Vars["name"])
+	})
+
+	server.Handle("DELETE", "/api/2.0/database/synced_tables/{name}", func(req testserver.Request) any {
+		return testserver.MapDelete(req.Workspace, req.Workspace.SyncedDatabaseTables, req.Vars["name"])
+	})
 }

bundle/config/mutator/resourcemutator/apply_bundle_permissions.go

@@ -15,7 +15,7 @@ import (
 	"github.com/databricks/cli/libs/dyn/convert"
 )
 
-var unsupportedResources = []string{"clusters", "volumes", "schemas", "quality_monitors", "registered_models", "database_catalogs"}
+var unsupportedResources = []string{"clusters", "volumes", "schemas", "quality_monitors", "registered_models", "database_catalogs", "synced_database_tables"}
 
 var (
 	allowedLevels = []string{permissions.CAN_MANAGE, permissions.CAN_VIEW, permissions.CAN_RUN}

bundle/config/mutator/resourcemutator/apply_target_mode_test.go

@@ -183,6 +183,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle {
 						},
 					},
 				},
+				SyncedDatabaseTables: map[string]*resources.SyncedDatabaseTable{
+					"synced_database_table1": {
+						SyncedDatabaseTable: database.SyncedDatabaseTable{
+							Name: "synced_database_table1",
+						},
+					},
+				},
 			},
 		},
 		SyncRoot: vfs.MustNew("/Users/lennart.kats@databricks.com"),
@@ -351,7 +358,7 @@ func TestAllNonUcResourcesAreRenamed(t *testing.T) {
 				resourceType := resources.Type().Field(i).Name
 
 				// Skip resources that are not renamed
-				if resourceType == "Apps" || resourceType == "SecretScopes" || resourceType == "DatabaseInstances" || resourceType == "DatabaseCatalogs" {
+				if resourceType == "Apps" || resourceType == "SecretScopes" || resourceType == "DatabaseInstances" || resourceType == "DatabaseCatalogs" || resourceType == "SyncedDatabaseTables" {
 					continue
 				}
 

bundle/config/mutator/resourcemutator/run_as_test.go

@@ -47,6 +47,7 @@ func allResourceTypes(t *testing.T) []string {
 		"schemas",
 		"secret_scopes",
 		"sql_warehouses",
+		"synced_database_tables",
 		"volumes",
 	},
 		resourceTypes,
@@ -143,6 +144,7 @@ var allowList = []string{
 	"clusters",
 	"database_catalogs",
 	"database_instances",
+	"synced_database_tables",
 	"jobs",
 	"models",
 	"registered_models",

bundle/config/resources.go

@@ -28,6 +28,7 @@ type Resources struct {
 	SqlWarehouses         map[string]*resources.SqlWarehouse         `json:"sql_warehouses,omitempty"`
 	DatabaseInstances     map[string]*resources.DatabaseInstance     `json:"database_instances,omitempty"`
 	DatabaseCatalogs      map[string]*resources.DatabaseCatalog      `json:"database_catalogs,omitempty"`
+	SyncedDatabaseTables  map[string]*resources.SyncedDatabaseTable  `json:"synced_database_tables,omitempty"`
 }
 
 type ConfigResource interface {
@@ -94,6 +95,7 @@ func (r *Resources) AllResources() []ResourceGroup {
 		collectResourceMap(descriptions["sql_warehouses"], r.SqlWarehouses),
 		collectResourceMap(descriptions["database_instances"], r.DatabaseInstances),
 		collectResourceMap(descriptions["database_catalogs"], r.DatabaseCatalogs),
+		collectResourceMap(descriptions["synced_database_tables"], r.SyncedDatabaseTables),
 	}
 }
 
@@ -189,6 +191,12 @@ func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error)
 		}
 	}
 
+	for k := range r.SyncedDatabaseTables {
+		if k == key {
+			found = append(found, r.SyncedDatabaseTables[k])
+		}
+	}
+
 	if len(found) == 0 {
 		return nil, fmt.Errorf("no such resource: %s", key)
 	}
@@ -223,5 +231,6 @@ func SupportedResources() map[string]resources.ResourceDescription {
 		"sql_warehouses":          (&resources.SqlWarehouse{}).ResourceDescription(),
 		"database_instances":      (&resources.DatabaseInstance{}).ResourceDescription(),
 		"database_catalogs":       (&resources.DatabaseCatalog{}).ResourceDescription(),
+		"synced_database_tables":  (&resources.SyncedDatabaseTable{}).ResourceDescription(),
 	}
 }

bundle/config/resources/synced_database_table.go

@@ -0,0 +1,53 @@
+package resources
+
+import (
+	"context"
+	"net/url"
+
+	"github.com/databricks/cli/libs/log"
+
+	"github.com/databricks/databricks-sdk-go"
+	"github.com/databricks/databricks-sdk-go/service/database"
+)
+
+type SyncedDatabaseTable struct {
+	ID             string         `json:"id,omitempty" bundle:"readonly"`
+	URL            string         `json:"url,omitempty" bundle:"internal"`
+	ModifiedStatus ModifiedStatus `json:"modified_status,omitempty" bundle:"internal"`
+
+	database.SyncedDatabaseTable
+}
+
+func (s *SyncedDatabaseTable) Exists(ctx context.Context, w *databricks.WorkspaceClient, name string) (bool, error) {
+	_, err := w.Database.GetSyncedDatabaseTable(ctx, database.GetSyncedDatabaseTableRequest{Name: name})
+	if err != nil {
+		log.Debugf(ctx, "synced database table %s does not exist", name)
+		return false, err
+	}
+	return true, nil
+}
+
+func (s *SyncedDatabaseTable) ResourceDescription() ResourceDescription {
+	return ResourceDescription{
+		SingularName:  "synced_database_table",
+		PluralName:    "synced_database_tables",
+		SingularTitle: "Synced database table",
+		PluralTitle:   "Synced database tables",
+	}
+}
+
+func (s *SyncedDatabaseTable) GetName() string {
+	return s.Name
+}
+
+func (s *SyncedDatabaseTable) GetURL() string {
+	return s.URL
+}
+
+func (s *SyncedDatabaseTable) InitializeURL(baseURL url.URL) {
+	if s.Name == "" {
+		return
+	}
+	baseURL.Path = "explore/data/" + s.Name
+	s.URL = baseURL.String()
+}

bundle/config/resources_test.go

@@ -190,6 +190,11 @@ func TestResourcesBindSupport(t *testing.T) {
 				DatabaseCatalog: database.DatabaseCatalog{},
 			},
 		},
+		SyncedDatabaseTables: map[string]*resources.SyncedDatabaseTable{
+			"my_synced_database_table": {
+				SyncedDatabaseTable: database.SyncedDatabaseTable{},
+			},
+		},
 	}
 	unbindableResources := map[string]bool{"model": true}
 
@@ -212,6 +217,7 @@ func TestResourcesBindSupport(t *testing.T) {
 	m.GetMockWarehousesAPI().EXPECT().GetById(mock.Anything, mock.Anything).Return(nil, nil)
 	m.GetMockDatabaseAPI().EXPECT().GetDatabaseInstance(mock.Anything, mock.Anything).Return(nil, nil)
 	m.GetMockDatabaseAPI().EXPECT().GetDatabaseCatalog(mock.Anything, mock.Anything).Return(nil, nil)
+	m.GetMockDatabaseAPI().EXPECT().GetSyncedDatabaseTable(mock.Anything, mock.Anything).Return(nil, nil)
 
 	allResources := supportedResources.AllResources()
 	for _, group := range allResources {

bundle/deploy/terraform/pkg.go

@@ -118,6 +118,7 @@ var GroupToTerraformName = map[string]string{
 	"sql_warehouses":          "databricks_sql_endpoint",
 	"database_instances":      "databricks_database_instance",
 	"database_catalogs":       "databricks_database_database_catalog",
+	"synced_database_tables":  "databricks_database_synced_database_table",
 }
 
 var TerraformToGroupName = func() map[string]string {

bundle/deploy/terraform/tfdyn/convert_synced_database_table.go

@@ -0,0 +1,28 @@
+package tfdyn
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle/internal/tf/schema"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/convert"
+	"github.com/databricks/cli/libs/log"
+	"github.com/databricks/databricks-sdk-go/service/database"
+)
+
+type syncedDatabaseTableConverter struct{}
+
+func (s syncedDatabaseTableConverter) Convert(ctx context.Context, key string, vin dyn.Value, out *schema.Resources) error {
+	// Normalize the output value to the target schema.
+	vout, diags := convert.Normalize(database.SyncedDatabaseTable{}, vin)
+	for _, diag := range diags {
+		log.Debugf(ctx, "synced database table normalization diagnostic: %s", diag.Summary)
+	}
+	out.DatabaseSyncedDatabaseTable[key] = vout.AsAny()
+
+	return nil
+}
+
+func init() {
+	registerConverter("synced_database_tables", syncedDatabaseTableConverter{})
+}

bundle/internal/schema/annotations.yml

@@ -225,6 +225,9 @@ github.com/databricks/cli/bundle/config.Resources:
       The SQL warehouse definitions for the bundle, where each key is the name of the warehouse.
     "markdown_description": |-
       The SQL warehouse definitions for the bundle, where each key is the name of the warehouse. See [\_](/dev-tools/bundles/resources.md#sql_warehouses).
+  "synced_database_tables":
+    "description": |-
+      PLACEHOLDER
   "volumes":
     "description": |-
       The volume definitions for the bundle, where each key is the name of the volume.
@@ -700,6 +703,31 @@ github.com/databricks/cli/bundle/config/resources.SqlWarehousePermission:
   "user_name":
     "description": |-
       PLACEHOLDER
+github.com/databricks/cli/bundle/config/resources.SyncedDatabaseTable:
+  "data_synchronization_status":
+    "description": |-
+      PLACEHOLDER
+  "database_instance_name":
+    "description": |-
+      PLACEHOLDER
+  "effective_database_instance_name":
+    "description": |-
+      PLACEHOLDER
+  "effective_logical_database_name":
+    "description": |-
+      PLACEHOLDER
+  "logical_database_name":
+    "description": |-
+      PLACEHOLDER
+  "name":
+    "description": |-
+      PLACEHOLDER
+  "spec":
+    "description": |-
+      PLACEHOLDER
+  "unity_catalog_provisioning_state":
+    "description": |-
+      PLACEHOLDER
 github.com/databricks/cli/bundle/config/resources.VolumeGrant:
   "principal":
     "description": |-