WIP merge wait methods in

acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.dashboardrequests.direct.txt

@@ -0,0 +1,39 @@
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace/mkdirs",
+  "body": {
+    "path": "/Workspace/Users/[USERNAME]/.bundle/publish-failure-cleans-up-dashboard/default/artifacts/.internal"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace/mkdirs",
+  "body": {
+    "path": "/Workspace/Users/[USERNAME]/.bundle/publish-failure-cleans-up-dashboard/default/files"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/workspace/mkdirs",
+  "body": {
+    "path": "/Workspace/Users/[USERNAME]/.bundle/publish-failure-cleans-up-dashboard/default/resources"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/lakeview/dashboards",
+  "body": {
+    "display_name": "my dashboard",
+    "parent_path": "/Workspace/Users/[USERNAME]/.bundle/publish-failure-cleans-up-dashboard/default/resources",
+    "serialized_dashboard": "{\"pages\":[{\"name\":\"test-page\",\"displayName\":\"Test Dashboard\"}]}\n",
+    "warehouse_id": "doesnotexist"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published",
+  "body": {
+    "embed_credentials": false,
+    "warehouse_id": "doesnotexist"
+  }
+}

acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.deploy.direct.txt

@@ -9,5 +9,6 @@ HTTP Status: 400 Bad Request
 API error_code: RESOURCE_DOES_NOT_EXIST
 API message: Warehouse doesnotexist does not exist
 
+Updating deployment state...
 
 Exit code: 1

acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.summary.direct.txt

@@ -0,0 +1,12 @@
+
+>>> [CLI] bundle summary
+Name: publish-failure-cleans-up-dashboard
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/publish-failure-cleans-up-dashboard/default
+Resources:
+  Dashboards:
+    dashboard1:
+      Name: my dashboard
+      URL:  [DATABRICKS_URL]/dashboardsv3/[DASHBOARD_ID]/published?[WSPARAM]=[NUMID]

acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/out.summary.terraform.txt

@@ -0,0 +1,12 @@
+
+>>> [CLI] bundle summary
+Name: publish-failure-cleans-up-dashboard
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/publish-failure-cleans-up-dashboard/default
+Resources:
+  Dashboards:
+    dashboard1:
+      Name: my dashboard
+      URL:  (not deployed)

acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/output.txt

@@ -1,12 +0,0 @@
-
->>> [CLI] bundle summary
-Name: publish-failure-cleans-up-dashboard
-Target: default
-Workspace:
-  User: [USERNAME]
-  Path: /Workspace/Users/[USERNAME]/.bundle/publish-failure-cleans-up-dashboard/default
-Resources:
-  Dashboards:
-    dashboard1:
-      Name: my dashboard
-      URL:  (not deployed)

acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/script

@@ -4,9 +4,9 @@ envsubst < databricks.yml.tmpl > databricks.yml
 # Deploy the dashboard. The dashboard will be created but publish will fail because the warehouse does not exist.
 errcode trace $CLI bundle deploy &> out.deploy.$DATABRICKS_BUNDLE_ENGINE.txt
 
-trace $CLI bundle summary
+# After publish failure the dashboard draft should be in state (direct) or cleaned up (terraform).
+trace $CLI bundle summary >> out.summary.$DATABRICKS_BUNDLE_ENGINE.txt 2>&1
 
-# API should record a DELETE call to clean up the draft dashboard that was not published.
-# Request sequence is identical across terraform and direct modes.
+# API request sequence differs between engines (direct: no DELETE; terraform: DELETE to clean up).
 unset MSYS_NO_PATHCONV
-print_requests.py //lakeview/dashboards //workspace/mkdirs > out.dashboardrequests.txt
+print_requests.py //lakeview/dashboards //workspace/mkdirs > out.dashboardrequests.$DATABRICKS_BUNDLE_ENGINE.txt

acceptance/bundle/resources/dashboards/publish-failure-cleans-up-dashboard/test.toml

@@ -10,3 +10,8 @@ Response.Body = '{"error_code": "RESOURCE_DOES_NOT_EXIST", "message": "Warehouse
 [[Repls]]
 Old = "[0-9a-f]{32}"
 New = "[DASHBOARD_ID]"
+
+# Dashboard published URLs use ?o= (local testserver) or ?w= (cloud) for the workspace/org ID.
+[[Repls]]
+Old = '\?[ow]=\d+'
+New = "?[WSPARAM]=[NUMID]"

acceptance/bundle/resources/dashboards/publish-failure-retry/dashboard.lvdash.json

@@ -0,0 +1 @@
+{"pages":[{"name":"test-page","displayName":"Test Dashboard"}]}

acceptance/bundle/resources/dashboards/publish-failure-retry/databricks.yml.tmpl

@@ -0,0 +1,9 @@
+bundle:
+  name: publish-failure-retry
+
+resources:
+  dashboards:
+    dashboard1:
+      display_name: my dashboard
+      warehouse_id: someid
+      file_path: ./dashboard.lvdash.json

acceptance/bundle/resources/dashboards/publish-failure-retry/output.txt

@@ -0,0 +1,64 @@
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/publish-failure-retry/default/files...
+Deploying resources...
+Error: cannot create resources.dashboards.dashboard1: Fault injected by test. (400 INJECTED)
+
+Endpoint: POST [DATABRICKS_URL]/api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published
+HTTP Status: 400 Bad Request
+API error_code: INJECTED
+API message: Fault injected by test.
+
+Updating deployment state...
+
+Exit code: 1
+
+>>> [CLI] bundle summary
+Name: publish-failure-retry
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/publish-failure-retry/default
+Resources:
+  Dashboards:
+    dashboard1:
+      Name: my dashboard
+      URL:  [DATABRICKS_URL]/dashboardsv3/[DASHBOARD_ID]/published?[WSPARAM]=[NUMID]
+
+>>> [CLI] bundle plan
+update dashboards.dashboard1
+
+Plan: 0 to add, 1 to change, 0 to delete, 0 unchanged
+
+>>> [CLI] bundle deploy
+Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/publish-failure-retry/default/files...
+Deploying resources...
+Updating deployment state...
+Deployment complete!
+{
+  "method": "PATCH",
+  "path": "/api/2.0/lakeview/dashboards/[DASHBOARD_ID]",
+  "body": {
+    "display_name": "my dashboard",
+    "parent_path": "/Workspace/Users/[USERNAME]/.bundle/publish-failure-retry/default/resources",
+    "serialized_dashboard": "{\"pages\":[{\"name\":\"test-page\",\"displayName\":\"Test Dashboard\"}]}\n",
+    "warehouse_id": "someid"
+  }
+}
+{
+  "method": "POST",
+  "path": "/api/2.0/lakeview/dashboards/[DASHBOARD_ID]/published",
+  "body": {
+    "embed_credentials": false,
+    "warehouse_id": "someid"
+  }
+}
+
+>>> [CLI] bundle destroy --auto-approve
+The following resources will be deleted:
+  delete resources.dashboards.dashboard1
+
+All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/publish-failure-retry/default
+
+Deleting files...
+Destroy complete!

acceptance/bundle/resources/dashboards/publish-failure-retry/script

@@ -0,0 +1,34 @@
+envsubst < databricks.yml.tmpl > databricks.yml
+
+cleanup() {
+    trace $CLI bundle destroy --auto-approve
+    rm -f out.requests.txt
+}
+trap cleanup EXIT
+
+# unset MSYS_NO_PATHCONV so MSYS2 converts the script path to a Windows path
+# when invoking the Python interpreter (required for fault.py to be found on Windows).
+unset MSYS_NO_PATHCONV
+
+# Inject a single publish failure so the first deploy creates the dashboard
+# draft but fails to publish it.
+fault.py "POST /api/2.0/lakeview/dashboards/*" 400 0 1
+
+# First deploy: dashboard is created and saved to state, but publish fails.
+errcode trace $CLI bundle deploy
+
+# Dashboard should be in state (tracked) despite the publish failure.
+trace $CLI bundle summary
+
+# Plan should show that publishing is still needed.
+trace $CLI bundle plan
+
+# Discard first-deploy requests so the output only contains second-deploy
+# calls, making it easy to confirm no CREATE was issued.
+rm out.requests.txt
+
+# Second deploy: fault is gone; must publish the existing draft, not create a new one.
+trace $CLI bundle deploy
+
+# Confirm: second deploy issued an UPDATE and a PUBLISH call but no CREATE.
+print_requests.py //lakeview/dashboards

acceptance/bundle/resources/dashboards/publish-failure-retry/test.toml

@@ -0,0 +1,17 @@
+Cloud = false
+Local = true
+RecordRequests = true
+
+# Only run with the direct engine: the test verifies direct engine's SaveState
+# behavior (draft persists on publish failure and is re-published on retry).
+[EnvMatrix]
+DATABRICKS_BUNDLE_ENGINE = ["direct"]
+
+[[Repls]]
+Old = "[0-9a-f]{32}"
+New = "[DASHBOARD_ID]"
+
+# Dashboard published URLs use ?o= (local testserver) or ?w= (cloud) for the workspace/org ID.
+[[Repls]]
+Old = '\?[ow]=\d+'
+New = "?[WSPARAM]=[NUMID]"

bundle/direct/apply.go

@@ -51,6 +51,10 @@ func (d *DeploymentUnit) Deploy(ctx context.Context, db *dstate.DeploymentState,
 }
 
 func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, newState any) error {
+	engine := dresources.NewEngine(d.Adapter.StateType(), func(id string, x any) error {
+		return db.SaveState(d.ResourceKey, id, x, d.DependsOn)
+	})
+
 	var newID string
 	var remoteState any
 	_, err := retryWith(ctx, func(err error) bool {
@@ -59,7 +63,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState,
 		return ok && isTransient(ctx, err)
 	}, func() (struct{}, error) {
 		var e error
-		newID, remoteState, e = d.Adapter.DoCreate(ctx, newState)
+		newID, remoteState, e = d.Adapter.DoCreate(ctx, engine, newState)
 		return struct{}{}, e
 	})
 	err = dresources.UnwrapRetrySafe(err)
@@ -80,18 +84,6 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState,
 		return fmt.Errorf("saving state after creating id=%s: %w", newID, err)
 	}
 
-	waitRemoteState, err := retryOnTransient(ctx, func() (any, error) {
-		return d.Adapter.WaitAfterCreate(ctx, newID, newState)
-	})
-	if err != nil {
-		return fmt.Errorf("waiting after creating id=%s: %w", newID, err)
-	}
-
-	err = d.SetRemoteState(waitRemoteState)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -134,8 +126,11 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState,
 		return fmt.Errorf("internal error: DoUpdate not implemented for resource %s", d.ResourceKey)
 	}
 
+	engine := dresources.NewEngine(d.Adapter.StateType(), func(_ string, x any) error {
+		return db.SaveState(d.ResourceKey, id, x, d.DependsOn)
+	})
 	remoteState, err := retryOnTransient(ctx, func() (any, error) {
-		return d.Adapter.DoUpdate(ctx, id, newState, planEntry)
+		return d.Adapter.DoUpdate(ctx, engine, id, newState, planEntry)
 	})
 	if err != nil {
 		return fmt.Errorf("updating id=%s: %w", id, err)
@@ -151,19 +146,6 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState,
 		return fmt.Errorf("saving state id=%s: %w", id, err)
 	}
 
-	waitRemoteState, err := retryOnTransient(ctx, func() (any, error) {
-		return d.Adapter.WaitAfterUpdate(ctx, id, newState)
-	})
-	if err != nil {
-		return fmt.Errorf("waiting after updating id=%s: %w", id, err)
-	}
-
-	// Update remote state with the result from wait operation
-	err = d.SetRemoteState(waitRemoteState)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 
@@ -195,19 +177,6 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment
 		return fmt.Errorf("saving state id=%s: %w", oldID, err)
 	}
 
-	waitRemoteState, err := retryOnTransient(ctx, func() (any, error) {
-		return d.Adapter.WaitAfterUpdate(ctx, newID, newState)
-	})
-	if err != nil {
-		return fmt.Errorf("waiting after updating id=%s: %w", newID, err)
-	}
-
-	// Update remote state with the result from wait operation
-	err = d.SetRemoteState(waitRemoteState)
-	if err != nil {
-		return err
-	}
-
 	return nil
 }
 

bundle/direct/dresources/README.md

@@ -34,9 +34,9 @@ Do **not** derive update mask field names from `entry.Changes`. The paths in `en
 
 If a resource has fields that must not be sent in updates (deploy-only, lifecycle-only, etc.), document them explicitly with a `var` block and a comment explaining each exclusion.
 
-## Async APIs: WaitAfterCreate / WaitAfterUpdate
+## Async APIs
 
-For resources whose create or update is asynchronous (the resource is not immediately ready after the call returns), implement `WaitAfterCreate` and/or `WaitAfterUpdate` instead of polling inline inside DoCreate/DoUpdate. These are the correct extension points in the framework, and polling inline bypasses state persistence timing.
+For resources whose create or update is asynchronous, poll inline inside `DoCreate`/`DoUpdate` after the initial API call. To prevent orphaning if deployment is interrupted during a long wait, call `engine.SetID(id)` then `engine.SaveState(config)` immediately after the resource is created and before any waiting. The framework provides a `*Engine` as the second argument to both methods.
 
 ## Slice ordering: KeyedSlices