From 73c914a7ba011bb88ce04d68e83f1f9a825947c1 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Nov 2025 14:15:59 +0100 Subject: [PATCH 1/4] add tasks-reorder --- .../whl_dynamic/out.plan_update.direct.json | 4 +-- .../bundle/migrate/basic/out.plan_update.json | 2 +- .../delete_task/out.plan_update.direct.json | 2 +- .../jobs/tasks-reorder/databricks.yml | 14 ++++++++++ .../jobs/tasks-reorder/databricks.yml.tmpl | 18 +++++++++++++ .../resources/jobs/tasks-reorder/foo.py | 1 + .../jobs/tasks-reorder/out.test.toml | 5 ++++ .../resources/jobs/tasks-reorder/output.txt | 26 +++++++++++++++++++ .../resources/jobs/tasks-reorder/script | 12 +++++++++ .../resources/jobs/tasks-reorder/task_aa.py | 1 + .../resources/jobs/tasks-reorder/task_bb.py | 1 + .../resources/jobs/tasks-reorder/test.toml | 2 ++ bundle/direct/dresources/job.go | 10 +++++++ 13 files changed, 94 insertions(+), 4 deletions(-) create mode 100644 acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml create mode 100644 acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/jobs/tasks-reorder/foo.py create mode 100644 acceptance/bundle/resources/jobs/tasks-reorder/out.test.toml create mode 100644 acceptance/bundle/resources/jobs/tasks-reorder/output.txt create mode 100644 acceptance/bundle/resources/jobs/tasks-reorder/script create mode 100644 acceptance/bundle/resources/jobs/tasks-reorder/task_aa.py create mode 100644 acceptance/bundle/resources/jobs/tasks-reorder/task_bb.py create mode 100644 acceptance/bundle/resources/jobs/tasks-reorder/test.toml diff --git a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json index d1a7465413..4baab1f343 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json +++ b/acceptance/bundle/artifacts/whl_dynamic/out.plan_update.direct.json @@ -150,10 +150,10 @@ "environments[0].spec.dependencies[0]": { "action": "update" }, - "tasks[1].for_each_task.task.libraries[0].whl": { + "tasks[task_key='TestTask'].for_each_task.task.libraries[0].whl": { "action": "update" }, - "tasks[1].libraries[0].whl": { + "tasks[task_key='TestTask'].libraries[0].whl": { "action": "update" } }, diff --git a/acceptance/bundle/migrate/basic/out.plan_update.json b/acceptance/bundle/migrate/basic/out.plan_update.json index 98f08f8616..46f5f0e219 100644 --- a/acceptance/bundle/migrate/basic/out.plan_update.json +++ b/acceptance/bundle/migrate/basic/out.plan_update.json @@ -67,7 +67,7 @@ "action": "skip", "reason": "server_side_default" }, - "tasks[0].notebook_task.source": { + "tasks[task_key='main'].notebook_task.source": { "action": "skip", "reason": "server_side_default" }, diff --git a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json index 7c30a2de28..5c809bfe39 100644 --- a/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json +++ b/acceptance/bundle/resources/jobs/delete_task/out.plan_update.direct.json @@ -96,7 +96,7 @@ }, "changes": { "local": { - "tasks": { + "tasks[task_key='TestTask1']": { "action": "update" } }, diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml b/acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml new file mode 100644 index 0000000000..4461f375b2 --- /dev/null +++ b/acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml @@ -0,0 +1,14 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + jobs: + foo: + tasks: + - task_key: task_bb + notebook_task: + notebook_path: task_bb.py + - task_key: task_aa + notebook_task: + notebook_path: task_aa.py + diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml.tmpl new file mode 100644 index 0000000000..b185bf952d --- /dev/null +++ b/acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml.tmpl @@ -0,0 +1,18 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + jobs: + foo: + tasks: + - task_key: task_bb + notebook_task: + notebook_path: task_bb.py + - task_key: task_aa + notebook_task: + notebook_path: task_aa.py + bar: + tasks: + - task_key: task_cc + notebook_task: + notebook_path: ${resources.jobs.foo.tasks[1].notebook_task.notebook_path} diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/foo.py b/acceptance/bundle/resources/jobs/tasks-reorder/foo.py new file mode 100644 index 0000000000..1645e04b1d --- /dev/null +++ b/acceptance/bundle/resources/jobs/tasks-reorder/foo.py @@ -0,0 +1 @@ +# Databricks notebook source diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/out.test.toml b/acceptance/bundle/resources/jobs/tasks-reorder/out.test.toml new file mode 100644 index 0000000000..d560f1de04 --- /dev/null +++ b/acceptance/bundle/resources/jobs/tasks-reorder/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/output.txt b/acceptance/bundle/resources/jobs/tasks-reorder/output.txt new file mode 100644 index 0000000000..154eb52423 --- /dev/null +++ b/acceptance/bundle/resources/jobs/tasks-reorder/output.txt @@ -0,0 +1,26 @@ + +>>> [CLI] bundle plan +create jobs.foo + +Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-$UNIQUE_NAME/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-$UNIQUE_NAME/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle plan +Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/script b/acceptance/bundle/resources/jobs/tasks-reorder/script new file mode 100644 index 0000000000..797fee17cb --- /dev/null +++ b/acceptance/bundle/resources/jobs/tasks-reorder/script @@ -0,0 +1,12 @@ +trace $CLI bundle plan +trace $CLI bundle deploy +trace $CLI bundle plan | contains.py "0 to add, 0 to change, 0 to delete" + +update_file.py databricks.yml task_aa task_XX +update_file.py databricks.yml task_bb task_aa +update_file.py databricks.yml task_XX task_bb + +trace $CLI bundle plan + +trace $CLI bundle deploy +trace $CLI bundle plan | contains.py "0 to add, 0 to change, 0 to delete" diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/task_aa.py b/acceptance/bundle/resources/jobs/tasks-reorder/task_aa.py new file mode 100644 index 0000000000..1645e04b1d --- /dev/null +++ b/acceptance/bundle/resources/jobs/tasks-reorder/task_aa.py @@ -0,0 +1 @@ +# Databricks notebook source diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/task_bb.py b/acceptance/bundle/resources/jobs/tasks-reorder/task_bb.py new file mode 100644 index 0000000000..1645e04b1d --- /dev/null +++ b/acceptance/bundle/resources/jobs/tasks-reorder/task_bb.py @@ -0,0 +1 @@ +# Databricks notebook source diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/test.toml b/acceptance/bundle/resources/jobs/tasks-reorder/test.toml new file mode 100644 index 0000000000..e1290bad9f --- /dev/null +++ b/acceptance/bundle/resources/jobs/tasks-reorder/test.toml @@ -0,0 +1,2 @@ +RecordRequests = false +Ignore = [".databricks"] diff --git a/bundle/direct/dresources/job.go b/bundle/direct/dresources/job.go index 7d64217d96..c3a668409e 100644 --- a/bundle/direct/dresources/job.go +++ b/bundle/direct/dresources/job.go @@ -29,6 +29,16 @@ func (*ResourceJob) RemapState(jobs *jobs.Job) *jobs.JobSettings { return jobs.Settings } +func getTaskKey(x jobs.Task) (string, string) { + return "task_key", x.TaskKey +} + +func (*ResourceJob) KeyedSlices() map[string]any { + return map[string]any{ + "tasks": getTaskKey, + } +} + func (r *ResourceJob) DoRead(ctx context.Context, id string) (*jobs.Job, error) { idInt, err := parseJobID(id) if err != nil { From 84cf19f57594fd1acd15553287e0cd20d21633ad Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Nov 2025 14:17:44 +0100 Subject: [PATCH 2/4] update --- .../databricks.yml | 0 .../foo.py | 0 .../out.test.toml | 0 .../output.txt | 0 .../script | 0 .../task_aa.py | 0 .../task_bb.py | 0 .../test.toml | 0 .../jobs/tasks-reorder/databricks.yml.tmpl | 18 ------------------ 9 files changed, 18 deletions(-) rename acceptance/bundle/resources/jobs/{tasks-reorder => tasks-reorder-locally}/databricks.yml (100%) rename acceptance/bundle/resources/jobs/{tasks-reorder => tasks-reorder-locally}/foo.py (100%) rename acceptance/bundle/resources/jobs/{tasks-reorder => tasks-reorder-locally}/out.test.toml (100%) rename acceptance/bundle/resources/jobs/{tasks-reorder => tasks-reorder-locally}/output.txt (100%) rename acceptance/bundle/resources/jobs/{tasks-reorder => tasks-reorder-locally}/script (100%) rename acceptance/bundle/resources/jobs/{tasks-reorder => tasks-reorder-locally}/task_aa.py (100%) rename acceptance/bundle/resources/jobs/{tasks-reorder => tasks-reorder-locally}/task_bb.py (100%) rename acceptance/bundle/resources/jobs/{tasks-reorder => tasks-reorder-locally}/test.toml (100%) delete mode 100644 acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml.tmpl diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml b/acceptance/bundle/resources/jobs/tasks-reorder-locally/databricks.yml similarity index 100% rename from acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml rename to acceptance/bundle/resources/jobs/tasks-reorder-locally/databricks.yml diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/foo.py b/acceptance/bundle/resources/jobs/tasks-reorder-locally/foo.py similarity index 100% rename from acceptance/bundle/resources/jobs/tasks-reorder/foo.py rename to acceptance/bundle/resources/jobs/tasks-reorder-locally/foo.py diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/out.test.toml b/acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml similarity index 100% rename from acceptance/bundle/resources/jobs/tasks-reorder/out.test.toml rename to acceptance/bundle/resources/jobs/tasks-reorder-locally/out.test.toml diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/output.txt b/acceptance/bundle/resources/jobs/tasks-reorder-locally/output.txt similarity index 100% rename from acceptance/bundle/resources/jobs/tasks-reorder/output.txt rename to acceptance/bundle/resources/jobs/tasks-reorder-locally/output.txt diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/script b/acceptance/bundle/resources/jobs/tasks-reorder-locally/script similarity index 100% rename from acceptance/bundle/resources/jobs/tasks-reorder/script rename to acceptance/bundle/resources/jobs/tasks-reorder-locally/script diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/task_aa.py b/acceptance/bundle/resources/jobs/tasks-reorder-locally/task_aa.py similarity index 100% rename from acceptance/bundle/resources/jobs/tasks-reorder/task_aa.py rename to acceptance/bundle/resources/jobs/tasks-reorder-locally/task_aa.py diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/task_bb.py b/acceptance/bundle/resources/jobs/tasks-reorder-locally/task_bb.py similarity index 100% rename from acceptance/bundle/resources/jobs/tasks-reorder/task_bb.py rename to acceptance/bundle/resources/jobs/tasks-reorder-locally/task_bb.py diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/test.toml b/acceptance/bundle/resources/jobs/tasks-reorder-locally/test.toml similarity index 100% rename from acceptance/bundle/resources/jobs/tasks-reorder/test.toml rename to acceptance/bundle/resources/jobs/tasks-reorder-locally/test.toml diff --git a/acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml.tmpl b/acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml.tmpl deleted file mode 100644 index b185bf952d..0000000000 --- a/acceptance/bundle/resources/jobs/tasks-reorder/databricks.yml.tmpl +++ /dev/null @@ -1,18 +0,0 @@ -bundle: - name: test-bundle-$UNIQUE_NAME - -resources: - jobs: - foo: - tasks: - - task_key: task_bb - notebook_task: - notebook_path: task_bb.py - - task_key: task_aa - notebook_task: - notebook_path: task_aa.py - bar: - tasks: - - task_key: task_cc - notebook_task: - notebook_path: ${resources.jobs.foo.tasks[1].notebook_task.notebook_path} From 780a509ee858286e31316e93212c6084983c9ad2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Nov 2025 14:21:08 +0100 Subject: [PATCH 3/4] update --- .../bundle/resources/jobs/tasks-reorder-locally/databricks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/resources/jobs/tasks-reorder-locally/databricks.yml b/acceptance/bundle/resources/jobs/tasks-reorder-locally/databricks.yml index 4461f375b2..47c0a1b94a 100644 --- a/acceptance/bundle/resources/jobs/tasks-reorder-locally/databricks.yml +++ b/acceptance/bundle/resources/jobs/tasks-reorder-locally/databricks.yml @@ -1,5 +1,5 @@ bundle: - name: test-bundle-$UNIQUE_NAME + name: test-bundle resources: jobs: From 27d32a9b8f9f319ec2535a3522fec44abb278c5e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 28 Nov 2025 14:32:49 +0100 Subject: [PATCH 4/4] test update --- .../resources/jobs/tasks-reorder-locally/databricks.yml | 1 - .../bundle/resources/jobs/tasks-reorder-locally/output.txt | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/resources/jobs/tasks-reorder-locally/databricks.yml b/acceptance/bundle/resources/jobs/tasks-reorder-locally/databricks.yml index 47c0a1b94a..1ecc09b9cf 100644 --- a/acceptance/bundle/resources/jobs/tasks-reorder-locally/databricks.yml +++ b/acceptance/bundle/resources/jobs/tasks-reorder-locally/databricks.yml @@ -11,4 +11,3 @@ resources: - task_key: task_aa notebook_task: notebook_path: task_aa.py - diff --git a/acceptance/bundle/resources/jobs/tasks-reorder-locally/output.txt b/acceptance/bundle/resources/jobs/tasks-reorder-locally/output.txt index 154eb52423..66ef56be74 100644 --- a/acceptance/bundle/resources/jobs/tasks-reorder-locally/output.txt +++ b/acceptance/bundle/resources/jobs/tasks-reorder-locally/output.txt @@ -5,7 +5,7 @@ create jobs.foo Plan: 1 to add, 0 to change, 0 to delete, 0 unchanged >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-$UNIQUE_NAME/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... Updating deployment state... Deployment complete! @@ -17,7 +17,7 @@ Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged Plan: 0 to add, 0 to change, 0 to delete, 1 unchanged >>> [CLI] bundle deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle-$UNIQUE_NAME/default/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Deploying resources... Updating deployment state... Deployment complete!