diff --git a/acceptance/bundle/includes/yml_outside_root/output.txt b/acceptance/bundle/includes/yml_outside_root/output.txt index b98428269d..c8cb641b9c 100644 --- a/acceptance/bundle/includes/yml_outside_root/output.txt +++ b/acceptance/bundle/includes/yml_outside_root/output.txt @@ -16,6 +16,7 @@ Validation OK! }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, "name": "include_outside_root", "permissions": [], "queue": { diff --git a/acceptance/bundle/integration_whl/script.prepare b/acceptance/bundle/integration_whl/script.prepare new file mode 100644 index 0000000000..926c63ea89 --- /dev/null +++ b/acceptance/bundle/integration_whl/script.prepare @@ -0,0 +1,5 @@ +export PYTHONDONTWRITEBYTECODE=1 + +uv venv -q .venv +venv_activate +uv pip install -q setuptools diff --git a/acceptance/bundle/integration_whl/test.toml b/acceptance/bundle/integration_whl/test.toml index 50228c95a1..f5e229d4be 100644 --- a/acceptance/bundle/integration_whl/test.toml +++ b/acceptance/bundle/integration_whl/test.toml @@ -2,6 +2,7 @@ Local = false CloudSlow = true Ignore = [ ".databricks", + ".venv", "build", "dist", "my_test_code", diff --git a/acceptance/bundle/override/job_cluster/output.txt b/acceptance/bundle/override/job_cluster/output.txt index e6ce97acef..cc187eab54 100644 --- a/acceptance/bundle/override/job_cluster/output.txt +++ b/acceptance/bundle/override/job_cluster/output.txt @@ -18,6 +18,7 @@ } } ], + "max_concurrent_runs": 1, "name": "job", "permissions": [], "queue": { @@ -45,6 +46,7 @@ } } ], + "max_concurrent_runs": 1, "name": "job", "permissions": [], "queue": { diff --git a/acceptance/bundle/override/job_cluster_var/output.txt b/acceptance/bundle/override/job_cluster_var/output.txt index 44f8eeace6..02c048425a 100644 --- a/acceptance/bundle/override/job_cluster_var/output.txt +++ b/acceptance/bundle/override/job_cluster_var/output.txt @@ -18,6 +18,7 @@ } } ], + "max_concurrent_runs": 1, "name": "job", "permissions": [], "queue": { @@ -54,6 +55,7 @@ Validation OK! } } ], + "max_concurrent_runs": 1, "name": "job", "permissions": [], "queue": { diff --git a/acceptance/bundle/override/job_tasks/output.txt b/acceptance/bundle/override/job_tasks/output.txt index f14faa7577..c873437448 100644 --- a/acceptance/bundle/override/job_tasks/output.txt +++ b/acceptance/bundle/override/job_tasks/output.txt @@ -1,4 +1,5 @@ { + "max_concurrent_runs": 1, "name": "job", "permissions": [], "queue": { @@ -34,6 +35,7 @@ Error: file test1.py not found Exit code: 1 { + "max_concurrent_runs": 1, "name": "job", "permissions": [], "queue": { diff --git a/acceptance/bundle/paths/git_source_jobs/output.txt b/acceptance/bundle/paths/git_source_jobs/output.txt index 134d30d96a..17a50354ee 100644 --- a/acceptance/bundle/paths/git_source_jobs/output.txt +++ b/acceptance/bundle/paths/git_source_jobs/output.txt @@ -13,6 +13,7 @@ "git_provider": "gitHub", "git_url": "https://github.com/foo" }, + "max_concurrent_runs": 1, "name": "my_job", "permissions": [], "queue": { diff --git a/acceptance/bundle/paths/relative_path_outside_root/output.txt b/acceptance/bundle/paths/relative_path_outside_root/output.txt index b73e8fc3ab..df27e5c8a0 100644 --- a/acceptance/bundle/paths/relative_path_outside_root/output.txt +++ b/acceptance/bundle/paths/relative_path_outside_root/output.txt @@ -10,6 +10,7 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, "name": "include_outside_root", "permissions": [], "queue": { diff --git a/acceptance/bundle/python/mutator-ordering/output.txt b/acceptance/bundle/python/mutator-ordering/output.txt index bec6a8d506..fb7be392ba 100644 --- a/acceptance/bundle/python/mutator-ordering/output.txt +++ b/acceptance/bundle/python/mutator-ordering/output.txt @@ -18,6 +18,8 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Untitled", "permissions": [], "queue": { "enabled": true diff --git a/acceptance/bundle/python/resolve-variable/output.txt b/acceptance/bundle/python/resolve-variable/output.txt index 4b071ad10c..a49e62a123 100644 --- a/acceptance/bundle/python/resolve-variable/output.txt +++ b/acceptance/bundle/python/resolve-variable/output.txt @@ -17,6 +17,7 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, "name": "abc", "permissions": [], "queue": { diff --git a/acceptance/bundle/python/resource-loading/output.txt b/acceptance/bundle/python/resource-loading/output.txt index e8c452d3ca..fb3e809c80 100644 --- a/acceptance/bundle/python/resource-loading/output.txt +++ b/acceptance/bundle/python/resource-loading/output.txt @@ -18,6 +18,7 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, "name": "Job 1", "permissions": [], "queue": { @@ -31,6 +32,7 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, "name": "Job 2", "permissions": [], "queue": { @@ -44,6 +46,7 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, "name": "Job 3", "permissions": [], "queue": { diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index 408d7cd651..03851cecdd 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -168,11 +168,12 @@ Validation OK! + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json" }, "edit_mode": "UI_LOCKED", -@@ -59,12 +51,8 @@ +@@ -59,12 +51,9 @@ } ], - "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_job", ++ "max_concurrent_runs": 1, + "name": "project_name_[UNIQUE_NAME]_job", "permissions": [], "queue": { @@ -182,20 +183,21 @@ Validation OK! - "dev": "[USERNAME]" }, "tasks": [ -@@ -72,5 +60,5 @@ +@@ -72,5 +61,5 @@ "job_cluster_key": "job_cluster", "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook" }, "task_key": "notebook_task" -@@ -107,5 +95,4 @@ +@@ -107,5 +96,5 @@ ], "trigger": { - "pause_status": "PAUSED", ++ "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -118,21 +105,20 @@ +@@ -118,21 +107,20 @@ "project_name_[UNIQUE_NAME]_pipeline": { "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" @@ -222,7 +224,7 @@ Validation OK! + "schema": "project_name_[UNIQUE_NAME]_prod" } } -@@ -145,10 +131,10 @@ +@@ -145,10 +133,10 @@ "targets": null, "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts", @@ -317,11 +319,12 @@ Resources: + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json" }, "edit_mode": "UI_LOCKED", -@@ -60,11 +56,7 @@ +@@ -60,11 +56,8 @@ } ], - "max_concurrent_runs": 4, - "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_job", ++ "max_concurrent_runs": 1, + "name": "project_name_[UNIQUE_NAME]_job", "queue": { "enabled": true @@ -330,20 +333,21 @@ Resources: - "dev": "[USERNAME]" }, "tasks": [ -@@ -72,5 +64,5 @@ +@@ -72,5 +65,5 @@ "job_cluster_key": "job_cluster", "notebook_task": { - "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/notebook" + "notebook_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src/notebook" }, "task_key": "notebook_task" -@@ -107,5 +99,4 @@ +@@ -107,5 +100,5 @@ ], "trigger": { - "pause_status": "PAUSED", ++ "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -119,21 +110,20 @@ +@@ -119,21 +112,20 @@ "project_name_[UNIQUE_NAME]_pipeline": { "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" @@ -370,7 +374,7 @@ Resources: + "schema": "project_name_[UNIQUE_NAME]_prod", "url": "[DATABRICKS_URL]/pipelines/[UUID]" } -@@ -146,10 +136,10 @@ +@@ -146,10 +138,10 @@ }, "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts", diff --git a/acceptance/bundle/validate/empty_resources/empty_dict/output.txt b/acceptance/bundle/validate/empty_resources/empty_dict/output.txt index 8bdfb1cb9e..a8371658f7 100644 --- a/acceptance/bundle/validate/empty_resources/empty_dict/output.txt +++ b/acceptance/bundle/validate/empty_resources/empty_dict/output.txt @@ -9,6 +9,8 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Untitled", "permissions": [], "queue": { "enabled": true diff --git a/acceptance/bundle/validate/empty_resources/with_grants/output.txt b/acceptance/bundle/validate/empty_resources/with_grants/output.txt index 6e7ac17c2b..0a15578775 100644 --- a/acceptance/bundle/validate/empty_resources/with_grants/output.txt +++ b/acceptance/bundle/validate/empty_resources/with_grants/output.txt @@ -13,6 +13,8 @@ Warning: unknown field: grants }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Untitled", "permissions": [], "queue": { "enabled": true diff --git a/acceptance/bundle/validate/empty_resources/with_permissions/output.txt b/acceptance/bundle/validate/empty_resources/with_permissions/output.txt index c4a3537266..d118ace0ce 100644 --- a/acceptance/bundle/validate/empty_resources/with_permissions/output.txt +++ b/acceptance/bundle/validate/empty_resources/with_permissions/output.txt @@ -9,6 +9,8 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Untitled", "permissions": [], "queue": { "enabled": true diff --git a/acceptance/bundle/validate/presets_name_prefix/output.txt b/acceptance/bundle/validate/presets_name_prefix/output.txt index 10ae2b6f44..e81ea99190 100644 --- a/acceptance/bundle/validate/presets_name_prefix/output.txt +++ b/acceptance/bundle/validate/presets_name_prefix/output.txt @@ -12,6 +12,7 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, "name": "prefix-job1", "permissions": [], "queue": { @@ -47,6 +48,7 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, "name": "[prefix]job1", "permissions": [], "queue": { @@ -82,6 +84,7 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, "name": "job1", "permissions": [], "queue": { diff --git a/acceptance/bundle/variables/complex/out.default.json b/acceptance/bundle/variables/complex/out.default.json index 7551313086..c300fe818a 100644 --- a/acceptance/bundle/variables/complex/out.default.json +++ b/acceptance/bundle/variables/complex/out.default.json @@ -24,6 +24,8 @@ } } ], + "max_concurrent_runs": 1, + "name": "Untitled", "permissions": [], "queue": { "enabled": true diff --git a/acceptance/bundle/variables/complex/out.dev.json b/acceptance/bundle/variables/complex/out.dev.json index 9bc3d60b88..3ca97f37b2 100644 --- a/acceptance/bundle/variables/complex/out.dev.json +++ b/acceptance/bundle/variables/complex/out.dev.json @@ -22,6 +22,8 @@ } } ], + "max_concurrent_runs": 1, + "name": "Untitled", "permissions": [], "queue": { "enabled": true diff --git a/acceptance/bundle/variables/complex_multiple_files/output.txt b/acceptance/bundle/variables/complex_multiple_files/output.txt index 2d16817701..19ede5e563 100644 --- a/acceptance/bundle/variables/complex_multiple_files/output.txt +++ b/acceptance/bundle/variables/complex_multiple_files/output.txt @@ -58,6 +58,8 @@ } } ], + "max_concurrent_runs": 1, + "name": "Untitled", "permissions": [], "queue": { "enabled": true diff --git a/acceptance/bundle/variables/prepend-workspace-var/output.txt b/acceptance/bundle/variables/prepend-workspace-var/output.txt index 95cfcac59d..c63f49fa27 100644 --- a/acceptance/bundle/variables/prepend-workspace-var/output.txt +++ b/acceptance/bundle/variables/prepend-workspace-var/output.txt @@ -19,6 +19,8 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Untitled", "permissions": [], "queue": { "enabled": true diff --git a/acceptance/bundle/variables/resolve-nonstrings/output.txt b/acceptance/bundle/variables/resolve-nonstrings/output.txt index 31040412f2..67537eded9 100644 --- a/acceptance/bundle/variables/resolve-nonstrings/output.txt +++ b/acceptance/bundle/variables/resolve-nonstrings/output.txt @@ -24,6 +24,8 @@ }, "edit_mode": "UI_LOCKED", "format": "MULTI_TASK", + "max_concurrent_runs": 1, + "name": "Untitled", "notification_settings": { "no_alert_for_canceled_runs": true, "no_alert_for_skipped_runs": false diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 5dce713971..e34e7ab28b 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -51,6 +51,26 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) diag.Diagnos {"resources.dashboards.*.parent_path", b.Config.Workspace.ResourcePath}, {"resources.dashboards.*.embed_credentials", false}, {"resources.volumes.*.volume_type", "MANAGED"}, + + // Jobs: + + // The defaults are the same as for terraform provider latest version (v1.75.0) + // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/jobs/resource_job.go#L532 + {"resources.jobs.*.name", "Untitled"}, + {"resources.jobs.*.max_concurrent_runs", 1}, + {"resources.jobs.*.schedule.pause_status", "UNPAUSED"}, + {"resources.jobs.*.trigger.pause_status", "UNPAUSED"}, + {"resources.jobs.*.continuous.pause_status", "UNPAUSED"}, + + // This is converted from single-task to multi-task + {"resources.jobs.*.task[*].dbt_task.schema", "default"}, + {"resources.jobs.*.task[*].for_each_task.task.dbt_task.schema", "default"}, + + // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/clusters/resource_cluster.go + // This triggers SingleNodeCluster() cluster validator. It needs to be run before applying defaults. + //{"resources.jobs.*.job_clusters[*].new_cluster.num_workers", 0}, + {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks", true}, + {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs", true}, } for _, defaultDef := range defaults {