From 6bd92822c121d355d40badda8a2336b5a39adc82 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 29 Apr 2025 14:15:22 +0200 Subject: [PATCH 1/2] Add defaults on pipelines that terraform applies --- acceptance/bundle/override/pipeline_cluster/output.txt | 4 ++++ acceptance/bundle/paths/invalid_pipeline_globs/output.txt | 2 ++ .../paths/pipeline_expected_file_got_notebook/output.txt | 2 ++ acceptance/bundle/paths/pipeline_globs/output.txt | 2 ++ acceptance/bundle/python/pipelines-support/output.txt | 4 ++++ .../bundle/templates/experimental-jobs-as-code/output.txt | 2 ++ .../bundle/validate/empty_resources/empty_dict/output.txt | 6 ++++++ .../validate/empty_resources/with_grants/output.txt | 6 ++++++ .../validate/empty_resources/with_permissions/output.txt | 6 ++++++ .../variables/variable_overrides_in_target/output.txt | 8 ++++++++ bundle/config/mutator/resourcemutator/resource_mutator.go | 5 +++++ 11 files changed, 47 insertions(+) diff --git a/acceptance/bundle/override/pipeline_cluster/output.txt b/acceptance/bundle/override/pipeline_cluster/output.txt index d1a67f6b90..43e5c72184 100644 --- a/acceptance/bundle/override/pipeline_cluster/output.txt +++ b/acceptance/bundle/override/pipeline_cluster/output.txt @@ -2,6 +2,7 @@ >>> [CLI] bundle validate -o json -t development { "foo": { + "channel": "CURRENT", "clusters": [ { "label": "default", @@ -16,6 +17,7 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_pipeline_cluster/development/state/metadata.json" }, + "edition": "ADVANCED", "name": "job", "permissions": [] } @@ -24,6 +26,7 @@ >>> [CLI] bundle validate -o json -t staging { "foo": { + "channel": "CURRENT", "clusters": [ { "label": "default", @@ -38,6 +41,7 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/override_pipeline_cluster/staging/state/metadata.json" }, + "edition": "ADVANCED", "name": "job", "permissions": [] } diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/output.txt b/acceptance/bundle/paths/invalid_pipeline_globs/output.txt index a8a5784aa7..54f45b4070 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/output.txt +++ b/acceptance/bundle/paths/invalid_pipeline_globs/output.txt @@ -8,6 +8,8 @@ Exit code: 1 "resources": { "pipelines": { "nyc_taxi_pipeline": { + "channel": "CURRENT", + "edition": "ADVANCED", "libraries": [ { "notebook": { diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/output.txt b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/output.txt index e294bf6c1c..234341863b 100644 --- a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/output.txt +++ b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/output.txt @@ -8,6 +8,8 @@ Exit code: 1 "resources": { "pipelines": { "nyc_taxi_pipeline": { + "channel": "CURRENT", + "edition": "ADVANCED", "libraries": [ { "file": { diff --git a/acceptance/bundle/paths/pipeline_globs/output.txt b/acceptance/bundle/paths/pipeline_globs/output.txt index e4c8a2c4c2..9eb6b98c95 100644 --- a/acceptance/bundle/paths/pipeline_globs/output.txt +++ b/acceptance/bundle/paths/pipeline_globs/output.txt @@ -4,10 +4,12 @@ "resources": { "pipelines": { "nyc_taxi_pipeline": { + "channel": "CURRENT", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/state/metadata.json" }, + "edition": "ADVANCED", "libraries": [ { "notebook": { diff --git a/acceptance/bundle/python/pipelines-support/output.txt b/acceptance/bundle/python/pipelines-support/output.txt index 97d4ebe2e7..6c6a83a9f5 100644 --- a/acceptance/bundle/python/pipelines-support/output.txt +++ b/acceptance/bundle/python/pipelines-support/output.txt @@ -14,18 +14,22 @@ "resources": { "pipelines": { "my_pipeline_1": { + "channel": "CURRENT", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json" }, + "edition": "ADVANCED", "name": "My Pipeline 1 (updated)", "permissions": [] }, "my_pipeline_2": { + "channel": "CURRENT", "deployment": { "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_project/default/state/metadata.json" }, + "edition": "ADVANCED", "name": "My Pipeline 2 (updated)", "permissions": [] } diff --git a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt index 3b0816f597..f9dcd03681 100644 --- a/acceptance/bundle/templates/experimental-jobs-as-code/output.txt +++ b/acceptance/bundle/templates/experimental-jobs-as-code/output.txt @@ -83,6 +83,7 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi "pipelines": { "my_jobs_as_code_pipeline": { "catalog": "catalog_name", + "channel": "CURRENT", "configuration": { "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/files/src" }, @@ -91,6 +92,7 @@ Warning: Ignoring Databricks CLI version constraint for development build. Requi "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/my_jobs_as_code/dev/state/metadata.json" }, "development": true, + "edition": "ADVANCED", "libraries": [ { "notebook": { diff --git a/acceptance/bundle/validate/empty_resources/empty_dict/output.txt b/acceptance/bundle/validate/empty_resources/empty_dict/output.txt index a8371658f7..80f6359e17 100644 --- a/acceptance/bundle/validate/empty_resources/empty_dict/output.txt +++ b/acceptance/bundle/validate/empty_resources/empty_dict/output.txt @@ -23,6 +23,12 @@ { "pipelines": { "rname": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/BUNDLE/default/state/metadata.json" + }, + "edition": "ADVANCED", "permissions": [] } } diff --git a/acceptance/bundle/validate/empty_resources/with_grants/output.txt b/acceptance/bundle/validate/empty_resources/with_grants/output.txt index 0a15578775..b027a75036 100644 --- a/acceptance/bundle/validate/empty_resources/with_grants/output.txt +++ b/acceptance/bundle/validate/empty_resources/with_grants/output.txt @@ -31,6 +31,12 @@ Warning: unknown field: grants { "pipelines": { "rname": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/BUNDLE/default/state/metadata.json" + }, + "edition": "ADVANCED", "permissions": [] } } diff --git a/acceptance/bundle/validate/empty_resources/with_permissions/output.txt b/acceptance/bundle/validate/empty_resources/with_permissions/output.txt index d118ace0ce..438667ad7a 100644 --- a/acceptance/bundle/validate/empty_resources/with_permissions/output.txt +++ b/acceptance/bundle/validate/empty_resources/with_permissions/output.txt @@ -23,6 +23,12 @@ { "pipelines": { "rname": { + "channel": "CURRENT", + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/BUNDLE/default/state/metadata.json" + }, + "edition": "ADVANCED", "permissions": [] } } diff --git a/acceptance/bundle/variables/variable_overrides_in_target/output.txt b/acceptance/bundle/variables/variable_overrides_in_target/output.txt index d112cf2dec..e495615f39 100644 --- a/acceptance/bundle/variables/variable_overrides_in_target/output.txt +++ b/acceptance/bundle/variables/variable_overrides_in_target/output.txt @@ -3,6 +3,7 @@ { "pipelines": { "my_pipeline": { + "channel": "CURRENT", "clusters": [ { "label": "default", @@ -14,6 +15,7 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/use-default-variable-values/state/metadata.json" }, + "edition": "ADVANCED", "name": "a_string", "permissions": [] } @@ -24,6 +26,7 @@ { "pipelines": { "my_pipeline": { + "channel": "CURRENT", "clusters": [ { "label": "default", @@ -35,6 +38,7 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/override-string-variable/state/metadata.json" }, + "edition": "ADVANCED", "name": "overridden_string", "permissions": [] } @@ -45,6 +49,7 @@ { "pipelines": { "my_pipeline": { + "channel": "CURRENT", "clusters": [ { "label": "default", @@ -56,6 +61,7 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/override-int-variable/state/metadata.json" }, + "edition": "ADVANCED", "name": "a_string", "permissions": [] } @@ -66,6 +72,7 @@ { "pipelines": { "my_pipeline": { + "channel": "CURRENT", "clusters": [ { "label": "default", @@ -77,6 +84,7 @@ "kind": "BUNDLE", "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/foobar/override-both-bool-and-string-variables/state/metadata.json" }, + "edition": "ADVANCED", "name": "overridden_string", "permissions": [] } diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index e34e7ab28b..38b4aea066 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -71,6 +71,11 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) diag.Diagnos //{"resources.jobs.*.job_clusters[*].new_cluster.num_workers", 0}, {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.notebooks", true}, {"resources.jobs.*.job_clusters[*].new_cluster.workload_type.clients.jobs", true}, + + // Pipelines (same as terraform) + // https://github.com/databricks/terraform-provider-databricks/blob/v1.75.0/pipelines/resource_pipeline.go#L253 + {"resources.pipelines.*.edition", "ADVANCED"}, + {"resources.pipelines.*.channel", "CURRENT"}, } for _, defaultDef := range defaults { From 23ce3837bc41c94250921da59c7e377a25607783 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 29 Apr 2025 15:01:04 +0200 Subject: [PATCH 2/2] update integration tests --- .../integration_classic/out.validate.dev.json | 2 ++ .../integration_classic/output.txt | 24 ++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json b/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json index 6dd846e535..c2f417a195 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json +++ b/acceptance/bundle/templates/default-python/integration_classic/out.validate.dev.json @@ -116,6 +116,7 @@ }, "pipelines": { "project_name_[UNIQUE_NAME]_pipeline": { + "channel": "CURRENT", "configuration": { "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" }, @@ -124,6 +125,7 @@ "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/state/metadata.json" }, "development": true, + "edition": "ADVANCED", "libraries": [ { "notebook": { diff --git a/acceptance/bundle/templates/default-python/integration_classic/output.txt b/acceptance/bundle/templates/default-python/integration_classic/output.txt index 03851cecdd..cbf1741cb6 100644 --- a/acceptance/bundle/templates/default-python/integration_classic/output.txt +++ b/acceptance/bundle/templates/default-python/integration_classic/output.txt @@ -74,13 +74,13 @@ Resources: + "url": "[DATABRICKS_URL]/jobs/[NUMBER]" } }, -@@ -125,4 +126,5 @@ - }, +@@ -127,4 +128,5 @@ "development": true, + "edition": "ADVANCED", + "id": "[UUID]", "libraries": [ { -@@ -133,6 +135,6 @@ +@@ -135,6 +137,6 @@ ], "name": "[dev [USERNAME]] project_name_[UNIQUE_NAME]_pipeline", - "permissions": [], @@ -89,7 +89,7 @@ Resources: + "url": "[DATABRICKS_URL]/pipelines/[UUID]" } } -@@ -143,5 +145,4 @@ +@@ -145,5 +147,4 @@ ] }, - "targets": null, @@ -197,8 +197,8 @@ Validation OK! + "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -118,21 +107,20 @@ - "project_name_[UNIQUE_NAME]_pipeline": { +@@ -119,22 +108,21 @@ + "channel": "CURRENT", "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" + "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src" @@ -209,6 +209,7 @@ Validation OK! + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json" }, - "development": true, + "edition": "ADVANCED", "libraries": [ { "notebook": { @@ -224,7 +225,7 @@ Validation OK! + "schema": "project_name_[UNIQUE_NAME]_prod" } } -@@ -145,10 +133,10 @@ +@@ -147,10 +135,10 @@ "targets": null, "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts", @@ -347,8 +348,8 @@ Resources: + "pause_status": "UNPAUSED", "periodic": { "interval": 1, -@@ -119,21 +112,20 @@ - "project_name_[UNIQUE_NAME]_pipeline": { +@@ -120,11 +113,10 @@ + "channel": "CURRENT", "configuration": { - "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src" + "bundle.sourcePath": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/files/src" @@ -359,8 +360,9 @@ Resources: + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/prod/state/metadata.json" }, - "development": true, + "edition": "ADVANCED", "id": "[UUID]", - "libraries": [ +@@ -132,10 +124,10 @@ { "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/files/src/dlt_pipeline" @@ -374,7 +376,7 @@ Resources: + "schema": "project_name_[UNIQUE_NAME]_prod", "url": "[DATABRICKS_URL]/pipelines/[UUID]" } -@@ -146,10 +138,10 @@ +@@ -148,10 +140,10 @@ }, "workspace": { - "artifact_path": "/Workspace/Users/[USERNAME]/.bundle/project_name_[UNIQUE_NAME]/dev/artifacts",