From 1be0d915b7f262e1d24c27dbe59fa2d9873c86a6 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 13:17:09 +0200 Subject: [PATCH 01/18] Fix pipeline globs --- acceptance/bundle/debug/out.stderr.txt | 3 ++- .../process_static_resources.go | 23 ++++++++++++++++++- bundle/phases/initialize.go | 1 - .../tests/pipeline_glob_paths/databricks.yml | 9 ++------ .../resources/pipeline.yml | 8 +++++++ 5 files changed, 34 insertions(+), 10 deletions(-) create mode 100644 bundle/tests/pipeline_glob_paths/resources/pipeline.yml diff --git a/acceptance/bundle/debug/out.stderr.txt b/acceptance/bundle/debug/out.stderr.txt index 3db100bec8..490d984a7f 100644 --- a/acceptance/bundle/debug/out.stderr.txt +++ b/acceptance/bundle/debug/out.stderr.txt @@ -40,7 +40,8 @@ 10:07:59 Debug: Apply pid=12345 mutator=ResolveVariableReferences 10:07:59 Debug: Apply pid=12345 mutator=ApplyTargetMode 10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources -10:07:59 Debug: Apply pid=12345 mutator=NormalizePaths +10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=ResolveVariableReferences(resources) +10:07:59 Debug: Apply pid=12345 mutator=ProcessStaticResources mutator=NormalizePaths 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load) 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(init) 10:07:59 Debug: Apply pid=12345 mutator=PythonMutator(load_resources) diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index 5935d370a0..5fa6726152 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -3,8 +3,8 @@ package resourcemutator import ( "context" "fmt" - "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -32,6 +32,27 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia return diag.FromErr(err) } + // only YAML resources need to have paths normalized, before normalizing paths + // we need to resolve variables because they can change path values: + // - variable can be used a prefix + // - path can be part of a complex variable value + diags := bundle.ApplySeq( + ctx, + b, + // Reads (dynamic): * (strings) (searches for variable references in string values) + // Updates (dynamic): resources.* (strings) (resolves variable references to their actual values) + // Resolves variable references in 'resources' using bundle, workspace, and variables prefixes + mutator.ResolveVariableReferencesOnlyResources( + "bundle", + "workspace", + "variables", + ), + mutator.NormalizePaths(), + ) + if diags.HasError() { + return diags + } + return NormalizeAndInitializeResources(ctx, b, addedResources) } diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 21e1cebe62..bc66f3da9d 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -121,7 +121,6 @@ func Initialize(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Static resources (e.g. YAML) are already loaded, we initialize and normalize them before Python resourcemutator.ProcessStaticResources(), - mutator.NormalizePaths(), pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseLoad), pythonmutator.PythonMutator(pythonmutator.PythonMutatorPhaseInit), diff --git a/bundle/tests/pipeline_glob_paths/databricks.yml b/bundle/tests/pipeline_glob_paths/databricks.yml index d25b977ba2..227c0f88c6 100644 --- a/bundle/tests/pipeline_glob_paths/databricks.yml +++ b/bundle/tests/pipeline_glob_paths/databricks.yml @@ -1,13 +1,8 @@ bundle: name: pipeline_glob_paths -resources: - pipelines: - nyc_taxi_pipeline: - name: "nyc taxi loader" - libraries: - - notebook: - path: ./dlt/* +include: + - resources/pipeline.yml targets: default: diff --git a/bundle/tests/pipeline_glob_paths/resources/pipeline.yml b/bundle/tests/pipeline_glob_paths/resources/pipeline.yml new file mode 100644 index 0000000000..0c4044c963 --- /dev/null +++ b/bundle/tests/pipeline_glob_paths/resources/pipeline.yml @@ -0,0 +1,8 @@ +resources: + pipelines: + nyc_taxi_pipeline: + name: "nyc taxi loader" + libraries: + - notebook: + path: ../dlt/* + From 24357e323715ce1bdecd7c3292dbf56204190a1c Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 13:32:08 +0200 Subject: [PATCH 02/18] Fix fmt --- .../config/mutator/resourcemutator/process_static_resources.go | 1 + 1 file changed, 1 insertion(+) diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index 5fa6726152..cef71c62b3 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -3,6 +3,7 @@ package resourcemutator import ( "context" "fmt" + "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/libs/diag" From c1eba4149ea63a2b86104037cbcde04dc0691d86 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 13:32:16 +0200 Subject: [PATCH 03/18] Add acceptance test --- .../paths/pipeline_globs/databricks.yml | 19 +++++++++++++++++++ .../paths/pipeline_globs/dlt/wrong_file.py | 2 ++ .../bundle/paths/pipeline_globs/output.txt | 9 +++++++++ .../root/dlt/nyc_taxi_loader.py | 3 +++ .../root/resources/dlt/wrong_file.py | 2 ++ .../root/resources/pipeline.yml | 8 ++++++++ acceptance/bundle/paths/pipeline_globs/script | 2 ++ 7 files changed, 45 insertions(+) create mode 100644 acceptance/bundle/paths/pipeline_globs/databricks.yml create mode 100644 acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py create mode 100644 acceptance/bundle/paths/pipeline_globs/output.txt create mode 100644 acceptance/bundle/paths/pipeline_globs/root/dlt/nyc_taxi_loader.py create mode 100644 acceptance/bundle/paths/pipeline_globs/root/resources/dlt/wrong_file.py create mode 100644 acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml create mode 100644 acceptance/bundle/paths/pipeline_globs/script diff --git a/acceptance/bundle/paths/pipeline_globs/databricks.yml b/acceptance/bundle/paths/pipeline_globs/databricks.yml new file mode 100644 index 0000000000..227c0f88c6 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: pipeline_glob_paths + +include: + - resources/pipeline.yml + +targets: + default: + default: true + + error: + default: false + + resources: + pipelines: + nyc_taxi_pipeline: + libraries: + - notebook: + path: ./non-existent diff --git a/acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py new file mode 100644 index 0000000000..6b8ee602fd --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py @@ -0,0 +1,2 @@ +# Databricks notebook source +# this file should never be used, because glob points to a different file \ No newline at end of file diff --git a/acceptance/bundle/paths/pipeline_globs/output.txt b/acceptance/bundle/paths/pipeline_globs/output.txt new file mode 100644 index 0000000000..31c29c5997 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/output.txt @@ -0,0 +1,9 @@ + +>>> [CLI] bundle validate --output json +Error: resources/pipeline.yml defined in 'include' section does not match any files + +{ + "resources": null +} + +Exit code: 1 diff --git a/acceptance/bundle/paths/pipeline_globs/root/dlt/nyc_taxi_loader.py b/acceptance/bundle/paths/pipeline_globs/root/dlt/nyc_taxi_loader.py new file mode 100644 index 0000000000..83181c7099 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/root/dlt/nyc_taxi_loader.py @@ -0,0 +1,3 @@ +# Databricks notebook source + +print("Hello from notebook!") diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/dlt/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/root/resources/dlt/wrong_file.py new file mode 100644 index 0000000000..6b8ee602fd --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/root/resources/dlt/wrong_file.py @@ -0,0 +1,2 @@ +# Databricks notebook source +# this file should never be used, because glob points to a different file \ No newline at end of file diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml new file mode 100644 index 0000000000..0c4044c963 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml @@ -0,0 +1,8 @@ +resources: + pipelines: + nyc_taxi_pipeline: + name: "nyc taxi loader" + libraries: + - notebook: + path: ../dlt/* + diff --git a/acceptance/bundle/paths/pipeline_globs/script b/acceptance/bundle/paths/pipeline_globs/script new file mode 100644 index 0000000000..c0528fda69 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/script @@ -0,0 +1,2 @@ +cd root +trace $CLI bundle validate --output json | jq 'pick(.resources)' \ No newline at end of file From 7dcca59c9c7f3d8f21d309c8723ab7aba2a3623e Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 13:54:27 +0200 Subject: [PATCH 04/18] Fix fmt --- acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py | 2 +- .../paths/pipeline_globs/root/resources/dlt/wrong_file.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py index 6b8ee602fd..8af01a95aa 100644 --- a/acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py +++ b/acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py @@ -1,2 +1,2 @@ # Databricks notebook source -# this file should never be used, because glob points to a different file \ No newline at end of file +# this file should never be used, because glob points to a different file diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/dlt/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/root/resources/dlt/wrong_file.py index 6b8ee602fd..8af01a95aa 100644 --- a/acceptance/bundle/paths/pipeline_globs/root/resources/dlt/wrong_file.py +++ b/acceptance/bundle/paths/pipeline_globs/root/resources/dlt/wrong_file.py @@ -1,2 +1,2 @@ # Databricks notebook source -# this file should never be used, because glob points to a different file \ No newline at end of file +# this file should never be used, because glob points to a different file From e6fe266080122adfff1df46933df6d1139610056 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 14:10:18 +0200 Subject: [PATCH 05/18] Fix fmt --- .../bundle/paths/pipeline_globs/root/resources/pipeline.yml | 1 - acceptance/bundle/paths/pipeline_globs/script | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml index 0c4044c963..006f30aa7c 100644 --- a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml +++ b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml @@ -5,4 +5,3 @@ resources: libraries: - notebook: path: ../dlt/* - diff --git a/acceptance/bundle/paths/pipeline_globs/script b/acceptance/bundle/paths/pipeline_globs/script index c0528fda69..6844cc0090 100644 --- a/acceptance/bundle/paths/pipeline_globs/script +++ b/acceptance/bundle/paths/pipeline_globs/script @@ -1,2 +1,2 @@ cd root -trace $CLI bundle validate --output json | jq 'pick(.resources)' \ No newline at end of file +trace $CLI bundle validate --output json | jq 'pick(.resources)' From 8e0ad0a2878ea0cc59783990da8d7a1a7186dbcd Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 14:45:13 +0200 Subject: [PATCH 06/18] Add more tests --- .../invalid_pipeline_globs/databricks.yml | 13 +++++ .../paths/invalid_pipeline_globs/output.txt | 23 ++++++++ .../paths/invalid_pipeline_globs/script | 1 + .../paths/pipeline_globs/databricks.yml | 19 ------- .../bundle/paths/pipeline_globs/output.txt | 57 +++++++++++++++++-- .../paths/pipeline_globs/root/databricks.yml | 10 ++++ .../root/resources/pipeline.yml | 16 +++++- .../tests/pipeline_glob_paths/databricks.yml | 19 ------- .../dlt/nyc_taxi_loader.py | 3 - .../resources/pipeline.yml | 8 --- bundle/tests/pipeline_glob_paths_test.go | 22 ------- 11 files changed, 112 insertions(+), 79 deletions(-) create mode 100644 acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml create mode 100644 acceptance/bundle/paths/invalid_pipeline_globs/output.txt create mode 100644 acceptance/bundle/paths/invalid_pipeline_globs/script delete mode 100644 acceptance/bundle/paths/pipeline_globs/databricks.yml create mode 100644 acceptance/bundle/paths/pipeline_globs/root/databricks.yml delete mode 100644 bundle/tests/pipeline_glob_paths/databricks.yml delete mode 100644 bundle/tests/pipeline_glob_paths/dlt/nyc_taxi_loader.py delete mode 100644 bundle/tests/pipeline_glob_paths/resources/pipeline.yml delete mode 100644 bundle/tests/pipeline_glob_paths_test.go diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml b/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml new file mode 100644 index 0000000000..92c852a20c --- /dev/null +++ b/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml @@ -0,0 +1,13 @@ +bundle: + name: pipeline_glob_paths + +resources: + pipelines: + nyc_taxi_pipeline: + libraries: + - notebook: { path: "${var.notebook_dir}/*.ipynb" } + +variables: + notebook_dir: + description: Directory with DLT notebooks + default: non-existent \ No newline at end of file diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/output.txt b/acceptance/bundle/paths/invalid_pipeline_globs/output.txt new file mode 100644 index 0000000000..da0cb854e0 --- /dev/null +++ b/acceptance/bundle/paths/invalid_pipeline_globs/output.txt @@ -0,0 +1,23 @@ + +>>> [CLI] bundle validate --output json +Error: notebook non-existent/*.ipynb not found + +{ + "resources": { + "pipelines": { + "nyc_taxi_pipeline": { + "libraries": [ + { + "notebook": { + "path": "non-existent/*.ipynb" + } + } + ], + "name": "nyc taxi loader", + "permissions": [] + } + } + } +} + +Exit code: 1 diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/script b/acceptance/bundle/paths/invalid_pipeline_globs/script new file mode 100644 index 0000000000..53e257931e --- /dev/null +++ b/acceptance/bundle/paths/invalid_pipeline_globs/script @@ -0,0 +1 @@ +trace $CLI bundle validate --output json | jq 'pick(.resources)' diff --git a/acceptance/bundle/paths/pipeline_globs/databricks.yml b/acceptance/bundle/paths/pipeline_globs/databricks.yml deleted file mode 100644 index 227c0f88c6..0000000000 --- a/acceptance/bundle/paths/pipeline_globs/databricks.yml +++ /dev/null @@ -1,19 +0,0 @@ -bundle: - name: pipeline_glob_paths - -include: - - resources/pipeline.yml - -targets: - default: - default: true - - error: - default: false - - resources: - pipelines: - nyc_taxi_pipeline: - libraries: - - notebook: - path: ./non-existent diff --git a/acceptance/bundle/paths/pipeline_globs/output.txt b/acceptance/bundle/paths/pipeline_globs/output.txt index 31c29c5997..8f61529997 100644 --- a/acceptance/bundle/paths/pipeline_globs/output.txt +++ b/acceptance/bundle/paths/pipeline_globs/output.txt @@ -1,9 +1,56 @@ >>> [CLI] bundle validate --output json -Error: resources/pipeline.yml defined in 'include' section does not match any files - { - "resources": null + "resources": { + "pipelines": { + "nyc_taxi_pipeline": { + "deployment": { + "kind": "BUNDLE", + "metadata_file_path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/state/metadata.json" + }, + "libraries": [ + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader" + } + }, + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader" + } + }, + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader" + } + }, + { + "notebook": { + "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader" + } + }, + { + "maven": { + "coordinates": "org.jsoup:jsoup:1.7.2" + } + }, + { + "jar": "*/*.jar" + }, + { + "notebook": { + "path": "/Workspace/Users/me@company.com/*.ipynb" + } + }, + { + "notebook": { + "path": "s3://notebooks/*.ipynb" + } + } + ], + "name": "nyc taxi loader", + "permissions": [] + } + } + } } - -Exit code: 1 diff --git a/acceptance/bundle/paths/pipeline_globs/root/databricks.yml b/acceptance/bundle/paths/pipeline_globs/root/databricks.yml new file mode 100644 index 0000000000..c2f043c665 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/root/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: pipeline_glob_paths + +include: + - resources/pipeline.yml + +variables: + notebook_dir: + description: Directory with DLT notebooks + default: dlt \ No newline at end of file diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml index 006f30aa7c..4ffe462a26 100644 --- a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml +++ b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml @@ -1,7 +1,17 @@ resources: pipelines: nyc_taxi_pipeline: - name: "nyc taxi loader" libraries: - - notebook: - path: ../dlt/* + # globs for notebooks and files are expanded + - notebook: { path: "../${var.notebook_dir}/*" } + - file: { path: "../${var.notebook_dir}/*" } + # globs can include file extensions + - notebook: { path: "../${var.notebook_dir}/*.py" } + # non-glob files work + - notebook: { path: "../${var.notebook_dir}/nyc_taxi_loader.py" } + # maven libraries and jars remain as-is + - maven: { coordinates: "org.jsoup:jsoup:1.7.2" } + - jar: "*/*.jar" + # absolute paths and paths using URLs remain as-is + - notebook: { path: "/Workspace/Users/me@company.com/*.ipynb" } + - notebook: { path: "s3://notebooks/*.ipynb" } \ No newline at end of file diff --git a/bundle/tests/pipeline_glob_paths/databricks.yml b/bundle/tests/pipeline_glob_paths/databricks.yml deleted file mode 100644 index 227c0f88c6..0000000000 --- a/bundle/tests/pipeline_glob_paths/databricks.yml +++ /dev/null @@ -1,19 +0,0 @@ -bundle: - name: pipeline_glob_paths - -include: - - resources/pipeline.yml - -targets: - default: - default: true - - error: - default: false - - resources: - pipelines: - nyc_taxi_pipeline: - libraries: - - notebook: - path: ./non-existent diff --git a/bundle/tests/pipeline_glob_paths/dlt/nyc_taxi_loader.py b/bundle/tests/pipeline_glob_paths/dlt/nyc_taxi_loader.py deleted file mode 100644 index 83181c7099..0000000000 --- a/bundle/tests/pipeline_glob_paths/dlt/nyc_taxi_loader.py +++ /dev/null @@ -1,3 +0,0 @@ -# Databricks notebook source - -print("Hello from notebook!") diff --git a/bundle/tests/pipeline_glob_paths/resources/pipeline.yml b/bundle/tests/pipeline_glob_paths/resources/pipeline.yml deleted file mode 100644 index 0c4044c963..0000000000 --- a/bundle/tests/pipeline_glob_paths/resources/pipeline.yml +++ /dev/null @@ -1,8 +0,0 @@ -resources: - pipelines: - nyc_taxi_pipeline: - name: "nyc taxi loader" - libraries: - - notebook: - path: ../dlt/* - diff --git a/bundle/tests/pipeline_glob_paths_test.go b/bundle/tests/pipeline_glob_paths_test.go deleted file mode 100644 index 9053c658fc..0000000000 --- a/bundle/tests/pipeline_glob_paths_test.go +++ /dev/null @@ -1,22 +0,0 @@ -package config_tests - -import ( - "testing" - - "github.com/stretchr/testify/require" -) - -func TestExpandPipelineGlobPaths(t *testing.T) { - b, diags := initializeTarget(t, "./pipeline_glob_paths", "default") - require.NoError(t, diags.Error()) - require.Equal( - t, - "/Workspace/Users/user@domain.com/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader", - b.Config.Resources.Pipelines["nyc_taxi_pipeline"].Libraries[0].Notebook.Path, - ) -} - -func TestExpandPipelineGlobPathsWithNonExistent(t *testing.T) { - _, diags := initializeTarget(t, "./pipeline_glob_paths", "error") - require.ErrorContains(t, diags.Error(), `notebook "non-existent" not found`) -} From d2d7cd5d28aec0330ec33ffe00878a5857391714 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 14:49:57 +0200 Subject: [PATCH 07/18] Add changelog --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index f28fe2ff2b..8f51f6c3b5 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,5 +10,6 @@ ### Bundles * Do not exit early when checking incompatible tasks for specified DBR ([#2692](https://github.com/databricks/cli/pull/2692)) +* Fixed a regression with pipeline library globs introduced in v0.247.0 ([#2723](https://github.com/databricks/cli/pull/2723)) ### API Changes From 50c72a8255301a126b9ca5984a0a8258331c2a74 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 14:52:20 +0200 Subject: [PATCH 08/18] Update NEXT_CHANGELOG.md --- NEXT_CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index d2e57cc064..618bc634c9 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -12,6 +12,6 @@ * Do not exit early when checking incompatible tasks for specified DBR ([#2692](https://github.com/databricks/cli/pull/2692)) * Removed include/exclude flags support from bundle sync command ([#2718](https://github.com/databricks/cli/pull/2718)) * Do not clean up Python artifacts dist and build folder in "bundle validate", do it in "bundle deploy". This reverts the behaviour introduced in 0.245.0 ([#2722](https://github.com/databricks/cli/pull/2722)) -* Fixed a regression with pipeline library globs introduced in v0.247.0 ([#2723](https://github.com/databricks/cli/pull/2723)) +* Fixed a regression with pipeline library globs introduced in 0.247.0 ([#2723](https://github.com/databricks/cli/pull/2723)) ### API Changes From 60b194b919d0f996aed0057d691c77a508ffd9d0 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 14:58:14 +0200 Subject: [PATCH 09/18] Remove unused code --- bundle/tests/loader.go | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go index fd200a5b65..20d096cc93 100644 --- a/bundle/tests/loader.go +++ b/bundle/tests/loader.go @@ -9,12 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/phases" - "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/diag" - "github.com/databricks/databricks-sdk-go/config" - "github.com/databricks/databricks-sdk-go/experimental/mocks" - "github.com/databricks/databricks-sdk-go/service/iam" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" ) @@ -54,26 +49,3 @@ func loadTargetWithDiags(path, env string) (*bundle.Bundle, diag.Diagnostics) { )) return b, diags } - -func configureMock(t *testing.T, b *bundle.Bundle) { - // Configure mock workspace client - m := mocks.NewMockWorkspaceClient(t) - m.WorkspaceClient.Config = &config.Config{ - Host: "https://mock.databricks.workspace.com", - } - m.GetMockCurrentUserAPI().EXPECT().Me(mock.Anything).Return(&iam.User{ - UserName: "user@domain.com", - }, nil) - b.SetWorkpaceClient(m.WorkspaceClient) -} - -func initializeTarget(t *testing.T, path, env string) (*bundle.Bundle, diag.Diagnostics) { - b := load(t, path) - configureMock(t, b) - - ctx := dbr.MockRuntime(context.Background(), dbr.Environment{}) - diags := bundle.Apply(ctx, b, mutator.SelectTarget(env)) - diags = diags.Extend(phases.Initialize(ctx, b)) - - return b, diags -} From 9cfb05d8e1508285942665cadd5d59dee878c613 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 14:58:22 +0200 Subject: [PATCH 10/18] Fix tests --- acceptance/bundle/paths/invalid_pipeline_globs/output.txt | 1 - acceptance/bundle/paths/pipeline_globs/output.txt | 1 - 2 files changed, 2 deletions(-) diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/output.txt b/acceptance/bundle/paths/invalid_pipeline_globs/output.txt index da0cb854e0..068176fa66 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/output.txt +++ b/acceptance/bundle/paths/invalid_pipeline_globs/output.txt @@ -13,7 +13,6 @@ Error: notebook non-existent/*.ipynb not found } } ], - "name": "nyc taxi loader", "permissions": [] } } diff --git a/acceptance/bundle/paths/pipeline_globs/output.txt b/acceptance/bundle/paths/pipeline_globs/output.txt index 8f61529997..19afba0ad0 100644 --- a/acceptance/bundle/paths/pipeline_globs/output.txt +++ b/acceptance/bundle/paths/pipeline_globs/output.txt @@ -48,7 +48,6 @@ } } ], - "name": "nyc taxi loader", "permissions": [] } } From a099141c2f715240de9f45220cbb1f3d634ab983 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 15:15:50 +0200 Subject: [PATCH 11/18] Fix tests --- acceptance/bundle/paths/invalid_pipeline_globs/output.txt | 4 ++-- acceptance/bundle/paths/invalid_pipeline_globs/script | 2 +- acceptance/bundle/paths/pipeline_globs/script.cleanup | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 acceptance/bundle/paths/pipeline_globs/script.cleanup diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/output.txt b/acceptance/bundle/paths/invalid_pipeline_globs/output.txt index 068176fa66..a8a5784aa7 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/output.txt +++ b/acceptance/bundle/paths/invalid_pipeline_globs/output.txt @@ -2,6 +2,8 @@ >>> [CLI] bundle validate --output json Error: notebook non-existent/*.ipynb not found + +Exit code: 1 { "resources": { "pipelines": { @@ -18,5 +20,3 @@ Error: notebook non-existent/*.ipynb not found } } } - -Exit code: 1 diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/script b/acceptance/bundle/paths/invalid_pipeline_globs/script index 53e257931e..a6a93f74a7 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/script +++ b/acceptance/bundle/paths/invalid_pipeline_globs/script @@ -1 +1 @@ -trace $CLI bundle validate --output json | jq 'pick(.resources)' +errcode trace $CLI bundle validate --output json | jq 'pick(.resources)' diff --git a/acceptance/bundle/paths/pipeline_globs/script.cleanup b/acceptance/bundle/paths/pipeline_globs/script.cleanup new file mode 100644 index 0000000000..e55880b75a --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/script.cleanup @@ -0,0 +1 @@ +rm -rf root/.databricks \ No newline at end of file From e45e3cf8dc7b0103c9bd141116c4549abdf3ba96 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 15:16:41 +0200 Subject: [PATCH 12/18] Fix whitespace --- acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml | 2 +- acceptance/bundle/paths/pipeline_globs/root/databricks.yml | 2 +- .../bundle/paths/pipeline_globs/root/resources/pipeline.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml b/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml index 92c852a20c..1618be9b4a 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml +++ b/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml @@ -10,4 +10,4 @@ resources: variables: notebook_dir: description: Directory with DLT notebooks - default: non-existent \ No newline at end of file + default: non-existent diff --git a/acceptance/bundle/paths/pipeline_globs/root/databricks.yml b/acceptance/bundle/paths/pipeline_globs/root/databricks.yml index c2f043c665..2801efcbe5 100644 --- a/acceptance/bundle/paths/pipeline_globs/root/databricks.yml +++ b/acceptance/bundle/paths/pipeline_globs/root/databricks.yml @@ -7,4 +7,4 @@ include: variables: notebook_dir: description: Directory with DLT notebooks - default: dlt \ No newline at end of file + default: dlt diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml index 4ffe462a26..1baf6dbebf 100644 --- a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml +++ b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml @@ -14,4 +14,4 @@ resources: - jar: "*/*.jar" # absolute paths and paths using URLs remain as-is - notebook: { path: "/Workspace/Users/me@company.com/*.ipynb" } - - notebook: { path: "s3://notebooks/*.ipynb" } \ No newline at end of file + - notebook: { path: "s3://notebooks/*.ipynb" } From 05aa3169945cd02b7dbf71481fe852e4f78c8761 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 15:20:40 +0200 Subject: [PATCH 13/18] Fix whitespace --- acceptance/bundle/paths/pipeline_globs/script.cleanup | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/paths/pipeline_globs/script.cleanup b/acceptance/bundle/paths/pipeline_globs/script.cleanup index e55880b75a..533eaf8f85 100644 --- a/acceptance/bundle/paths/pipeline_globs/script.cleanup +++ b/acceptance/bundle/paths/pipeline_globs/script.cleanup @@ -1 +1 @@ -rm -rf root/.databricks \ No newline at end of file +rm -rf root/.databricks From 5205e6cc5589af17e2a959a9884d510658cc7e20 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 15:40:58 +0200 Subject: [PATCH 14/18] Update tests --- .../invalid_pipeline_globs/databricks.yml | 2 +- .../databricks.yml | 10 +++++++++ .../notebooks}/nyc_taxi_loader.py | 0 .../output.txt | 22 +++++++++++++++++++ .../resources/pipeline.yml | 6 +++++ .../script | 1 + .../paths/pipeline_globs/files/wrong_file.py | 1 + .../{dlt => notebooks}/wrong_file.py | 0 .../bundle/paths/pipeline_globs/output.txt | 13 +++++++---- .../paths/pipeline_globs/root/databricks.yml | 5 ++++- .../root/files/nyc_taxi_loader.py | 1 + .../root/notebooks/nyc_taxi_loader.py | 3 +++ .../root/resources/files/wrong_file.py | 1 + .../{dlt => notebooks}/wrong_file.py | 0 .../root/resources/pipeline.yml | 3 ++- 15 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 acceptance/bundle/paths/pipeline_expected_file_got_notebook/databricks.yml rename acceptance/bundle/paths/{pipeline_globs/root/dlt => pipeline_expected_file_got_notebook/notebooks}/nyc_taxi_loader.py (100%) create mode 100644 acceptance/bundle/paths/pipeline_expected_file_got_notebook/output.txt create mode 100644 acceptance/bundle/paths/pipeline_expected_file_got_notebook/resources/pipeline.yml create mode 100644 acceptance/bundle/paths/pipeline_expected_file_got_notebook/script create mode 100644 acceptance/bundle/paths/pipeline_globs/files/wrong_file.py rename acceptance/bundle/paths/pipeline_globs/{dlt => notebooks}/wrong_file.py (100%) create mode 100644 acceptance/bundle/paths/pipeline_globs/root/files/nyc_taxi_loader.py create mode 100644 acceptance/bundle/paths/pipeline_globs/root/notebooks/nyc_taxi_loader.py create mode 100644 acceptance/bundle/paths/pipeline_globs/root/resources/files/wrong_file.py rename acceptance/bundle/paths/pipeline_globs/root/resources/{dlt => notebooks}/wrong_file.py (100%) diff --git a/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml b/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml index 1618be9b4a..6a94727dde 100644 --- a/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml +++ b/acceptance/bundle/paths/invalid_pipeline_globs/databricks.yml @@ -1,5 +1,5 @@ bundle: - name: pipeline_glob_paths + name: invalid_pipeline_glob_paths resources: pipelines: diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/databricks.yml b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/databricks.yml new file mode 100644 index 0000000000..7d176f0cd5 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/databricks.yml @@ -0,0 +1,10 @@ +bundle: + name: pipeline_expected_file_got_notebook + +include: + - resources/pipeline.yml + +variables: + notebook_dir: + description: Directory with DLT notebooks + default: notebooks diff --git a/acceptance/bundle/paths/pipeline_globs/root/dlt/nyc_taxi_loader.py b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/notebooks/nyc_taxi_loader.py similarity index 100% rename from acceptance/bundle/paths/pipeline_globs/root/dlt/nyc_taxi_loader.py rename to acceptance/bundle/paths/pipeline_expected_file_got_notebook/notebooks/nyc_taxi_loader.py diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/output.txt b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/output.txt new file mode 100644 index 0000000000..e294bf6c1c --- /dev/null +++ b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/output.txt @@ -0,0 +1,22 @@ + +>>> [CLI] bundle validate --output json +Error: expected a file for "resources.pipelines.nyc_taxi_pipeline.libraries[0].file.path" but got a notebook: file at [TEST_TMP_DIR]/notebooks/nyc_taxi_loader.py is a notebook + + +Exit code: 1 +{ + "resources": { + "pipelines": { + "nyc_taxi_pipeline": { + "libraries": [ + { + "file": { + "path": "notebooks/nyc_taxi_loader.py" + } + } + ], + "permissions": [] + } + } + } +} diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/resources/pipeline.yml b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/resources/pipeline.yml new file mode 100644 index 0000000000..d4a7e2b037 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/resources/pipeline.yml @@ -0,0 +1,6 @@ +resources: + pipelines: + nyc_taxi_pipeline: + libraries: + # path points to a notebook, not a file, it should error out + - file: { path: "../${var.notebook_dir}/*.py" } diff --git a/acceptance/bundle/paths/pipeline_expected_file_got_notebook/script b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/script new file mode 100644 index 0000000000..a6a93f74a7 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_expected_file_got_notebook/script @@ -0,0 +1 @@ +errcode trace $CLI bundle validate --output json | jq 'pick(.resources)' diff --git a/acceptance/bundle/paths/pipeline_globs/files/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/files/wrong_file.py new file mode 100644 index 0000000000..a798241447 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/files/wrong_file.py @@ -0,0 +1 @@ +# this file should never be used, because glob points to a different file diff --git a/acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/notebooks/wrong_file.py similarity index 100% rename from acceptance/bundle/paths/pipeline_globs/dlt/wrong_file.py rename to acceptance/bundle/paths/pipeline_globs/notebooks/wrong_file.py diff --git a/acceptance/bundle/paths/pipeline_globs/output.txt b/acceptance/bundle/paths/pipeline_globs/output.txt index 19afba0ad0..e4c8a2c4c2 100644 --- a/acceptance/bundle/paths/pipeline_globs/output.txt +++ b/acceptance/bundle/paths/pipeline_globs/output.txt @@ -11,22 +11,27 @@ "libraries": [ { "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader" + "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/notebooks/nyc_taxi_loader" } }, { "file": { - "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader" + "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/files/nyc_taxi_loader.py" } }, { "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader" + "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/notebooks/nyc_taxi_loader" + } + }, + { + "file": { + "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/files/nyc_taxi_loader.py" } }, { "notebook": { - "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/dlt/nyc_taxi_loader" + "path": "/Workspace/Users/[USERNAME]/.bundle/pipeline_glob_paths/default/files/notebooks/nyc_taxi_loader" } }, { diff --git a/acceptance/bundle/paths/pipeline_globs/root/databricks.yml b/acceptance/bundle/paths/pipeline_globs/root/databricks.yml index 2801efcbe5..a2b3f77698 100644 --- a/acceptance/bundle/paths/pipeline_globs/root/databricks.yml +++ b/acceptance/bundle/paths/pipeline_globs/root/databricks.yml @@ -7,4 +7,7 @@ include: variables: notebook_dir: description: Directory with DLT notebooks - default: dlt + default: notebooks + file_dir: + description: Directory with DLT files + default: files diff --git a/acceptance/bundle/paths/pipeline_globs/root/files/nyc_taxi_loader.py b/acceptance/bundle/paths/pipeline_globs/root/files/nyc_taxi_loader.py new file mode 100644 index 0000000000..9089eea9d6 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/root/files/nyc_taxi_loader.py @@ -0,0 +1 @@ +print("Hello from file!") diff --git a/acceptance/bundle/paths/pipeline_globs/root/notebooks/nyc_taxi_loader.py b/acceptance/bundle/paths/pipeline_globs/root/notebooks/nyc_taxi_loader.py new file mode 100644 index 0000000000..83181c7099 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/root/notebooks/nyc_taxi_loader.py @@ -0,0 +1,3 @@ +# Databricks notebook source + +print("Hello from notebook!") diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/files/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/root/resources/files/wrong_file.py new file mode 100644 index 0000000000..a798241447 --- /dev/null +++ b/acceptance/bundle/paths/pipeline_globs/root/resources/files/wrong_file.py @@ -0,0 +1 @@ +# this file should never be used, because glob points to a different file diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/dlt/wrong_file.py b/acceptance/bundle/paths/pipeline_globs/root/resources/notebooks/wrong_file.py similarity index 100% rename from acceptance/bundle/paths/pipeline_globs/root/resources/dlt/wrong_file.py rename to acceptance/bundle/paths/pipeline_globs/root/resources/notebooks/wrong_file.py diff --git a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml index 1baf6dbebf..89e4d33ada 100644 --- a/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml +++ b/acceptance/bundle/paths/pipeline_globs/root/resources/pipeline.yml @@ -4,9 +4,10 @@ resources: libraries: # globs for notebooks and files are expanded - notebook: { path: "../${var.notebook_dir}/*" } - - file: { path: "../${var.notebook_dir}/*" } + - file: { path: "../${var.file_dir}/*" } # globs can include file extensions - notebook: { path: "../${var.notebook_dir}/*.py" } + - file: { path: "../${var.file_dir}/*.py" } # non-glob files work - notebook: { path: "../${var.notebook_dir}/nyc_taxi_loader.py" } # maven libraries and jars remain as-is From 801f77a476223df93fd220b5858ef4d794c9f862 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 15:58:26 +0200 Subject: [PATCH 15/18] Update changelog --- NEXT_CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 618bc634c9..8893b5e732 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -3,6 +3,7 @@ ## Release v0.248.0 ### Notable Changes +* Fixed a regression with pipeline library globs introduced in 0.247.0 ([#2723](https://github.com/databricks/cli/pull/2723)). The issue caused glob patterns to fail when using paths relative to a directory outside the bundle root. ### Dependency updates @@ -12,6 +13,5 @@ * Do not exit early when checking incompatible tasks for specified DBR ([#2692](https://github.com/databricks/cli/pull/2692)) * Removed include/exclude flags support from bundle sync command ([#2718](https://github.com/databricks/cli/pull/2718)) * Do not clean up Python artifacts dist and build folder in "bundle validate", do it in "bundle deploy". This reverts the behaviour introduced in 0.245.0 ([#2722](https://github.com/databricks/cli/pull/2722)) -* Fixed a regression with pipeline library globs introduced in 0.247.0 ([#2723](https://github.com/databricks/cli/pull/2723)) ### API Changes From a536382bd6986640ff97480ced97de2996ec7c10 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 15:58:41 +0200 Subject: [PATCH 16/18] Fix script --- acceptance/bundle/paths/pipeline_globs/script | 1 + acceptance/bundle/paths/pipeline_globs/script.cleanup | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) delete mode 100644 acceptance/bundle/paths/pipeline_globs/script.cleanup diff --git a/acceptance/bundle/paths/pipeline_globs/script b/acceptance/bundle/paths/pipeline_globs/script index 6844cc0090..286d5c2e00 100644 --- a/acceptance/bundle/paths/pipeline_globs/script +++ b/acceptance/bundle/paths/pipeline_globs/script @@ -1,2 +1,3 @@ cd root trace $CLI bundle validate --output json | jq 'pick(.resources)' +rm -rf root/.databricks diff --git a/acceptance/bundle/paths/pipeline_globs/script.cleanup b/acceptance/bundle/paths/pipeline_globs/script.cleanup deleted file mode 100644 index 533eaf8f85..0000000000 --- a/acceptance/bundle/paths/pipeline_globs/script.cleanup +++ /dev/null @@ -1 +0,0 @@ -rm -rf root/.databricks From 033f4ed805fb1f8a06f60eb3fa7784d0e18d3495 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 16:11:39 +0200 Subject: [PATCH 17/18] Fix script --- acceptance/bundle/paths/pipeline_globs/script | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/paths/pipeline_globs/script b/acceptance/bundle/paths/pipeline_globs/script index 286d5c2e00..d6b3baf2a3 100644 --- a/acceptance/bundle/paths/pipeline_globs/script +++ b/acceptance/bundle/paths/pipeline_globs/script @@ -1,3 +1,3 @@ cd root trace $CLI bundle validate --output json | jq 'pick(.resources)' -rm -rf root/.databricks +rm -rf .databricks From 471ee2100a2110df2a1d9c0e33014af2108a98d3 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Tue, 15 Apr 2025 16:18:22 +0200 Subject: [PATCH 18/18] Update NEXT_CHANGELOG.md Co-authored-by: shreyas-goenka <88374338+shreyas-goenka@users.noreply.github.com> --- NEXT_CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 8893b5e732..601eaae151 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -3,7 +3,7 @@ ## Release v0.248.0 ### Notable Changes -* Fixed a regression with pipeline library globs introduced in 0.247.0 ([#2723](https://github.com/databricks/cli/pull/2723)). The issue caused glob patterns to fail when using paths relative to a directory outside the bundle root. +* Fixed a regression with pipeline library globs introduced in 0.247.0 ([#2723](https://github.com/databricks/cli/pull/2723)). The issue caused glob patterns to fail when using paths relative to a directory that is not the bundle root. ### Dependency updates