From e1e973fe7a7dd56c5def572e6aace3151615b044 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 20 Mar 2025 16:54:58 +0100 Subject: [PATCH 1/8] Add a test that show the wheel being uploaded twice Bug introduced in #2526. --- .../artifacts/glob_exact_whl/databricks.yml | 21 +++++++++++++++++++ .../artifacts/glob_exact_whl/output.txt | 20 ++++++++++++++++++ .../bundle/artifacts/glob_exact_whl/script | 5 +++++ .../bundle/artifacts/glob_exact_whl/test.toml | 2 ++ 4 files changed, 48 insertions(+) create mode 100644 acceptance/bundle/artifacts/glob_exact_whl/databricks.yml create mode 100644 acceptance/bundle/artifacts/glob_exact_whl/output.txt create mode 100644 acceptance/bundle/artifacts/glob_exact_whl/script create mode 100644 acceptance/bundle/artifacts/glob_exact_whl/test.toml diff --git a/acceptance/bundle/artifacts/glob_exact_whl/databricks.yml b/acceptance/bundle/artifacts/glob_exact_whl/databricks.yml new file mode 100644 index 0000000000..7e6d6b8114 --- /dev/null +++ b/acceptance/bundle/artifacts/glob_exact_whl/databricks.yml @@ -0,0 +1,21 @@ +bundle: + name: python-wheel + +artifacts: + my_prebuilt_whl: + type: whl + files: + - source: prebuilt/other_test_code-0.0.1-py3-none-any.whl + +resources: + jobs: + test_job: + name: "[${bundle.target}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/glob_exact_whl/output.txt b/acceptance/bundle/artifacts/glob_exact_whl/output.txt new file mode 100644 index 0000000000..d39a559b68 --- /dev/null +++ b/acceptance/bundle/artifacts/glob_exact_whl/output.txt @@ -0,0 +1,20 @@ + +>>> [CLI] bundle validate -o json +{ + "my_prebuilt_whl": { + "files": [ + { + "source": "prebuilt/other_test_code-0.0.1-py3-none-any.whl" + } + ], + "type": "whl" + } +} + +>>> [CLI] bundle deploy +Uploading prebuilt/other_test_code-0.0.1-py3-none-any.whl... +Uploading prebuilt/other_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! diff --git a/acceptance/bundle/artifacts/glob_exact_whl/script b/acceptance/bundle/artifacts/glob_exact_whl/script new file mode 100644 index 0000000000..3244477ed3 --- /dev/null +++ b/acceptance/bundle/artifacts/glob_exact_whl/script @@ -0,0 +1,5 @@ +trap "rm -r prebuilt" EXIT +mkdir prebuilt +cp -r $TESTDIR/../whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl prebuilt +trace $CLI bundle validate -o json | jq .artifacts +trace $CLI bundle deploy diff --git a/acceptance/bundle/artifacts/glob_exact_whl/test.toml b/acceptance/bundle/artifacts/glob_exact_whl/test.toml new file mode 100644 index 0000000000..2c93721696 --- /dev/null +++ b/acceptance/bundle/artifacts/glob_exact_whl/test.toml @@ -0,0 +1,2 @@ +Badness = 'Uploading the same wheel twice' +RecordRequests = false From 2e1ba04ff42ae1d7cc35e330418d97eaad79432e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 20 Mar 2025 22:41:24 +0100 Subject: [PATCH 2/8] Fix --- .../artifacts/glob_exact_whl/output.txt | 4 ++-- .../artifacts/globs_in_files/output.txt | 23 +++++++++++-------- .../bundle/artifacts/globs_invalid/output.txt | 13 ++++++----- bundle/artifacts/expand_globs.go | 16 ++++++++++++- 4 files changed, 37 insertions(+), 19 deletions(-) diff --git a/acceptance/bundle/artifacts/glob_exact_whl/output.txt b/acceptance/bundle/artifacts/glob_exact_whl/output.txt index d39a559b68..e3f050312e 100644 --- a/acceptance/bundle/artifacts/glob_exact_whl/output.txt +++ b/acceptance/bundle/artifacts/glob_exact_whl/output.txt @@ -4,16 +4,16 @@ "my_prebuilt_whl": { "files": [ { - "source": "prebuilt/other_test_code-0.0.1-py3-none-any.whl" + "source": "[TEST_TMP_DIR]/prebuilt/other_test_code-0.0.1-py3-none-any.whl" } ], + "path": "[TEST_TMP_DIR]", "type": "whl" } } >>> [CLI] bundle deploy Uploading prebuilt/other_test_code-0.0.1-py3-none-any.whl... -Uploading prebuilt/other_test_code-0.0.1-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/artifacts/globs_in_files/output.txt b/acceptance/bundle/artifacts/globs_in_files/output.txt index 5d5b39de48..88c92dcf06 100644 --- a/acceptance/bundle/artifacts/globs_in_files/output.txt +++ b/acceptance/bundle/artifacts/globs_in_files/output.txt @@ -14,12 +14,13 @@ Exit code: 1 "test": { "files": [ { - "source": "a*.txt" + "source": "[TEST_TMP_DIR]/a*.txt" }, { - "source": "subdir/*.txt" + "source": "[TEST_TMP_DIR]/subdir/*.txt" } - ] + ], + "path": "[TEST_TMP_DIR]" } } @@ -34,12 +35,13 @@ Exit code: 1 "test": { "files": [ { - "source": "a*.txt" + "source": "[TEST_TMP_DIR]/a*.txt" }, { - "source": "subdir/*.txt" + "source": "[TEST_TMP_DIR]/subdir/*.txt" } - ] + ], + "path": "[TEST_TMP_DIR]" } } @@ -48,14 +50,15 @@ Exit code: 1 "test": { "files": [ { - "source": "a1.txt" + "source": "[TEST_TMP_DIR]/a1.txt" }, { - "source": "a2.txt" + "source": "[TEST_TMP_DIR]/a2.txt" }, { - "source": "subdir/hello.txt" + "source": "[TEST_TMP_DIR]/subdir/hello.txt" } - ] + ], + "path": "[TEST_TMP_DIR]" } } diff --git a/acceptance/bundle/artifacts/globs_invalid/output.txt b/acceptance/bundle/artifacts/globs_invalid/output.txt index 178a9dddbd..e325075c83 100644 --- a/acceptance/bundle/artifacts/globs_invalid/output.txt +++ b/acceptance/bundle/artifacts/globs_invalid/output.txt @@ -4,7 +4,7 @@ Error: a[.txt: syntax error in pattern at artifacts.test.files[0].source in databricks.yml:7:17 -Error: ./a[.txt: syntax error in pattern +Error: a[.txt: syntax error in pattern at artifacts.test.files[1].source in databricks.yml:8:17 @@ -22,17 +22,18 @@ Exit code: 1 "test": { "files": [ { - "source": "a[.txt" + "source": "[TEST_TMP_DIR]/a[.txt" }, { - "source": "./a[.txt" + "source": "[TEST_TMP_DIR]/a[.txt" }, { - "source": "../a[.txt" + "source": "[TEST_TMP_DIR]_PARENT/a[.txt" }, { - "source": "subdir/a[.txt" + "source": "[TEST_TMP_DIR]/subdir/a[.txt" } - ] + ], + "path": "[TEST_TMP_DIR]" } } diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index 889a62e268..438f7ac116 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" ) func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { @@ -30,6 +31,19 @@ func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { } func expandGlobs(ctx context.Context, b *bundle.Bundle, name string) diag.Diagnostics { + err := b.Config.MarkMutatorEntry(ctx) + if err != nil { + log.Errorf(ctx, "entry error: %s", err) + return diag.Errorf("entry error: %s", err) + } + + defer func() { + err := b.Config.MarkMutatorExit(ctx) + if err != nil { + log.Errorf(ctx, "exit error: %s", err) + } + }() + // Base path for this mutator. // This path is set with the list of expanded globs when done. base := dyn.NewPath( @@ -45,7 +59,7 @@ func expandGlobs(ctx context.Context, b *bundle.Bundle, name string) diag.Diagno ) var diags diag.Diagnostics - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { var output []dyn.Value _, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { if v.Kind() != dyn.KindString { From b8a1e204e6ea660077ecfa302bf0b19cd61acd9a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 20 Mar 2025 22:58:54 +0100 Subject: [PATCH 3/8] fix windows --- acceptance/bundle/artifacts/glob_exact_whl/test.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/artifacts/glob_exact_whl/test.toml b/acceptance/bundle/artifacts/glob_exact_whl/test.toml index 2c93721696..f29e875485 100644 --- a/acceptance/bundle/artifacts/glob_exact_whl/test.toml +++ b/acceptance/bundle/artifacts/glob_exact_whl/test.toml @@ -1,2 +1,6 @@ Badness = 'Uploading the same wheel twice' RecordRequests = false + +[[Repls]] +Old = '\\\\' +New = '/' From 6f775df29c1fd34fd970081a933ed86aaf05f877 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 21 Mar 2025 09:12:05 +0100 Subject: [PATCH 4/8] fix windows --- acceptance/bundle/artifacts/globs_invalid/test.toml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/acceptance/bundle/artifacts/globs_invalid/test.toml b/acceptance/bundle/artifacts/globs_invalid/test.toml index a030353d57..3b639e6af6 100644 --- a/acceptance/bundle/artifacts/globs_invalid/test.toml +++ b/acceptance/bundle/artifacts/globs_invalid/test.toml @@ -1 +1,9 @@ RecordRequests = false + +[[Repls]] +Old = '\\\\' +New = '/' + +[[Repls]] +Old = '\\' +New = '/' From 53ed10965b093103c26ea242a611381768cb2b64 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 21 Mar 2025 09:24:12 +0100 Subject: [PATCH 5/8] fix globs_in_files on windows --- acceptance/bundle/artifacts/globs_in_files/test.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/artifacts/globs_in_files/test.toml b/acceptance/bundle/artifacts/globs_in_files/test.toml index 93ee159927..3b639e6af6 100644 --- a/acceptance/bundle/artifacts/globs_in_files/test.toml +++ b/acceptance/bundle/artifacts/globs_in_files/test.toml @@ -3,3 +3,7 @@ RecordRequests = false [[Repls]] Old = '\\\\' New = '/' + +[[Repls]] +Old = '\\' +New = '/' From 72e0485057e33acba81d052379c1e38e9f141c09 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 21 Mar 2025 09:51:00 +0100 Subject: [PATCH 6/8] Add a test showing a bug with glob expansion --- .../globs_in_files_in_include/databricks.yml | 5 ++ .../globs_in_files_in_include/output.txt | 58 +++++++++++++++++++ .../resources/artifacts.yml | 5 ++ .../globs_in_files_in_include/script | 12 ++++ .../globs_in_files_in_include/test.toml | 6 ++ 5 files changed, 86 insertions(+) create mode 100644 acceptance/bundle/artifacts/globs_in_files_in_include/databricks.yml create mode 100644 acceptance/bundle/artifacts/globs_in_files_in_include/output.txt create mode 100644 acceptance/bundle/artifacts/globs_in_files_in_include/resources/artifacts.yml create mode 100644 acceptance/bundle/artifacts/globs_in_files_in_include/script create mode 100644 acceptance/bundle/artifacts/globs_in_files_in_include/test.toml diff --git a/acceptance/bundle/artifacts/globs_in_files_in_include/databricks.yml b/acceptance/bundle/artifacts/globs_in_files_in_include/databricks.yml new file mode 100644 index 0000000000..1723173759 --- /dev/null +++ b/acceptance/bundle/artifacts/globs_in_files_in_include/databricks.yml @@ -0,0 +1,5 @@ +bundle: + name: expand_globs + +include: + - resources/artifacts.yml diff --git a/acceptance/bundle/artifacts/globs_in_files_in_include/output.txt b/acceptance/bundle/artifacts/globs_in_files_in_include/output.txt new file mode 100644 index 0000000000..b97cf7119d --- /dev/null +++ b/acceptance/bundle/artifacts/globs_in_files_in_include/output.txt @@ -0,0 +1,58 @@ + +>>> [CLI] bundle validate -o json +Error: subdir/*.txt: no matching files + at artifacts.test.files[1].source + in resources/artifacts.yml:5:17 + + +Exit code: 1 +{ + "test": { + "files": [ + { + "source": "a*.txt" + }, + { + "source": "subdir/*.txt" + } + ] + } +} + +>>> withdir resources [CLI] bundle validate -o json +{ + "test": { + "files": [ + { + "source": "a1.txt" + }, + { + "source": "subdir/hello.txt" + } + ] + } +} + +>>> withdir other_dir [CLI] bundle validate -o json +Error: a*.txt: no matching files + at artifacts.test.files[0].source + in resources/artifacts.yml:4:17 + +Error: subdir/*.txt: no matching files + at artifacts.test.files[1].source + in resources/artifacts.yml:5:17 + + +Exit code: 1 +{ + "test": { + "files": [ + { + "source": "a*.txt" + }, + { + "source": "subdir/*.txt" + } + ] + } +} diff --git a/acceptance/bundle/artifacts/globs_in_files_in_include/resources/artifacts.yml b/acceptance/bundle/artifacts/globs_in_files_in_include/resources/artifacts.yml new file mode 100644 index 0000000000..90753fa990 --- /dev/null +++ b/acceptance/bundle/artifacts/globs_in_files_in_include/resources/artifacts.yml @@ -0,0 +1,5 @@ +artifacts: + test: + files: + - source: a*.txt + - source: subdir/*.txt diff --git a/acceptance/bundle/artifacts/globs_in_files_in_include/script b/acceptance/bundle/artifacts/globs_in_files_in_include/script new file mode 100644 index 0000000000..5fb6ac726d --- /dev/null +++ b/acceptance/bundle/artifacts/globs_in_files_in_include/script @@ -0,0 +1,12 @@ +touch resources/a1.txt +touch a2.txt +mkdir -p resources/subdir +touch resources/subdir/hello.txt + +errcode trace $CLI bundle validate -o json | jq .artifacts +errcode trace withdir resources $CLI bundle validate -o json | jq .artifacts + +mkdir other_dir +errcode trace withdir other_dir $CLI bundle validate -o json | jq .artifacts + +rm -fr resources/a1.txt a2.txt resources/subdir diff --git a/acceptance/bundle/artifacts/globs_in_files_in_include/test.toml b/acceptance/bundle/artifacts/globs_in_files_in_include/test.toml new file mode 100644 index 0000000000..ac63ed45c3 --- /dev/null +++ b/acceptance/bundle/artifacts/globs_in_files_in_include/test.toml @@ -0,0 +1,6 @@ +Badness = "Glob expansion is performed within cwd, not relative to where path is defined" +RecordRequests = false + +[[Repls]] +Old = '\\\\' +New = '/' From ce209c5f46906de250bd903ce7e9049d9be7a199 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 21 Mar 2025 10:15:32 +0100 Subject: [PATCH 7/8] update --- .../globs_in_files_in_include/output.txt | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/acceptance/bundle/artifacts/globs_in_files_in_include/output.txt b/acceptance/bundle/artifacts/globs_in_files_in_include/output.txt index b97cf7119d..7ccd8c3478 100644 --- a/acceptance/bundle/artifacts/globs_in_files_in_include/output.txt +++ b/acceptance/bundle/artifacts/globs_in_files_in_include/output.txt @@ -1,21 +1,16 @@ >>> [CLI] bundle validate -o json -Error: subdir/*.txt: no matching files - at artifacts.test.files[1].source - in resources/artifacts.yml:5:17 - - -Exit code: 1 { "test": { "files": [ { - "source": "a*.txt" + "source": "[TEST_TMP_DIR]/resources/a1.txt" }, { - "source": "subdir/*.txt" + "source": "[TEST_TMP_DIR]/resources/subdir/hello.txt" } - ] + ], + "path": "[TEST_TMP_DIR]" } } @@ -24,35 +19,27 @@ Exit code: 1 "test": { "files": [ { - "source": "a1.txt" + "source": "[TEST_TMP_DIR]/resources/a1.txt" }, { - "source": "subdir/hello.txt" + "source": "[TEST_TMP_DIR]/resources/subdir/hello.txt" } - ] + ], + "path": "[TEST_TMP_DIR]" } } >>> withdir other_dir [CLI] bundle validate -o json -Error: a*.txt: no matching files - at artifacts.test.files[0].source - in resources/artifacts.yml:4:17 - -Error: subdir/*.txt: no matching files - at artifacts.test.files[1].source - in resources/artifacts.yml:5:17 - - -Exit code: 1 { "test": { "files": [ { - "source": "a*.txt" + "source": "[TEST_TMP_DIR]/resources/a1.txt" }, { - "source": "subdir/*.txt" + "source": "[TEST_TMP_DIR]/resources/subdir/hello.txt" } - ] + ], + "path": "[TEST_TMP_DIR]" } } From 240d4428b28756f5a131e00ca484124c483f78fc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 21 Mar 2025 10:42:17 +0100 Subject: [PATCH 8/8] make expandGlobs a mutator --- bundle/artifacts/build.go | 2 +- bundle/artifacts/expand_globs.go | 24 +++++++++--------------- bundle/artifacts/prepare.go | 2 +- 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 62529b7ab6..8cc407b06f 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -77,7 +77,7 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // We need to expand glob reference after build mutator is applied because // if we do it before, any files that are generated by build command will // not be included into artifact.Files and thus will not be uploaded. - diags = diags.Extend(expandGlobs(ctx, b, artifactName)) + diags = diags.Extend(bundle.Apply(ctx, b, expandGlobs{name: artifactName})) if diags.HasError() { break } diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index 438f7ac116..72d1390889 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/log" ) func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { @@ -30,25 +29,20 @@ func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { } } -func expandGlobs(ctx context.Context, b *bundle.Bundle, name string) diag.Diagnostics { - err := b.Config.MarkMutatorEntry(ctx) - if err != nil { - log.Errorf(ctx, "entry error: %s", err) - return diag.Errorf("entry error: %s", err) - } +type expandGlobs struct { + name string +} - defer func() { - err := b.Config.MarkMutatorExit(ctx) - if err != nil { - log.Errorf(ctx, "exit error: %s", err) - } - }() +func (e expandGlobs) Name() string { + return "expandGlobs" +} +func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Base path for this mutator. // This path is set with the list of expanded globs when done. base := dyn.NewPath( dyn.Key("artifacts"), - dyn.Key(name), + dyn.Key(e.name), dyn.Key("files"), ) @@ -59,7 +53,7 @@ func expandGlobs(ctx context.Context, b *bundle.Bundle, name string) diag.Diagno ) var diags diag.Diagnostics - err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { var output []dyn.Value _, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { if v.Kind() != dyn.KindString { diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index fb3b718ab9..100a9dbb60 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -74,7 +74,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics } if len(artifact.Files) > 0 && artifact.BuildCommand == "" { - diags = diags.Extend(expandGlobs(ctx, b, artifactName)) + diags = diags.Extend(bundle.Apply(ctx, b, expandGlobs{name: artifactName})) } if diags.HasError() {