From 4f16c4b68c32f30c31f4e69174be1b5ae22e8afa Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 20 Mar 2025 14:39:23 +0100 Subject: [PATCH 1/6] Consistent upload messages with rel path --- .../bundle/artifacts/unique_name_libraries/output.txt | 4 ++-- .../bundle/artifacts/unique_name_libraries/test.toml | 5 ----- acceptance/bundle/artifacts/whl_explicit/output.txt | 2 +- acceptance/bundle/artifacts/whl_implicit/output.txt | 2 +- .../bundle/artifacts/whl_implicit_custom_path/output.txt | 2 +- .../bundle/artifacts/whl_implicit_notebook/output.txt | 2 +- acceptance/bundle/artifacts/whl_multiple/output.txt | 8 ++++---- acceptance/bundle/artifacts/whl_multiple/script | 2 +- .../bundle/artifacts/whl_prebuilt_multiple/output.txt | 8 ++++---- acceptance/bundle/artifacts/whl_prebuilt_multiple/script | 2 +- .../bundle/artifacts/whl_via_environment_key/output.txt | 2 +- bundle/libraries/upload.go | 8 +++++++- 12 files changed, 24 insertions(+), 23 deletions(-) diff --git a/acceptance/bundle/artifacts/unique_name_libraries/output.txt b/acceptance/bundle/artifacts/unique_name_libraries/output.txt index 634574bf56..ce3567a023 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/output.txt +++ b/acceptance/bundle/artifacts/unique_name_libraries/output.txt @@ -2,8 +2,8 @@ >>> errcode [CLI] bundle deploy Building whl1... Building whl2... -Uploading [package name]... -Uploading [package name]... +Uploading whl1/dist/my_package-0.0.1-py3-none-any.whl... +Uploading whl2/dist/my_other_package-0.0.1-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/unique_name_libraries/default/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/artifacts/unique_name_libraries/test.toml b/acceptance/bundle/artifacts/unique_name_libraries/test.toml index 7956c19095..a030353d57 100644 --- a/acceptance/bundle/artifacts/unique_name_libraries/test.toml +++ b/acceptance/bundle/artifacts/unique_name_libraries/test.toml @@ -1,6 +1 @@ RecordRequests = false - -# The order in which files are uploaded can be different, so we just replace the name -[[Repls]] -Old="Uploading (my_package|my_other_package)-0.0.1-py3-none-any.whl" -New="Uploading [package name]" diff --git a/acceptance/bundle/artifacts/whl_explicit/output.txt b/acceptance/bundle/artifacts/whl_explicit/output.txt index 1018501db5..984b724e9a 100644 --- a/acceptance/bundle/artifacts/whl_explicit/output.txt +++ b/acceptance/bundle/artifacts/whl_explicit/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Building my_test_code... -Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/artifacts/whl_implicit/output.txt b/acceptance/bundle/artifacts/whl_implicit/output.txt index 69ff56c42c..dfa154ffe6 100644 --- a/acceptance/bundle/artifacts/whl_implicit/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Building python_artifact... -Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading dist/my_test_code-0.0.1-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt index 0658dce3ab..c5fac67f52 100644 --- a/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_custom_path/output.txt @@ -1,6 +1,6 @@ >>> [CLI] bundle deploy -Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading package/my_test_code-0.0.1-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-local/default/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt index 9c7296b9a6..4800a3d36e 100644 --- a/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt +++ b/acceptance/bundle/artifacts/whl_implicit_notebook/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Building python_artifact... -Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading dist/my_test_code-0.0.1-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel-notebook/default/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/artifacts/whl_multiple/output.txt b/acceptance/bundle/artifacts/whl_multiple/output.txt index 9335b9cc57..aba6607bdf 100644 --- a/acceptance/bundle/artifacts/whl_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_multiple/output.txt @@ -2,12 +2,12 @@ >>> [CLI] bundle deploy Building my_test_code... Building my_test_code_2... +Uploading my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl... +Uploading my_test_code/dist/my_test_code_2-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... Deploying resources... -Deployment complete! Updating deployment state... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... -Uploading my_test_code-0.0.1-py3-none-any.whl... -Uploading my_test_code_2-0.0.1-py3-none-any.whl... +Deployment complete! >>> find.py --expect 2 whl my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl diff --git a/acceptance/bundle/artifacts/whl_multiple/script b/acceptance/bundle/artifacts/whl_multiple/script index a475e9f730..abc089da28 100644 --- a/acceptance/bundle/artifacts/whl_multiple/script +++ b/acceptance/bundle/artifacts/whl_multiple/script @@ -1,4 +1,4 @@ -trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order +trace $CLI bundle deploy 2>&1 trace find.py --expect 2 whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt index c9cd82fb88..e213df4d3d 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/output.txt @@ -4,12 +4,12 @@ dist/lib/other_test_code-0.0.1-py3-none-any.whl dist/my_test_code-0.0.1-py3-none-any.whl >>> [CLI] bundle deploy +Uploading dist/lib/other_test_code-0.0.1-py3-none-any.whl... +Uploading dist/my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... Deploying resources... -Deployment complete! Updating deployment state... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... -Uploading my_test_code-0.0.1-py3-none-any.whl... -Uploading other_test_code-0.0.1-py3-none-any.whl... +Deployment complete! === Expecting to find 2 wheels, same as initially provided >>> find.py --expect 2 whl diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script index 5265151e26..d48698ae37 100644 --- a/acceptance/bundle/artifacts/whl_prebuilt_multiple/script +++ b/acceptance/bundle/artifacts/whl_prebuilt_multiple/script @@ -1,6 +1,6 @@ trace find.py --expect 2 whl -trace $CLI bundle deploy 2>&1 | sort # sorting because 'Uploading ...whl...' messages change order +trace $CLI bundle deploy 2>&1 title "Expecting to find 2 wheels, same as initially provided" trace find.py --expect 2 whl diff --git a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt index 8b6b781aae..1e0fb6317b 100644 --- a/acceptance/bundle/artifacts/whl_via_environment_key/output.txt +++ b/acceptance/bundle/artifacts/whl_via_environment_key/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Building my_test_code... -Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/environment_key/default/files... Deploying resources... Updating deployment state... diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index a2162fb7b5..6cd8717ef8 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -122,6 +122,8 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error return nil, err } + // AI: sort libs + return libs, nil } @@ -146,6 +148,11 @@ func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { errs.SetLimit(maxFilesRequestsInFlight) for source := range libs { + relPath, err := filepath.Rel(b.SyncRootPath, source) + if err != nil { + relPath = source + } + cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", relPath)) errs.Go(func() error { return UploadFile(errCtx, source, u.client) }) @@ -188,7 +195,6 @@ func (u *upload) Name() string { // Function to upload file (a library, artifact and etc) to Workspace or UC volume func UploadFile(ctx context.Context, file string, client filer.Filer) error { filename := filepath.Base(file) - cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", filename)) f, err := os.Open(file) if err != nil { From e68c91f4f2ecc9a97aea89434a1fedff94fc7959 Mon Sep 17 00:00:00 2001 From: "Denis Bilenko (aider)" Date: Thu, 20 Mar 2025 14:41:03 +0100 Subject: [PATCH 2/6] feat: Sort libraries for deterministic processing and upload --- bundle/libraries/upload.go | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index 6cd8717ef8..8d0dedf557 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -7,6 +7,7 @@ import ( "os" "path" "path/filepath" + "sort" "strings" "github.com/databricks/cli/bundle" @@ -122,8 +123,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error return nil, err } - // AI: sort libs - + // Sort the libraries by their source path for deterministic processing order return libs, nil } @@ -144,10 +144,16 @@ func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return diag.FromErr(err) } + sources := make([]string, 0, len(libs)) + for source := range libs { + sources = append(sources, source) + } + sort.Strings(sources) + errs, errCtx := errgroup.WithContext(ctx) errs.SetLimit(maxFilesRequestsInFlight) - for source := range libs { + for _, source := range sources { relPath, err := filepath.Rel(b.SyncRootPath, source) if err != nil { relPath = source @@ -163,7 +169,9 @@ func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { } // Update all the config paths to point to the uploaded location - for source, locations := range libs { + // Process sources in sorted order for deterministic behavior + for _, source := range sources { + locations := libs[source] err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { remotePath := path.Join(uploadPath, filepath.Base(source)) From 2af27c8097cad625c4af2454a394373f4f387ac2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 20 Mar 2025 14:48:52 +0100 Subject: [PATCH 3/6] clean up --- bundle/libraries/upload.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index 8d0dedf557..143a1da1c7 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -123,7 +123,6 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error return nil, err } - // Sort the libraries by their source path for deterministic processing order return libs, nil } @@ -169,7 +168,6 @@ func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { } // Update all the config paths to point to the uploaded location - // Process sources in sorted order for deterministic behavior for _, source := range sources { locations := libs[source] err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { From e486c45bedc0b1370259729605280ad24d3f7a6c Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 20 Mar 2025 14:52:51 +0100 Subject: [PATCH 4/6] update NEXT_CHANGELOG.md --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 246418a3a1..e738352986 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,5 +7,6 @@ ### Bundles * Processing 'artifacts' section is now done in "bundle validate" (adding defaults, inferring "build", asserting required fields) ([#2526])(https://github.com/databricks/cli/pull/2526)) +* When uploading artifacts, include relative path in log message ([#2539])(https://github.com/databricks/cli/pull/2539)) ### API Changes From 21e8fb82b2fb881cff5d97c854a9b0e7b29fb422 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 20 Mar 2025 15:07:10 +0100 Subject: [PATCH 5/6] use forward slashes --- bundle/libraries/upload.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index 143a1da1c7..b8fe1bac74 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -156,6 +156,8 @@ func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { relPath, err := filepath.Rel(b.SyncRootPath, source) if err != nil { relPath = source + } else { + relPath = filepath.ToSlash(relPath) } cmdio.LogString(ctx, fmt.Sprintf("Uploading %s...", relPath)) errs.Go(func() error { From e046bdbf7f9eb1da30773f99d49860892b41ff74 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 20 Mar 2025 15:43:23 +0100 Subject: [PATCH 6/6] fix integration test --- integration/bundle/testdata/default_python/bundle_deploy.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/bundle/testdata/default_python/bundle_deploy.txt b/integration/bundle/testdata/default_python/bundle_deploy.txt index fe1cc4fac0..d304f6e0bd 100644 --- a/integration/bundle/testdata/default_python/bundle_deploy.txt +++ b/integration/bundle/testdata/default_python/bundle_deploy.txt @@ -1,5 +1,5 @@ Building python_artifact... -Uploading project_name_$UNIQUE_PRJ-0.0.1+[NUMID].[NUMID]-py3-none-any.whl... +Uploading dist/project_name_$UNIQUE_PRJ-0.0.1+[NUMID].[NUMID]-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/project_name_$UNIQUE_PRJ/dev/files... Deploying resources... Updating deployment state...