From 0cbc362c104ddac38d97381f2d0d66e5cedc451f Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 15 Apr 2025 12:55:02 +0200 Subject: [PATCH 1/4] Extract clean up into separate function and run it at the end of validate --- bundle/artifacts/cleanup.go | 38 +++++++++++++++++++++++++++++++++++++ bundle/artifacts/prepare.go | 21 +------------------- 2 files changed, 39 insertions(+), 20 deletions(-) create mode 100644 bundle/artifacts/cleanup.go diff --git a/bundle/artifacts/cleanup.go b/bundle/artifacts/cleanup.go new file mode 100644 index 0000000000..9937178206 --- /dev/null +++ b/bundle/artifacts/cleanup.go @@ -0,0 +1,38 @@ +package artifacts + +import ( + "context" + "os" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/python" + "github.com/databricks/cli/libs/utils" +) + +func cleanupPythonDistBuild(ctx context.Context, b *bundle.Bundle) { + removeFolders := make(map[string]bool, len(b.Config.Artifacts)) + cleanupWheelFolders := make(map[string]bool, len(b.Config.Artifacts)) + + for _, artifactName := range utils.SortedKeys(b.Config.Artifacts) { + artifact := b.Config.Artifacts[artifactName] + if artifact.Type == "whl" && artifact.BuildCommand != "" { + dir := artifact.Path + removeFolders[filepath.Join(dir, "dist")] = true + cleanupWheelFolders[dir] = true + } + } + + for _, dir := range utils.SortedKeys(removeFolders) { + err := os.RemoveAll(dir) + if err != nil { + log.Infof(ctx, "Failed to remove %s: %s", dir, err) + } + } + + for _, dir := range utils.SortedKeys(cleanupWheelFolders) { + log.Infof(ctx, "Cleaning up Python build artifacts in %s", dir) + python.CleanupWheelFolder(dir) + } +} diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index f360aa33d1..c40a16d014 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -33,9 +33,6 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics return diag.FromErr(err) } - removeFolders := make(map[string]bool, len(b.Config.Artifacts)) - cleanupWheelFolders := make(map[string]bool, len(b.Config.Artifacts)) - for _, artifactName := range utils.SortedKeys(b.Config.Artifacts) { artifact := b.Config.Artifacts[artifactName] b.Metrics.AddBoolValue(metrics.ArtifactBuildCommandIsSet, artifact.BuildCommand != "") @@ -66,12 +63,6 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics artifact.Path = filepath.Join(dirPath, artifact.Path) } - if artifact.Type == "whl" && artifact.BuildCommand != "" { - dir := artifact.Path - removeFolders[filepath.Join(dir, "dist")] = true - cleanupWheelFolders[dir] = true - } - if artifact.BuildCommand == "" && len(artifact.Files) == 0 { diags = diags.Extend(diag.Errorf("misconfigured artifact: please specify 'build' or 'files' property")) } @@ -89,17 +80,7 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics return diags } - for _, dir := range utils.SortedKeys(removeFolders) { - err := os.RemoveAll(dir) - if err != nil { - log.Infof(ctx, "Failed to remove %s: %s", dir, err) - } - } - - for _, dir := range utils.SortedKeys(cleanupWheelFolders) { - log.Infof(ctx, "Cleaning up Python build artifacts in %s", dir) - python.CleanupWheelFolder(dir) - } + cleanupPythonDistBuild(ctx, b) return diags } From 20c66dfcc810f1c9cf1ea3f99bea7c8de90aedde Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 15 Apr 2025 12:57:11 +0200 Subject: [PATCH 2/4] Move artifacts clean up to deploy phase --- bundle/artifacts/build.go | 2 ++ bundle/artifacts/prepare.go | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 846c7800b4..3c54f3037a 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -39,6 +39,8 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { }) } + cleanupPythonDistBuild(ctx, b) + for _, artifactName := range utils.SortedKeys(b.Config.Artifacts) { a := b.Config.Artifacts[artifactName] diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index c40a16d014..c2a2d67a1c 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -80,8 +80,6 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics return diags } - cleanupPythonDistBuild(ctx, b) - return diags } From 90d4fcb57d8fe8b63a46a3268fd6321974b5e337 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 15 Apr 2025 13:03:02 +0200 Subject: [PATCH 3/4] update NEXT_CHANGELOG --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 95acdbc46d..ea7cd8ba78 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,5 +11,6 @@ ### Bundles * Do not exit early when checking incompatible tasks for specified DBR ([#2692](https://github.com/databricks/cli/pull/2692)) * Removed include/exclude flags support from bundle sync command ([#2718](https://github.com/databricks/cli/pull/2718)) +* Do not clean up Python artifacts dist and build folder in "bundle validate", do it in "bundle deploy". This reverts the behaviour introduced in 0.255.0 ([#2722](https://github.com/databricks/cli/pull/2722)) ### API Changes From 5d703e516778bf313b7c5503f1da3faa24ca3aa3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 15 Apr 2025 14:12:40 +0200 Subject: [PATCH 4/4] Correct NEXT_CHANGELOG --- NEXT_CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index ea7cd8ba78..4eae3617af 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -11,6 +11,6 @@ ### Bundles * Do not exit early when checking incompatible tasks for specified DBR ([#2692](https://github.com/databricks/cli/pull/2692)) * Removed include/exclude flags support from bundle sync command ([#2718](https://github.com/databricks/cli/pull/2718)) -* Do not clean up Python artifacts dist and build folder in "bundle validate", do it in "bundle deploy". This reverts the behaviour introduced in 0.255.0 ([#2722](https://github.com/databricks/cli/pull/2722)) +* Do not clean up Python artifacts dist and build folder in "bundle validate", do it in "bundle deploy". This reverts the behaviour introduced in 0.245.0 ([#2722](https://github.com/databricks/cli/pull/2722)) ### API Changes