From 6010578a9f177a8448e284994d82c6292f60c6b1 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 9 Apr 2025 16:47:48 +0200 Subject: [PATCH 1/4] Do not exit early when checking incompatible tasks --- bundle/trampoline/python_dbr_warning.go | 40 ++++++++++++++------ bundle/trampoline/python_dbr_warning_test.go | 23 +++++++---- 2 files changed, 44 insertions(+), 19 deletions(-) diff --git a/bundle/trampoline/python_dbr_warning.go b/bundle/trampoline/python_dbr_warning.go index 18fbbb353e..743e77213f 100644 --- a/bundle/trampoline/python_dbr_warning.go +++ b/bundle/trampoline/python_dbr_warning.go @@ -2,6 +2,7 @@ package trampoline import ( "context" + "fmt" "strconv" "strings" @@ -29,22 +30,31 @@ func (m *wrapperWarning) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn return nil } - if hasIncompatibleWheelTasks(ctx, b) { - return diag.Errorf("Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.") + diags := hasIncompatibleWheelTasks(ctx, b) + if len(diags) > 0 { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.", + }) } - return nil + return diags } func isPythonWheelWrapperOn(b *bundle.Bundle) bool { return b.Config.Experimental != nil && b.Config.Experimental.PythonWheelWrapper } -func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { +func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + var diags diag.Diagnostics + tasks := libraries.FindTasksWithLocalLibraries(b) for _, task := range tasks { if task.NewCluster != nil { if lowerThanExpectedVersion(task.NewCluster.SparkVersion) { - return true + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("task %s uses incompatible DBR version %s", task.TaskKey, task.NewCluster.SparkVersion), + }) } } @@ -53,7 +63,10 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { for _, cluster := range job.JobClusters { if task.JobClusterKey == cluster.JobClusterKey && cluster.NewCluster.SparkVersion != "" { if lowerThanExpectedVersion(cluster.NewCluster.SparkVersion) { - return true + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("job cluster %s uses incompatible DBR version %s", cluster.JobClusterKey, cluster.NewCluster.SparkVersion), + }) } } } @@ -70,19 +83,19 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { p, ok := dynvar.PureReferenceToPath(task.ExistingClusterId) if !ok || len(p) < 3 { log.Warnf(ctx, "unable to parse cluster key from %s", task.ExistingClusterId) - return false + continue } if p[0].Key() != "resources" || p[1].Key() != "clusters" { log.Warnf(ctx, "incorrect variable reference for cluster id %s", task.ExistingClusterId) - return false + continue } clusterKey := p[2].Key() cluster, ok := b.Config.Resources.Clusters[clusterKey] if !ok { log.Warnf(ctx, "unable to find cluster with key %s", clusterKey) - return false + continue } version = cluster.SparkVersion } else { @@ -90,17 +103,20 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) bool { // If there's error getting spark version for cluster, do not mark it as incompatible if err != nil { log.Warnf(ctx, "unable to get spark version for cluster %s, err: %s", task.ExistingClusterId, err.Error()) - return false + continue } } if lowerThanExpectedVersion(version) { - return true + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("existing cluster %s uses incompatible DBR version %s", task.ExistingClusterId, version), + }) } } } - return false + return diags } func lowerThanExpectedVersion(sparkVersion string) bool { diff --git a/bundle/trampoline/python_dbr_warning_test.go b/bundle/trampoline/python_dbr_warning_test.go index 96fac7329d..d7c1eb5f46 100644 --- a/bundle/trampoline/python_dbr_warning_test.go +++ b/bundle/trampoline/python_dbr_warning_test.go @@ -50,7 +50,8 @@ func TestIncompatibleWheelTasksWithNewCluster(t *testing.T) { }, } - require.True(t, hasIncompatibleWheelTasks(context.Background(), b)) + diags := hasIncompatibleWheelTasks(context.Background(), b) + require.NotEmpty(t, diags) } func TestIncompatibleWheelTasksWithJobClusterKey(t *testing.T) { @@ -99,10 +100,11 @@ func TestIncompatibleWheelTasksWithJobClusterKey(t *testing.T) { }, } - require.True(t, hasIncompatibleWheelTasks(context.Background(), b)) + diags := hasIncompatibleWheelTasks(context.Background(), b) + require.NotEmpty(t, diags) - diags := bundle.Apply(context.Background(), b, WrapperWarning()) - require.ErrorContains(t, diags.Error(), "require compute with DBR 13.3") + diags = bundle.Apply(context.Background(), b, WrapperWarning()) + require.ErrorContains(t, diags.Error(), "uses incompatible DBR version 12.2.x-scala2.12") } func TestIncompatibleWheelTasksWithExistingClusterId(t *testing.T) { @@ -143,8 +145,13 @@ func TestIncompatibleWheelTasksWithExistingClusterId(t *testing.T) { clustersApi.EXPECT().GetByClusterId(mock.Anything, "test-key-1").Return(&compute.ClusterDetails{ SparkVersion: "12.2.x-scala2.12", }, nil) + clustersApi.EXPECT().GetByClusterId(mock.Anything, "test-key-2").Return(&compute.ClusterDetails{ + SparkVersion: "12.2.x-scala2.12", + }, nil) - require.True(t, hasIncompatibleWheelTasks(context.Background(), b)) + diags := hasIncompatibleWheelTasks(context.Background(), b) + require.NotEmpty(t, diags) + require.ErrorContains(t, diags.Error(), "uses incompatible DBR version 12.2.x-scala2.12") } func TestNoIncompatibleWheelTasks(t *testing.T) { @@ -249,7 +256,8 @@ func TestNoIncompatibleWheelTasks(t *testing.T) { SparkVersion: "13.2.x-scala2.12", }, nil) - require.False(t, hasIncompatibleWheelTasks(context.Background(), b)) + diags := hasIncompatibleWheelTasks(context.Background(), b) + require.Empty(t, diags) } func TestTasksWithPyPiPackageAreCompatible(t *testing.T) { @@ -289,7 +297,8 @@ func TestTasksWithPyPiPackageAreCompatible(t *testing.T) { m := mocks.NewMockWorkspaceClient(t) b.SetWorkpaceClient(m.WorkspaceClient) - require.False(t, hasIncompatibleWheelTasks(context.Background(), b)) + diags := hasIncompatibleWheelTasks(context.Background(), b) + require.Empty(t, diags) } func TestNoWarningWhenPythonWheelWrapperIsOn(t *testing.T) { From f9c823499888e8018b44b63dde4f3aa71f847c9c Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 9 Apr 2025 17:04:33 +0200 Subject: [PATCH 2/4] fix tests --- acceptance/bundle/trampoline/warning_message/output.txt | 4 +++- .../trampoline/warning_message_with_old_spark/output.txt | 4 +++- bundle/trampoline/python_dbr_warning.go | 4 ++-- bundle/trampoline/python_dbr_warning_test.go | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/trampoline/warning_message/output.txt b/acceptance/bundle/trampoline/warning_message/output.txt index 2f7d69e1fc..5161a7f907 100644 --- a/acceptance/bundle/trampoline/warning_message/output.txt +++ b/acceptance/bundle/trampoline/warning_message/output.txt @@ -1,5 +1,7 @@ >>> errcode [CLI] bundle validate -t dev +Error: task test_task uses cluster with incompatible DBR version 12.2.x-cpu-ml-scala2.12 + Error: Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information. Name: trampoline_warning_message @@ -8,7 +10,7 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/trampoline_warning_message/dev -Found 1 error +Found 2 errors Exit code: 1 diff --git a/acceptance/bundle/trampoline/warning_message_with_old_spark/output.txt b/acceptance/bundle/trampoline/warning_message_with_old_spark/output.txt index 551cd17bc7..0863802845 100644 --- a/acceptance/bundle/trampoline/warning_message_with_old_spark/output.txt +++ b/acceptance/bundle/trampoline/warning_message_with_old_spark/output.txt @@ -1,5 +1,7 @@ >>> errcode [CLI] bundle validate +Error: task test_task uses cluster with incompatible DBR version 7.3.x-scala2.12 + Error: Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information. Name: trampoline_warning_message_with_old_spark @@ -8,6 +10,6 @@ Workspace: User: [USERNAME] Path: /Workspace/Users/[USERNAME]/.bundle/trampoline_warning_message_with_old_spark/dev -Found 1 error +Found 2 errors Exit code: 1 diff --git a/bundle/trampoline/python_dbr_warning.go b/bundle/trampoline/python_dbr_warning.go index 743e77213f..a7f77ff8ec 100644 --- a/bundle/trampoline/python_dbr_warning.go +++ b/bundle/trampoline/python_dbr_warning.go @@ -53,7 +53,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagn if lowerThanExpectedVersion(task.NewCluster.SparkVersion) { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("task %s uses incompatible DBR version %s", task.TaskKey, task.NewCluster.SparkVersion), + Summary: fmt.Sprintf("task %s uses cluster with incompatible DBR version %s", task.TaskKey, task.NewCluster.SparkVersion), }) } } @@ -110,7 +110,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagn if lowerThanExpectedVersion(version) { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, - Summary: fmt.Sprintf("existing cluster %s uses incompatible DBR version %s", task.ExistingClusterId, version), + Summary: fmt.Sprintf("task %s uses cluster with incompatible DBR version %s", task.TaskKey, version), }) } } diff --git a/bundle/trampoline/python_dbr_warning_test.go b/bundle/trampoline/python_dbr_warning_test.go index d7c1eb5f46..4d9fa7b334 100644 --- a/bundle/trampoline/python_dbr_warning_test.go +++ b/bundle/trampoline/python_dbr_warning_test.go @@ -151,7 +151,7 @@ func TestIncompatibleWheelTasksWithExistingClusterId(t *testing.T) { diags := hasIncompatibleWheelTasks(context.Background(), b) require.NotEmpty(t, diags) - require.ErrorContains(t, diags.Error(), "uses incompatible DBR version 12.2.x-scala2.12") + require.ErrorContains(t, diags.Error(), "uses cluster with incompatible DBR version 12.2.x-scala2.12") } func TestNoIncompatibleWheelTasks(t *testing.T) { From 5410d88a9133e8638211f69c84273c44bb40895d Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 9 Apr 2025 17:07:01 +0200 Subject: [PATCH 3/4] add missing continue --- bundle/trampoline/python_dbr_warning.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bundle/trampoline/python_dbr_warning.go b/bundle/trampoline/python_dbr_warning.go index a7f77ff8ec..d871193f6e 100644 --- a/bundle/trampoline/python_dbr_warning.go +++ b/bundle/trampoline/python_dbr_warning.go @@ -55,6 +55,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagn Severity: diag.Error, Summary: fmt.Sprintf("task %s uses cluster with incompatible DBR version %s", task.TaskKey, task.NewCluster.SparkVersion), }) + continue } } @@ -67,6 +68,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagn Severity: diag.Error, Summary: fmt.Sprintf("job cluster %s uses incompatible DBR version %s", cluster.JobClusterKey, cluster.NewCluster.SparkVersion), }) + continue } } } @@ -112,6 +114,7 @@ func hasIncompatibleWheelTasks(ctx context.Context, b *bundle.Bundle) diag.Diagn Severity: diag.Error, Summary: fmt.Sprintf("task %s uses cluster with incompatible DBR version %s", task.TaskKey, version), }) + continue } } } From 0feaea368466b381e50e1227fc48a5fb2e1f58fd Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Fri, 11 Apr 2025 15:28:15 +0200 Subject: [PATCH 4/4] next changelog --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 1c129d08cf..f28fe2ff2b 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -9,5 +9,6 @@ ### CLI ### Bundles +* Do not exit early when checking incompatible tasks for specified DBR ([#2692](https://github.com/databricks/cli/pull/2692)) ### API Changes