diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index d53d3a2816..54e0ad8e8d 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -3,6 +3,11 @@ ## Release v0.266.0 ### Notable Changes +* Breaking change: DABs now return an error when paths are incorrectly defined relative to the job or +pipeline definition location instead of the configuration file location. Previously, the CLI would show a +warning and fallback to resolving the path relative to the resource location. Users must update their bundle +configurations to define all relative paths relative to the configuration file where the path is specified. +See more details here: ([#3225](https://github.com/databricks/cli/pull/3225)) * Add support volumes in Python support ([#3383])(https://github.com/databricks/cli/pull/3383)) ### CLI @@ -10,5 +15,6 @@ ### Dependency updates ### Bundles +* [Breaking Change] Remove deprecated path fallback mechanism for jobs and pipelines ([#3225](https://github.com/databricks/cli/pull/3225)) ### API Changes diff --git a/acceptance/bundle/paths/fallback/output.txt b/acceptance/bundle/paths/fallback/output.txt index 6b819fe9c0..5bdbb9942c 100644 --- a/acceptance/bundle/paths/fallback/output.txt +++ b/acceptance/bundle/paths/fallback/output.txt @@ -5,14 +5,36 @@ Warning: required field "entry_point" is not set in resources/my_job.yml:33:13 override_job.yml:29:17 -Warn: path ../src/notebook.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:9:32). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. -Warn: path ../src/file.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:13:30). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. -Warn: path ../src/dbt_project is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:17:36). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. -Warn: path ../src/sql.sql is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:24:25). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. -Warn: path ../dist/wheel2.whl is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:33:24). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. -Warn: path ../target/jar2.jar is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:41:24). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. -Warn: path ../src/notebook2.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_pipeline.yml:13:23). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. -Warn: path ../src/file2.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_pipeline.yml:10:23). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release. +Error: path ../src/notebook.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:9:32). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier). + in override_job.yml:9:32 + resources/my_job.yml:9:28 + +Error: path ../src/file.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:13:30). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier). + in override_job.yml:13:30 + resources/my_job.yml:14:26 + +Error: path ../src/dbt_project is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:17:36). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier). + in override_job.yml:17:36 + resources/my_job.yml:19:32 + +Error: path ../src/sql.sql is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:24:25). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier). + in override_job.yml:24:25 + resources/my_job.yml:27:21 + +Error: path ../dist/wheel2.whl is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:33:24). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier). + in override_job.yml:33:24 + +Error: path ../target/jar2.jar is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_job.yml:41:24). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier). + in override_job.yml:41:24 + +Error: path ../src/notebook2.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_pipeline.yml:13:23). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier). + in override_pipeline.yml:13:23 + +Error: path ../src/file2.py is defined relative to the [TEST_TMP_DIR]/resources directory ([TEST_TMP_DIR]/override_pipeline.yml:10:23). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier). + in override_pipeline.yml:10:23 + + +Exit code: 1 >>> [CLI] bundle validate -t error Warning: required field "entry_point" is not set diff --git a/acceptance/bundle/paths/fallback_metric/databricks.yml b/acceptance/bundle/paths/fallback_metric/databricks.yml deleted file mode 100644 index 3655dc1d2d..0000000000 --- a/acceptance/bundle/paths/fallback_metric/databricks.yml +++ /dev/null @@ -1,18 +0,0 @@ -bundle: - name: fallback_metric - -include: - - "resources/*.yml" - -targets: - fallback: - resources: - jobs: - my_job: - tasks: - - task_key: notebook_example - notebook_task: - notebook_path: ../src/notebook.py - - default: - default: true diff --git a/acceptance/bundle/paths/fallback_metric/out.test.toml b/acceptance/bundle/paths/fallback_metric/out.test.toml deleted file mode 100644 index 8f3575be7b..0000000000 --- a/acceptance/bundle/paths/fallback_metric/out.test.toml +++ /dev/null @@ -1,5 +0,0 @@ -Local = true -Cloud = false - -[EnvMatrix] - DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/paths/fallback_metric/resources/job.yml b/acceptance/bundle/paths/fallback_metric/resources/job.yml deleted file mode 100644 index a345674ab6..0000000000 --- a/acceptance/bundle/paths/fallback_metric/resources/job.yml +++ /dev/null @@ -1,18 +0,0 @@ -resources: - jobs: - my_job: - name: "placeholder" - tasks: - - task_key: notebook_example - new_cluster: - spark_version: 15.4.x-scala2.12 - node_type_id: i3.xlarge - data_security_mode: SINGLE_USER - num_workers: 0 - spark_conf: - spark.master: "local[*, 4]" - spark.databricks.cluster.profile: singleNode - custom_tags: - ResourceClass: SingleNode - notebook_task: - notebook_path: "../src/notebook.py" diff --git a/acceptance/bundle/paths/fallback_metric/script b/acceptance/bundle/paths/fallback_metric/script deleted file mode 100644 index 6fbe46ab57..0000000000 --- a/acceptance/bundle/paths/fallback_metric/script +++ /dev/null @@ -1,5 +0,0 @@ -errcode trace $CLI bundle deploy -errcode trace $CLI bundle deploy -t fallback -trace cat out.requests.txt | jq 'select(has("path") and .path == "/telemetry-ext") | .body.protoLogs[] | fromjson | .entry.databricks_cli_log.bundle_deploy_event.experimental.bool_values' - -rm -rf out.requests.txt diff --git a/acceptance/bundle/paths/fallback_metric/src/notebook.py b/acceptance/bundle/paths/fallback_metric/src/notebook.py deleted file mode 100644 index 38d86b79c7..0000000000 --- a/acceptance/bundle/paths/fallback_metric/src/notebook.py +++ /dev/null @@ -1,2 +0,0 @@ -# Databricks notebook source -print("Hello, World!") diff --git a/acceptance/bundle/paths/fallback_metric/test.toml b/acceptance/bundle/paths/fallback_metric/test.toml deleted file mode 100644 index cfd0d852bc..0000000000 --- a/acceptance/bundle/paths/fallback_metric/test.toml +++ /dev/null @@ -1,10 +0,0 @@ -Cloud = false -RecordRequests = true - -Ignore = [ - ".databricks" -] - -[[Repls]] -Old = '\\' -New = '/' diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index 854c13fbca..9a3e492a89 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -2,13 +2,15 @@ package mutator import ( "context" + "fmt" "path/filepath" "slices" "github.com/databricks/cli/bundle/config/mutator/paths" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" ) func (t *translateContext) applyJobTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { @@ -45,8 +47,7 @@ func (t *translateContext) applyJobTranslations(ctx context.Context, v dyn.Value return nv, nil } - // If we failed to rewrite the path, try to rewrite it relative to the fallback directory. - // We only do this for jobs and pipelines because of the comment in [gatherFallbackPaths]. + // If we failed to rewrite the path, it uses an old path format which relied on fallback. if fallback[key] != "" { dir, nerr := locationDirectory(v.Location()) if nerr != nil { @@ -66,8 +67,11 @@ func (t *translateContext) applyJobTranslations(ctx context.Context, v dyn.Value originalValue := dyn.NewValue(originalPath, v.Locations()) nv, nerr := t.rewriteValue(ctx, p, originalValue, fallback[key], opts) if nerr == nil { - t.b.Metrics.AddBoolValue("is_job_path_fallback", true) - log.Warnf(ctx, "path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.", originalPath, fallback[key], v.Location()) + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier).", originalPath, fallback[key], v.Location()), + Locations: v.Locations(), + }) return nv, nil } } diff --git a/bundle/config/mutator/translate_paths_pipelines.go b/bundle/config/mutator/translate_paths_pipelines.go index b65146f796..828981248a 100644 --- a/bundle/config/mutator/translate_paths_pipelines.go +++ b/bundle/config/mutator/translate_paths_pipelines.go @@ -2,12 +2,14 @@ package mutator import ( "context" + "fmt" "path/filepath" "github.com/databricks/cli/bundle/config/mutator/paths" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" ) func (t *translateContext) applyPipelineTranslations(ctx context.Context, v dyn.Value) (dyn.Value, error) { @@ -30,8 +32,7 @@ func (t *translateContext) applyPipelineTranslations(ctx context.Context, v dyn. return nv, nil } - // If we failed to rewrite the path, try to rewrite it relative to the fallback directory. - // We only do this for jobs and pipelines because of the comment in [gatherFallbackPaths]. + // If we failed to rewrite the path, it uses an old path format which relied on fallback. if fallback[key] != "" { dir, nerr := locationDirectory(v.Location()) if nerr != nil { @@ -51,8 +52,11 @@ func (t *translateContext) applyPipelineTranslations(ctx context.Context, v dyn. originalValue := dyn.NewValue(originalPath, v.Locations()) nv, nerr := t.rewriteValue(ctx, p, originalValue, fallback[key], opts) if nerr == nil { - t.b.Metrics.AddBoolValue("is_pipeline_path_fallback", true) - log.Warnf(ctx, "path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined. The current value will no longer be valid in the next release.", originalPath, fallback[key], v.Location()) + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier).", originalPath, fallback[key], v.Location()), + Locations: v.Locations(), + }) return nv, nil } }