From cff630682539d39fa54bdb95346975f6b3321a6a Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 13 May 2025 11:27:12 +0200 Subject: [PATCH 1/3] Fixed normalising requirements file path in dependencies section --- .../environments/dependencies/databricks.yml | 2 + .../environments/dependencies/output.txt | 44 +++++++++++++++---- .../dependencies/resources/job.yml | 13 ++++++ .../bundle/environments/dependencies/script | 2 +- bundle/config/mutator/normalize_paths.go | 14 ++++++ bundle/config/mutator/normalize_paths_test.go | 8 ++++ 6 files changed, 73 insertions(+), 10 deletions(-) create mode 100644 acceptance/bundle/environments/dependencies/resources/job.yml diff --git a/acceptance/bundle/environments/dependencies/databricks.yml b/acceptance/bundle/environments/dependencies/databricks.yml index 45164d7e23..d59bb36624 100644 --- a/acceptance/bundle/environments/dependencies/databricks.yml +++ b/acceptance/bundle/environments/dependencies/databricks.yml @@ -1,6 +1,8 @@ bundle: name: "dependencies" +include: + - resources/*.yml resources: jobs: diff --git a/acceptance/bundle/environments/dependencies/output.txt b/acceptance/bundle/environments/dependencies/output.txt index 7d11b1bf01..c44c81408a 100644 --- a/acceptance/bundle/environments/dependencies/output.txt +++ b/acceptance/bundle/environments/dependencies/output.txt @@ -33,18 +33,44 @@ Deployment complete! "requests[security] @ https://github.com/psf/requests/archive/refs/heads/main.zip" ] } + }, + { + "environment_key": "test_env_2", + "spec": { + "client": "1", + "dependencies": [ + "-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt" + ] + } } ] >>> [CLI] bundle validate -o json [ - "-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt", - "test_package", - "test_package==2.0.1", - "test_package>=2.0.1", - "./dist/*.whl", - "/Workspace/Users/test@databricks.com/test-package.whl", - "beautifulsoup4>=1.0.0,~=1.2.0,<2.0.0", - "beautifulsoup4[security, tests] ~= 4.12.3", - "requests[security] @ https://github.com/psf/requests/archive/refs/heads/main.zip" + { + "environment_key": "test_env", + "spec": { + "client": "1", + "dependencies": [ + "-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt", + "test_package", + "test_package==2.0.1", + "test_package>=2.0.1", + "./dist/*.whl", + "/Workspace/Users/test@databricks.com/test-package.whl", + "beautifulsoup4>=1.0.0,~=1.2.0,<2.0.0", + "beautifulsoup4[security, tests] ~= 4.12.3", + "requests[security] @ https://github.com/psf/requests/archive/refs/heads/main.zip" + ] + } + }, + { + "environment_key": "test_env_2", + "spec": { + "client": "1", + "dependencies": [ + "-r /Workspace/Users/[USERNAME]/.bundle/dependencies/default/files/requirements.txt" + ] + } + } ] diff --git a/acceptance/bundle/environments/dependencies/resources/job.yml b/acceptance/bundle/environments/dependencies/resources/job.yml new file mode 100644 index 0000000000..8d214e9823 --- /dev/null +++ b/acceptance/bundle/environments/dependencies/resources/job.yml @@ -0,0 +1,13 @@ +targets: + default: + default: true + resources: + jobs: + test_job: + name: "Test Job" + environments: + - environment_key: "test_env_2" + spec: + client: "1" + dependencies: + - "-r ../requirements.txt" diff --git a/acceptance/bundle/environments/dependencies/script b/acceptance/bundle/environments/dependencies/script index 3172de20fe..9fa2ed81cd 100644 --- a/acceptance/bundle/environments/dependencies/script +++ b/acceptance/bundle/environments/dependencies/script @@ -5,5 +5,5 @@ trace $CLI bundle validate trace $CLI bundle deploy trace jq -s '.[] | select(.path=="/api/2.2/jobs/create") | .body.environments' out.requests.txt -trace $CLI bundle validate -o json | jq '.resources.jobs.test_job.environments[0].spec.dependencies' +trace $CLI bundle validate -o json | jq '.resources.jobs.test_job.environments' rm out.requests.txt diff --git a/bundle/config/mutator/normalize_paths.go b/bundle/config/mutator/normalize_paths.go index 1a29e69fa5..17e16f14fa 100644 --- a/bundle/config/mutator/normalize_paths.go +++ b/bundle/config/mutator/normalize_paths.go @@ -7,6 +7,7 @@ import ( "net/url" pathlib "path" "path/filepath" + "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator/paths" @@ -84,6 +85,19 @@ func collectGitSourcePaths(b *bundle.Bundle) []dyn.Path { } func normalizePath(path string, location dyn.Location, bundleRootPath string) (string, error) { + // Handle requirements file paths with -r flag + reqPath, ok := strings.CutPrefix(path, "-r ") + if ok { + // Normalize the path part + normalizedPath, err := normalizePath(reqPath, location, bundleRootPath) + if err != nil { + return "", err + } + + // Reconstruct the path with -r flag + return "-r " + normalizedPath, nil + } + pathAsUrl, err := url.Parse(path) if err != nil { return "", err diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index d78c824ceb..10edce27b0 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -67,6 +67,14 @@ func TestNormalizePath_url(t *testing.T) { assert.Equal(t, "s3:///path/to/notebook.py", value) } +func TestNormalizePath_requirementsFile(t *testing.T) { + tmpDir := t.TempDir() + location := dyn.Location{File: filepath.Join(tmpDir, "resources", "job_1.yml")} + value, err := normalizePath("-r ../requirements.txt", location, tmpDir) + assert.NoError(t, err) + assert.Equal(t, "-r requirements.txt", value) +} + func TestLocationDirectory(t *testing.T) { loc := dyn.Location{File: "file", Line: 1, Column: 2} dir, err := locationDirectory(loc) From f588eed20795330d4cc0e71e464d49b46ed64342 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 13 May 2025 11:30:24 +0200 Subject: [PATCH 2/3] added changelog --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index e1de91e651..fbb5b1bd3d 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,5 +10,6 @@ ### Bundles * Removed unused fields from resources.models schema: creation\_timestamp, last\_updated\_timestamp, latest\_versions and user\_id. Using them now raises a warning. +* Fixed normalising requirements file path in dependencies section ([#2861](https://github.com/databricks/cli/pull/2861)) ### API Changes From 280e1a260ca7c90f310e37a2f108a9b33bf64ca7 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Tue, 13 May 2025 12:47:25 +0200 Subject: [PATCH 3/3] trim spaces --- bundle/config/mutator/normalize_paths.go | 1 + bundle/config/mutator/normalize_paths_test.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/bundle/config/mutator/normalize_paths.go b/bundle/config/mutator/normalize_paths.go index 17e16f14fa..992151f65d 100644 --- a/bundle/config/mutator/normalize_paths.go +++ b/bundle/config/mutator/normalize_paths.go @@ -89,6 +89,7 @@ func normalizePath(path string, location dyn.Location, bundleRootPath string) (s reqPath, ok := strings.CutPrefix(path, "-r ") if ok { // Normalize the path part + reqPath = strings.TrimSpace(reqPath) normalizedPath, err := normalizePath(reqPath, location, bundleRootPath) if err != nil { return "", err diff --git a/bundle/config/mutator/normalize_paths_test.go b/bundle/config/mutator/normalize_paths_test.go index 10edce27b0..0a83890ce6 100644 --- a/bundle/config/mutator/normalize_paths_test.go +++ b/bundle/config/mutator/normalize_paths_test.go @@ -73,6 +73,10 @@ func TestNormalizePath_requirementsFile(t *testing.T) { value, err := normalizePath("-r ../requirements.txt", location, tmpDir) assert.NoError(t, err) assert.Equal(t, "-r requirements.txt", value) + + value, err = normalizePath("-r ../requirements.txt", location, tmpDir) + assert.NoError(t, err) + assert.Equal(t, "-r requirements.txt", value) } func TestLocationDirectory(t *testing.T) {