From ebffb677da0c36b1be95a1f6883668936d3011f2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 7 Jul 2025 10:22:17 +0200 Subject: [PATCH 1/4] acc: Add a test case for issue #3109 --- .../artifacts/issue_3109/databricks.yml | 3 +++ .../bundle/artifacts/issue_3109/out.test.toml | 5 ++++ .../bundle/artifacts/issue_3109/output.txt | 7 ++++++ .../artifacts/issue_3109/pyproject.toml | 15 ++++++++++++ acceptance/bundle/artifacts/issue_3109/script | 1 + .../src/job1/databricks_job/hello_job1.py | 0 .../job1/databricks_job/resources/job1.yaml | 23 +++++++++++++++++++ .../bundle/artifacts/issue_3109/test.toml | 1 + 8 files changed, 55 insertions(+) create mode 100644 acceptance/bundle/artifacts/issue_3109/databricks.yml create mode 100644 acceptance/bundle/artifacts/issue_3109/out.test.toml create mode 100644 acceptance/bundle/artifacts/issue_3109/output.txt create mode 100644 acceptance/bundle/artifacts/issue_3109/pyproject.toml create mode 100644 acceptance/bundle/artifacts/issue_3109/script create mode 100644 acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/hello_job1.py create mode 100644 acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/resources/job1.yaml create mode 100644 acceptance/bundle/artifacts/issue_3109/test.toml diff --git a/acceptance/bundle/artifacts/issue_3109/databricks.yml b/acceptance/bundle/artifacts/issue_3109/databricks.yml new file mode 100644 index 0000000000..a95d152760 --- /dev/null +++ b/acceptance/bundle/artifacts/issue_3109/databricks.yml @@ -0,0 +1,3 @@ +include: + - src/job1/databricks_job/resources/*.yaml + - src/job2/databricks_job/resources/*.yaml diff --git a/acceptance/bundle/artifacts/issue_3109/out.test.toml b/acceptance/bundle/artifacts/issue_3109/out.test.toml new file mode 100644 index 0000000000..8f3575be7b --- /dev/null +++ b/acceptance/bundle/artifacts/issue_3109/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/artifacts/issue_3109/output.txt b/acceptance/bundle/artifacts/issue_3109/output.txt new file mode 100644 index 0000000000..29be0e6b54 --- /dev/null +++ b/acceptance/bundle/artifacts/issue_3109/output.txt @@ -0,0 +1,7 @@ +Building wheels... +Error: [TEST_TMP_DIR]/src/job1/databricks_job/resources/dist/*.whl: no matching files + at artifacts.wheels.files[0].source + in :0:0 + + +Exit code: 1 diff --git a/acceptance/bundle/artifacts/issue_3109/pyproject.toml b/acceptance/bundle/artifacts/issue_3109/pyproject.toml new file mode 100644 index 0000000000..e3fd775ecb --- /dev/null +++ b/acceptance/bundle/artifacts/issue_3109/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "my_default_python" +version = "0.0.1" +authors = [{ name = "[USERNAME]" }] +requires-python = ">= 3.11" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/job1/databricks_job"] + +[project.scripts] +main = "my_default_python.main:main" diff --git a/acceptance/bundle/artifacts/issue_3109/script b/acceptance/bundle/artifacts/issue_3109/script new file mode 100644 index 0000000000..ed2b7bca9b --- /dev/null +++ b/acceptance/bundle/artifacts/issue_3109/script @@ -0,0 +1 @@ +$CLI bundle deploy --target dev diff --git a/acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/hello_job1.py b/acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/hello_job1.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/resources/job1.yaml b/acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/resources/job1.yaml new file mode 100644 index 0000000000..4e83188236 --- /dev/null +++ b/acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/resources/job1.yaml @@ -0,0 +1,23 @@ +targets: + dev: + mode: development + artifacts: + wheels: + type: whl + #build: "pants package ::" + build: "uv build --wheel" + resources: + jobs: + job1: + name: job1 + tasks: + - task_key: wheel_task + job_cluster_key: job_cluster + python_wheel_task: + package_name: job1_databricks_job + job_clusters: + - job_cluster_key: job_cluster + new_cluster: + spark_version: 11.3.x-scala2.12 + node_type_id: i3.xlarge + num_workers: 2 diff --git a/acceptance/bundle/artifacts/issue_3109/test.toml b/acceptance/bundle/artifacts/issue_3109/test.toml new file mode 100644 index 0000000000..a030353d57 --- /dev/null +++ b/acceptance/bundle/artifacts/issue_3109/test.toml @@ -0,0 +1 @@ +RecordRequests = false From 148ea7bff5d1be08b8f880866bea67595626f19a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 7 Jul 2025 10:27:33 +0200 Subject: [PATCH 2/4] Fix default Files.Source initialization for whl artifacts --- bundle/artifacts/prepare.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index 3bca30a805..fef28d1175 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -49,21 +49,6 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics b.Metrics.AddBoolValue(metrics.ArtifactBuildCommandIsSet, artifact.BuildCommand != "") b.Metrics.AddBoolValue(metrics.ArtifactFilesIsSet, len(artifact.Files) != 0) - if artifact.Type == "whl" { - if artifact.BuildCommand == "" && len(artifact.Files) == 0 { - artifact.BuildCommand = python.GetExecutable() + " setup.py bdist_wheel" - } - - // Wheel builds write to `./dist`. Pick up all wheel files by default if nothing is specified. - if len(artifact.Files) == 0 { - artifact.Files = []config.ArtifactFile{ - { - Source: filepath.Join(artifact.Path, "dist", "*.whl"), - }, - } - } - } - l := b.Config.GetLocation("artifacts." + artifactName) dirPath := filepath.Dir(l.File) @@ -79,6 +64,21 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics artifact.Path = b.BundleRootPath } + if artifact.Type == "whl" { + if artifact.BuildCommand == "" && len(artifact.Files) == 0 { + artifact.BuildCommand = python.GetExecutable() + " setup.py bdist_wheel" + } + + // Wheel builds write to `./dist`. Pick up all wheel files by default if nothing is specified. + if len(artifact.Files) == 0 { + artifact.Files = []config.ArtifactFile{ + { + Source: filepath.Join(artifact.Path, "dist", "*.whl"), + }, + } + } + } + if !filepath.IsAbs(artifact.Path) { artifact.Path = filepath.Join(dirPath, artifact.Path) } From 2ce70befcb6007cff3781ad3f3876b97f3e9b5bf Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 7 Jul 2025 10:28:35 +0200 Subject: [PATCH 3/4] update test --- acceptance/bundle/artifacts/issue_3109/output.txt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/artifacts/issue_3109/output.txt b/acceptance/bundle/artifacts/issue_3109/output.txt index 29be0e6b54..4daf472fd4 100644 --- a/acceptance/bundle/artifacts/issue_3109/output.txt +++ b/acceptance/bundle/artifacts/issue_3109/output.txt @@ -1,7 +1,6 @@ Building wheels... -Error: [TEST_TMP_DIR]/src/job1/databricks_job/resources/dist/*.whl: no matching files - at artifacts.wheels.files[0].source - in :0:0 - - -Exit code: 1 +Uploading dist/my_default_python-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/dev/files... +Deploying resources... +Updating deployment state... +Deployment complete! From b73c8ac7b60e1367fed2eeae1570bcc728c087fc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 7 Jul 2025 10:57:33 +0200 Subject: [PATCH 4/4] clean up --- acceptance/bundle/artifacts/issue_3109/databricks.yml | 1 - .../issue_3109/src/job1/databricks_job/resources/job1.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/acceptance/bundle/artifacts/issue_3109/databricks.yml b/acceptance/bundle/artifacts/issue_3109/databricks.yml index a95d152760..7de3da62d9 100644 --- a/acceptance/bundle/artifacts/issue_3109/databricks.yml +++ b/acceptance/bundle/artifacts/issue_3109/databricks.yml @@ -1,3 +1,2 @@ include: - src/job1/databricks_job/resources/*.yaml - - src/job2/databricks_job/resources/*.yaml diff --git a/acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/resources/job1.yaml b/acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/resources/job1.yaml index 4e83188236..bd92b7c63d 100644 --- a/acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/resources/job1.yaml +++ b/acceptance/bundle/artifacts/issue_3109/src/job1/databricks_job/resources/job1.yaml @@ -4,7 +4,6 @@ targets: artifacts: wheels: type: whl - #build: "pants package ::" build: "uv build --wheel" resources: jobs: