From 8064e06eb4a105dd35cf2a767ea5cf18db3b8c44 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 13 Mar 2025 10:46:21 +0100 Subject: [PATCH 01/20] Support dynamic_version setting in 'artifacts' section --- acceptance/bin/ziplist.py | 14 ++ .../bundle/artifacts/whl_dynamic/output.txt | 122 +++++++++++++ .../bundle/artifacts/whl_dynamic/script | 36 ++++ .../bundle/artifacts/whl_dynamic/test.toml | 7 + .../other_test_code-0.0.1-py3-none-any.whl | Bin 1832 -> 1893 bytes bundle/artifacts/build.go | 167 +++++++++++++----- bundle/artifacts/expand_globs.go | 25 ++- bundle/config/artifact.go | 8 +- bundle/internal/schema/annotations.yml | 3 + bundle/libraries/switch_to_patched_wheels.go | 80 +++++++++ bundle/libraries/upload.go | 8 + bundle/phases/deploy.go | 2 + 12 files changed, 425 insertions(+), 47 deletions(-) create mode 100755 acceptance/bin/ziplist.py create mode 100644 acceptance/bundle/artifacts/whl_dynamic/output.txt create mode 100644 acceptance/bundle/artifacts/whl_dynamic/script create mode 100644 acceptance/bundle/artifacts/whl_dynamic/test.toml create mode 100644 bundle/libraries/switch_to_patched_wheels.go diff --git a/acceptance/bin/ziplist.py b/acceptance/bin/ziplist.py new file mode 100755 index 0000000000..35f2fe1239 --- /dev/null +++ b/acceptance/bin/ziplist.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +""" +List files in zip archive +""" + +import sys +import zipfile +from pathlib import Path + + +for zip_path in sys.argv[1:]: + with zipfile.ZipFile(zip_path) as z: + for info in z.infolist(): + print(f"{info.filename} {info.file_size}") diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt new file mode 100644 index 0000000000..8e54f02fd8 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -0,0 +1,122 @@ + +>>> [CLI] bundle validate -o json +{ + "my_prebuilt_whl": { + "dynamic_version": true, + "files": [ + { + "source": "[TEST_TMP_DIR]/prebuilt/other_test_code-0.0.1-py3-none-any.whl" + } + ], + "path": "[TEST_TMP_DIR]", + "type": "whl" + }, + "my_test_code": { + "build": "python setup.py bdist_wheel", + "dynamic_version": true, + "path": "[TEST_TMP_DIR]/my_test_code", + "type": "whl" + } +} + +>>> [CLI] bundle deploy +Building my_test_code... +Uploading .databricks/bundle/default/patched_wheels/my_prebuilt_whl_other_test_code/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl... +Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== There are 2 original wheels and 2 patched ones +>>> find.py --expect 4 whl +.databricks/bundle/default/patched_wheels/my_prebuilt_whl_other_test_code/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl +.databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl +my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl +prebuilt/other_test_code-0.0.1-py3-none-any.whl + +=== Verify contents, there should be no new_module.py +>>> find.py --expect 1 .databricks/.*my_test_code.*whl +src/__init__.py 48 +src/__main__.py 242 +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/METADATA 313 +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/WHEEL 91 +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/entry_points.txt 34 +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/top_level.txt 4 +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD 657 + +=== Expecting 2 wheels in libraries section in /jobs/create +>>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 2 wheels to be uploaded +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/prebuilt/other_test_code-0.0.1-py3-none-any.whl" + +=== Updating the local wheel and deploying again +>>> [CLI] bundle deploy +Building my_test_code... +Uploading .databricks/bundle/default/patched_wheels/my_prebuilt_whl_other_test_code/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl... +Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Verify contents, there should have new_module.py +>>> find.py --expect 1 .databricks/.*my_test_code.*whl +src/__init__.py 48 +src/__main__.py 242 +src/new_module.py 0 +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/METADATA 313 +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/WHEEL 91 +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/entry_points.txt 34 +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/top_level.txt 4 +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD 728 + +=== Expecting 2 wheels in libraries section in /jobs/reset +>>> jq -s .[] | select(.path=="/api/2.1/jobs/reset") | .body.new_settings.tasks out.requests.txt +[ + { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" + }, + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "TestTask" + } +] + +=== Expecting 2 wheels to be uploaded (Bad: it is currently uploaded twice) +>>> jq .path +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" +"/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/files/my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script new file mode 100644 index 0000000000..dad02bcdc1 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -0,0 +1,36 @@ +trap "rm -fr out.requests.txt databricks.yml my_test_code prebuilt" EXIT + +cp -r $TESTDIR/../whl_explicit/my_test_code . +mkdir prebuilt +cp -r $TESTDIR/../whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl prebuilt + +trace $CLI bundle validate -o json | jq .artifacts + +trace $CLI bundle deploy + +title "There are 2 original wheels and 2 patched ones" +trace find.py --expect 4 whl + +title "Verify contents, there should be no new_module.py" +trace find.py --expect 1 '.databricks/.*my_test_code.*whl' | xargs ziplist.py + +title "Expecting 2 wheels in libraries section in /jobs/create" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt + +title "Expecting 2 wheels to be uploaded" +trace jq .path < out.requests.txt | grep import | grep whl | sort + +rm out.requests.txt + +title "Updating the local wheel and deploying again" +touch my_test_code/src/new_module.py +trace $CLI bundle deploy + +title "Verify contents, there should have new_module.py" +trace find.py --expect 1 '.databricks/.*my_test_code.*whl' | xargs ziplist.py + +title "Expecting 2 wheels in libraries section in /jobs/reset" +trace jq -s '.[] | select(.path=="/api/2.1/jobs/reset") | .body.new_settings.tasks' out.requests.txt + +title "Expecting 2 wheels to be uploaded (Bad: it is currently uploaded twice)" +trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/acceptance/bundle/artifacts/whl_dynamic/test.toml b/acceptance/bundle/artifacts/whl_dynamic/test.toml new file mode 100644 index 0000000000..3f0fd04938 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_dynamic/test.toml @@ -0,0 +1,7 @@ +[[Repls]] +Old = '1\d{18}' +New = '[TIMESTAMP_NS]' + +[[Server]] +Pattern = "POST /api/2.1/jobs/reset" +Response.Body = '{}' diff --git a/acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl b/acceptance/bundle/artifacts/whl_prebuilt_multiple/dist/lib/other_test_code-0.0.1-py3-none-any.whl index 4bb80477caf51393354453b18c0c88711098825e..c2cde02cc53b63e1ae67fa5b66fa34ca3a3de3a2 100644 GIT binary patch literal 1893 zcmWIWW@Zs#U|`^2U~S2ZI$`~MsveMM0L1)2TwIi_A0MBYmst`YuUAmn-LLPht997mP0$hdk=)_t()qc|yO5W0%S_6*bXRwRCaqLPmxF6r&C6u76$xH1ZRQ z(YcA4d0?YM`z-km83?#m&wq5<#Pcwt($cm`Or4jyK1f7=QajH1>FrDvR)eSapLuE? zy(^?~?b3wF-J&NKY%0mVx~!vJ^^;SP-;N2vD`(u+X}ORhpeDY%PT+y{!@Rpa4tFk9 zSihfm+H8C7(@9~=J(IQ8tA5yA`Tk9#U);&xEWtqz83x`pr~X~vzvP!LU%lMsIX9!K z)r#1>j%rQ_l5qD}YG$|e31h{9E`Ee}G*U~WGUomASOfIRFCf+c;{1|~)S~#3)Z&u( zJwv^e%;FN=%)GRGeP7oQM;FHs$Jn!ud`$)dtq*JcFLH}qNDZI0!60z; z#TBo)7OxFrvv%g$F1{$~(YN#F`PU9?+mdrtP^ogw);Vgzi8&0q6J=eFI;`Q+aqLm? z-X!<-wm{CCEj)K;pK3g{oO#6&Cgv5#;(kjWzcaz_^24*1zaIH{Haeb(E|b`L=J+2^ zuBNY3r(( zNgcgTefE`s7Yr^Kn_V})@PzBsdHsuDb*}Mh=z5(!>9Z+FLsQH5DVL|u_8^TFCfClM zKJR_i_sr@ikC>3smpU*)-p)<}Fj(uWQ}?=Y7wd>F(do6c`hOW}DXDeGA!uwy*%PE)lkso=qGF&F+8OiWWXXNB%?DDTqJJ)0U_LTUKfAtT3ax0dGb309`o7kqsEdBTD5%c-W zwj}MfKlos-%>7MkFFfaSVE%mK>#NK=x7tmoebjMGHvJQx7kT$@P{7Oj^UEebed%E> z-f-77cGG2tzFX;GCOdgb&b!1fSrO#SB(P@23JvupzNRN{yi!90r*17;%dU7m_Fx>J zUr&z6;&%IG-!C8KzIV9Mh2!^8*_~3CoerN)J^!Bhdp?KohMk{|mZ@wAod2uywZ(6C zzvnk4GtXWV+Yw+s@6p$7#_Q&*ujg5Q)Jtcf+Vh0`trz<|cRzo>MYu-McD1#k#-nH( zBlFjb&!*p75g~F2iC1ukMpa zF+JP+x4B%D&F9!~%7o!V!Yc8WwLzjqmHA(cp0RV<`6e9a+NkTfx<0_0kx7IZcL52E zDKJ>l2%^x6OLWcXMIb~g1H+QWXt-v0frzXbTY-bntOZOqgi<@K_(3-nJyRo0l>sJ1 zLZ*VVII@x0G9tpr-9#9Po-NT$N6&Z&(>0k1I~XM!A{&h@3&M=v!dOLw(U1%o;LQpu Q78uxp&;aONc~%e)06LtWdjJ3c delta 1203 zcmaFLw}Ou^z?+#xgn@y9gW+<~UZ-UQV?k?pZAELZ%a4- zg(aK&_|*EAOl+P|z`fLT&)zMvSB(E{|6F+P=3N1eDKQfl@v;UkHhCG=-O0%B{6r$( zGOf02XTmz;WS@y`^K)MOYFv1ivphMyB1^Y*dnuo0e(~+FQ-xc*M3()Ds;M@6KiBp` zr?#c|!s0IH9cbA(TC@=b)KiKtySJ671VZe@)QYo4o!1A&7+JJ2R88| zJfL687;>>bnCwR#joNXS*aE4NLf~AnEi1U(_C`! z@MMd}FRG3mj=%9}_mZBZ*($Fb15PhF(=73s71iCjR8|+JLVEH9Hi5~U%)BPS8$vboysmZkpZ7g;rn`STQ(z1Osufy$_bp@t zYGeUoErbE@*mx(GFbja~NY0l29?QtUpv%O-pbS@1ALQ!nALP<{?sQR;0fWQEz4wJQ z5*%Vn4z5<}N~*c!%Vy}H!=fU1aQ%Cq^jn$cU)j&?-QyW&u&jrx=jkqXn|{-~6UcBRamNAAQng7++MBfIYl;h{W_}gg z5q;Tk`!n;rC4c{M65#QzyKJurYB

E{yIlt-XylkkFt0tYcWh#zh2+cx$JnU>&yq6 zUL5g1Y`KPSGVk^-{vFEGyJvN2Y}sY<{cE|sp!$pTWt&&tQW0Wt{!#B7Vyfri-F3fF zFz1Nf-l?yZj1RM4+L)GpF{rt5sf=$*de?+a$J;07&StRrI5B(u1m*(|gaf=8nM9Zo z`SFDH^Qn5kI51$CY{{yR$eNQ2Sxr>1rFWoOFj&&44U9ra>Yx0aRRpXUJ$)e*|Ai@5 zFhF(!wj_tpG!vLWpf*fklL6a+p1cr>S()LA-?2%86{9CKnBpys!3f2k>_S|MK#M+t HQY-@iqgc6< diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 8cc407b06f..0ee7ee05a3 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -3,6 +3,7 @@ package artifacts import ( "context" "fmt" + "os" "path/filepath" "github.com/databricks/cli/bundle" @@ -11,6 +12,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/exec" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/patchwheel" "github.com/databricks/cli/libs/python" ) @@ -27,61 +29,146 @@ func (m *build) Name() string { func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics + cacheDir, err := b.CacheDir(ctx) + if err != nil { + diags = append(diags, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "Failed to set up cache directory", + }) + } + for _, artifactName := range sortedKeys(b.Config.Artifacts) { a := b.Config.Artifacts[artifactName] - if a.BuildCommand == "" { - continue - } + if a.BuildCommand != "" { + err := doBuild(ctx, artifactName, a) + if err != nil { + diags = diags.Extend(diag.FromErr(err)) + break + } - cmdio.LogString(ctx, fmt.Sprintf("Building %s...", artifactName)) + if a.Type == "whl" { + dir := a.Path + distPath := filepath.Join(a.Path, "dist") + wheels := python.FindFilesWithSuffixInPath(distPath, ".whl") + if len(wheels) == 0 { + diags = diags.Extend(diag.Errorf("cannot find built wheel in %s for package %s", dir, artifactName)) + break + } + for _, wheel := range wheels { + a.Files = append(a.Files, config.ArtifactFile{ + Source: wheel, + }) + } + } else { + log.Warnf(ctx, "%s: Build succeeded", artifactName) + } - var executor *exec.Executor - var err error - if a.Executable != "" { - executor, err = exec.NewCommandExecutorWithExecutable(a.Path, a.Executable) - } else { - executor, err = exec.NewCommandExecutor(a.Path) - a.Executable = executor.ShellType() - } + // We need to expand glob reference after build mutator is applied because + // if we do it before, any files that are generated by build command will + // not be included into artifact.Files and thus will not be uploaded. + // We only do it if BuildCommand was specified because otherwise it should have been done already by artifacts.Prepare() + diags = diags.Extend(bundle.Apply(ctx, b, expandGlobs{name: artifactName})) + + // After bundle.Apply is called, all of b.Config is recreated and all pointers are invalidated (!) + a = b.Config.Artifacts[artifactName] - if err != nil { - diags = diags.Extend(diag.FromErr(err)) if diags.HasError() { break } - } - out, err := executor.Exec(ctx, a.BuildCommand) - if err != nil { - return diag.Errorf("build failed %s, error: %v, output: %s", artifactName, err, out) } - if a.Type == "whl" { - dir := a.Path - distPath := filepath.Join(a.Path, "dist") - wheels := python.FindFilesWithSuffixInPath(distPath, ".whl") - if len(wheels) == 0 { - return diag.Errorf("cannot find built wheel in %s for package %s", dir, artifactName) - } - for _, wheel := range wheels { - a.Files = append(a.Files, config.ArtifactFile{ - Source: wheel, - }) + if a.Type == "whl" && a.DynamicVersion && cacheDir != "" { + for ind, artifactFile := range a.Files { + patchedWheel, extraDiags := mkPatchedWheel(ctx, cacheDir, artifactName, artifactFile.Source) + log.Debugf(ctx, "Patching ind=%d artifactName=%s Source=%s patchedWheel=%s", ind, artifactName, artifactFile.Source, patchedWheel) + diags = append(diags, extraDiags...) + if patchedWheel != "" { + a.Files[ind].Patched = patchedWheel + } + if extraDiags.HasError() { + break + } } - log.Infof(ctx, "%s: Build succeeded, found %s", artifactName, wheels) - } else { - log.Infof(ctx, "%s: Build succeeded", artifactName) - } - - // We need to expand glob reference after build mutator is applied because - // if we do it before, any files that are generated by build command will - // not be included into artifact.Files and thus will not be uploaded. - diags = diags.Extend(bundle.Apply(ctx, b, expandGlobs{name: artifactName})) - if diags.HasError() { - break } } return diags } + +func doBuild(ctx context.Context, artifactName string, a *config.Artifact) error { + cmdio.LogString(ctx, fmt.Sprintf("Building %s...", artifactName)) + + var executor *exec.Executor + var err error + if a.Executable != "" { + executor, err = exec.NewCommandExecutorWithExecutable(a.Path, a.Executable) + } else { + executor, err = exec.NewCommandExecutor(a.Path) + a.Executable = executor.ShellType() + } + + if err != nil { + return err + } + + out, err := executor.Exec(ctx, a.BuildCommand) + if err != nil { + return fmt.Errorf("build failed %s, error: %v, output: %s", artifactName, err, out) + } + + return nil +} + +func mkPatchedWheel(ctx context.Context, cacheDir, artifactName, wheel string) (string, diag.Diagnostics) { + msg := "Failed to patch wheel with dynamic version" + info, err := patchwheel.ParseWheelFilename(wheel) + if err != nil { + return "", []diag.Diagnostic{{ + Severity: diag.Warning, + Summary: msg, + Detail: fmt.Sprintf("When parsing filename \"%s\" encountered an error: %s", wheel, err), + }} + } + + dir := filepath.Join(cacheDir, "patched_wheels", artifactName+"_"+info.Distribution) + createAndCleanupDirectory(ctx, dir) + patchedWheel, isBuilt, err := patchwheel.PatchWheel(wheel, dir) + if err != nil { + return "", []diag.Diagnostic{{ + Severity: diag.Warning, + Summary: msg, + Detail: fmt.Sprintf("When patching %s encountered an error: %s", wheel, err), + }} + } + + if isBuilt { + log.Infof(ctx, "Patched wheel (cache) %s -> %s", wheel, patchedWheel) + } else { + log.Infof(ctx, "Patched wheel (built) %s -> %s", wheel, patchedWheel) + } + + return patchedWheel, nil +} + +func createAndCleanupDirectory(ctx context.Context, dir string) { + err := os.MkdirAll(dir, 0o700) + if err != nil { + log.Infof(ctx, "Failed to create %s: %s", dir, err) + return + } + + files, err := os.ReadDir(dir) + if err != nil { + log.Infof(ctx, "Failed to clean up %s: %s", dir, err) + // ReadDir can return partial results, so continue here + } + for _, entry := range files { + path := filepath.Join(dir, entry.Name()) + err := os.Remove(path) + if err != nil { + log.Infof(ctx, "Failed to clean up %s: %s", path, err) + } + } +} diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index 72d1390889..1d9e188a0c 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/log" ) func createGlobError(v dyn.Value, p dyn.Path, message string) diag.Diagnostic { @@ -53,9 +54,9 @@ func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti ) var diags diag.Diagnostics - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + err := b.Config.Mutate(func(rootv dyn.Value) (dyn.Value, error) { var output []dyn.Value - _, err := dyn.MapByPattern(v, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + _, err := dyn.MapByPattern(rootv, pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { if v.Kind() != dyn.KindString { return v, nil } @@ -67,14 +68,26 @@ func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti if err != nil { diags = diags.Append(createGlobError(v, p, err.Error())) - // Continue processing and leave this value unchanged. + // Drop this value from the list; this does not matter since we've raised an error anyway + return v, nil + } + + if len(matches) == 1 && matches[0] == source { + // No glob expansion was performed. + // Keep node unchanged. We need to ensure that "patched" field remains and not wiped out by code below. + parent, err := dyn.GetByPath(rootv, p[0:len(p)-1]) + if err != nil { + log.Debugf(ctx, "Failed to get parent of %s", p.String()) + } else { + output = append(output, parent) + } return v, nil } if len(matches) == 0 { diags = diags.Append(createGlobError(v, p, "no matching files")) - // Continue processing and leave this value unchanged. + // Drop this value from the list; this does not matter since we've raised an error anyway return v, nil } @@ -90,11 +103,11 @@ func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti }) if err != nil || diags.HasError() { - return v, err + return rootv, err } // Set the expanded globs back into the configuration. - return dyn.SetByPath(v, base, dyn.V(output)) + return dyn.SetByPath(rootv, base, dyn.V(output)) }) if err != nil { diags = diags.Extend(diag.FromErr(err)) diff --git a/bundle/config/artifact.go b/bundle/config/artifact.go index 93c220136f..4491954ab9 100644 --- a/bundle/config/artifact.go +++ b/bundle/config/artifact.go @@ -11,7 +11,11 @@ type ArtifactType string const ArtifactPythonWheel ArtifactType = `whl` type ArtifactFile struct { - Source string `json:"source"` + Source string `json:"source"` + + // Patched is populated if DynamicVersion is set and patching was successful + Patched string `json:"patched" bundle:"readonly"` + RemotePath string `json:"remote_path" bundle:"readonly"` } @@ -30,4 +34,6 @@ type Artifact struct { BuildCommand string `json:"build,omitempty"` Executable exec.ExecutableType `json:"executable,omitempty"` + + DynamicVersion bool `json:"dynamic_version,omitempty"` } diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 92e76c70fe..2fd6373d0e 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -5,6 +5,9 @@ github.com/databricks/cli/bundle/config.Artifact: "executable": "description": |- The executable type. Valid values are `bash`, `sh`, and `cmd`. + "dynamic_version": + "description": |- + Whether to patch the wheel version with dynamic component, based on timestamp of whl file. This ensures that the new code is deployed without having to update version in setup.py / pyproject.toml. Only applies to type=whl. "files": "description": |- The relative or absolute path to the built artifact files. diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go new file mode 100644 index 0000000000..1a200ba97f --- /dev/null +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -0,0 +1,80 @@ +package libraries + +import ( + "context" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/log" +) + +type switchToPatchedWheels struct{} + +func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { + replacements := getReplacements(ctx, b.Config.Artifacts, b.BundleRoot.Native()) + + for jobName, jobRef := range b.Config.Resources.Jobs { + if jobRef == nil { + continue + } + + job := jobRef.JobSettings + + if job == nil { + continue + } + + // resources.jobs.*.task[*].libraries[*] + + for taskInd, task := range job.Tasks { + for libInd, lib := range task.Libraries { + repl := replacements[lib.Whl] + if repl != "" { + log.Debugf(ctx, "Updating resources.jobs.%s.task[%d].libraries[%d].whl from %s to %s", jobName, taskInd, libInd, lib.Whl, repl) + task.Libraries[libInd].Whl = repl + } + } + } + + // resources.jobs.*.task[*].*.for_each_task.task.libraries + // TODO + + // resources.jobs.*.environments.*.spec.dependencies + // TODO + } + + return nil +} + +func getReplacements(ctx context.Context, artifacts config.Artifacts, bundleRoot string) map[string]string { + result := make(map[string]string) + for _, artifact := range artifacts { + for _, file := range artifact.Files { + if file.Patched != "" { + source, err := filepath.Rel(bundleRoot, file.Source) + if err != nil { + log.Debugf(ctx, "Failed to get relative path (%s, %s): %s", bundleRoot, file.Source, err) + continue + } + patched, err := filepath.Rel(bundleRoot, file.Patched) + if err != nil { + log.Debugf(ctx, "Failed to get relative path (%s, %s): %s", bundleRoot, file.Patched, err) + continue + } + result[source] = patched + // There already was a check for duplicate by same_name_libraries.go + } + } + } + return result +} + +func (c switchToPatchedWheels) Name() string { + return "SwitchToPatchedWheels" +} + +func SwitchToPatchedWheels() bundle.Mutator { + return switchToPatchedWheels{} +} diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index b8fe1bac74..c4edff4042 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -111,6 +111,14 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error return v, fmt.Errorf("expected string, got %s", v.Kind()) } + sv, ok = file.GetByString("patched") + if ok { + patched, ok := sv.AsString() + if ok && patched != "" { + source = patched + } + } + libs[source] = append(libs[source], configLocation{ configPath: p.Append(dyn.Key("remote_path")), location: v.Location(), diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index e0a1761cf6..e15e7f1506 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -195,6 +195,8 @@ func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHand // mutator is part of the deploy step rather than validate. libraries.ExpandGlobReferences(), libraries.CheckForSameNameLibraries(), + // SwitchToPatchedWheels must be run after ExpandGlobReferences and after build phase because it Artifact.Source and Artifact.Patched populated + libraries.SwitchToPatchedWheels(), libraries.Upload(), trampoline.TransformWheelTask(), files.Upload(outputHandler), From a0d65e4071d97dc7b7709bca74e99d0f79694c75 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 21 Mar 2025 19:01:25 +0100 Subject: [PATCH 02/20] TMP run integration test --- .../interactive_cluster/databricks.yml.tmpl | 7 +++++++ acceptance/bundle/integration_whl/test.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl b/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl index d59e9a4328..bb47ff423d 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl +++ b/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl @@ -4,6 +4,13 @@ bundle: workspace: root_path: "~/.bundle/$UNIQUE_NAME" +artifacts: + python_artifact: + type: whl + files: + - source: dist/*.whl + dynamic_version: true + resources: clusters: test_cluster: diff --git a/acceptance/bundle/integration_whl/test.toml b/acceptance/bundle/integration_whl/test.toml index 50228c95a1..82e4d48d61 100644 --- a/acceptance/bundle/integration_whl/test.toml +++ b/acceptance/bundle/integration_whl/test.toml @@ -1,5 +1,5 @@ Local = false -CloudSlow = true +#CloudSlow = true Ignore = [ ".databricks", "build", From 0388ba3303a9c53d447168e995c2ea5bf5f10bcb Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Fri, 21 Mar 2025 19:02:16 +0100 Subject: [PATCH 03/20] add missing databricks.yml --- .../artifacts/whl_dynamic/databricks.yml | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 acceptance/bundle/artifacts/whl_dynamic/databricks.yml diff --git a/acceptance/bundle/artifacts/whl_dynamic/databricks.yml b/acceptance/bundle/artifacts/whl_dynamic/databricks.yml new file mode 100644 index 0000000000..7e31d36110 --- /dev/null +++ b/acceptance/bundle/artifacts/whl_dynamic/databricks.yml @@ -0,0 +1,29 @@ +bundle: + name: python-wheel + +artifacts: + my_test_code: + type: whl + path: "./my_test_code" + # using 'python' there because 'python3' does not exist in virtualenv on windows + build: python setup.py bdist_wheel + dynamic_version: true + my_prebuilt_whl: + type: whl + files: + - source: prebuilt/other_test_code-0.0.1-py3-none-any.whl + dynamic_version: true + +resources: + jobs: + test_job: + name: "[${bundle.target}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./my_test_code/dist/*.whl + - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl From b9b6e07e2a0443dfd0f837bfbfa9bc0575bdcc3d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 23 Mar 2025 11:13:00 +0100 Subject: [PATCH 04/20] windows fix --- acceptance/bin/ziplist.py | 2 +- acceptance/bundle/artifacts/whl_dynamic/test.toml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/acceptance/bin/ziplist.py b/acceptance/bin/ziplist.py index 35f2fe1239..38b128dee5 100755 --- a/acceptance/bin/ziplist.py +++ b/acceptance/bin/ziplist.py @@ -9,6 +9,6 @@ for zip_path in sys.argv[1:]: - with zipfile.ZipFile(zip_path) as z: + with zipfile.ZipFile(zip_path.strip()) as z: for info in z.infolist(): print(f"{info.filename} {info.file_size}") diff --git a/acceptance/bundle/artifacts/whl_dynamic/test.toml b/acceptance/bundle/artifacts/whl_dynamic/test.toml index 3f0fd04938..86e9e54cce 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/test.toml +++ b/acceptance/bundle/artifacts/whl_dynamic/test.toml @@ -1,3 +1,7 @@ +[[Repls]] +Old = '\\\\' +New = '/' + [[Repls]] Old = '1\d{18}' New = '[TIMESTAMP_NS]' From bb1c77107c8d72e248e908d3f84f40042cd0c6cf Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 23 Mar 2025 11:13:50 +0100 Subject: [PATCH 05/20] update schema --- bundle/internal/schema/annotations.yml | 6 +++--- bundle/schema/jsonschema.json | 4 ++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/bundle/internal/schema/annotations.yml b/bundle/internal/schema/annotations.yml index 2fd6373d0e..afe2431da0 100644 --- a/bundle/internal/schema/annotations.yml +++ b/bundle/internal/schema/annotations.yml @@ -2,12 +2,12 @@ github.com/databricks/cli/bundle/config.Artifact: "build": "description": |- An optional set of build commands to run locally before deployment. - "executable": - "description": |- - The executable type. Valid values are `bash`, `sh`, and `cmd`. "dynamic_version": "description": |- Whether to patch the wheel version with dynamic component, based on timestamp of whl file. This ensures that the new code is deployed without having to update version in setup.py / pyproject.toml. Only applies to type=whl. + "executable": + "description": |- + The executable type. Valid values are `bash`, `sh`, and `cmd`. "files": "description": |- The relative or absolute path to the built artifact files. diff --git a/bundle/schema/jsonschema.json b/bundle/schema/jsonschema.json index 38f1dd88ca..8cb57c99ff 100644 --- a/bundle/schema/jsonschema.json +++ b/bundle/schema/jsonschema.json @@ -1098,6 +1098,10 @@ "description": "An optional set of build commands to run locally before deployment.", "$ref": "#/$defs/string" }, + "dynamic_version": { + "description": "Whether to patch the wheel version with dynamic component, based on timestamp of whl file. This ensures that the new code is deployed without having to update version in setup.py / pyproject.toml. Only applies to type=whl.", + "$ref": "#/$defs/bool" + }, "executable": { "description": "The executable type. Valid values are `bash`, `sh`, and `cmd`.", "$ref": "#/$defs/github.com/databricks/cli/libs/exec.ExecutableType" From 13ee78bd5725766f9eb0246be5de7cc923dd32ed Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 23 Mar 2025 11:17:28 +0100 Subject: [PATCH 06/20] interactive_cluster: add build command --- .../integration_whl/interactive_cluster/databricks.yml.tmpl | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl b/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl index bb47ff423d..ebc396aeb0 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl +++ b/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl @@ -7,6 +7,7 @@ workspace: artifacts: python_artifact: type: whl + build: python setup.py bdist_wheel files: - source: dist/*.whl dynamic_version: true From 7f71b64618c22a946f20b4fd5ad2766dac245ece Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 23 Mar 2025 11:29:05 +0100 Subject: [PATCH 07/20] rm size, different on windows --- acceptance/bin/ziplist.py | 2 +- .../bundle/artifacts/whl_dynamic/output.txt | 30 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/acceptance/bin/ziplist.py b/acceptance/bin/ziplist.py index 38b128dee5..5e2fa99d5c 100755 --- a/acceptance/bin/ziplist.py +++ b/acceptance/bin/ziplist.py @@ -11,4 +11,4 @@ for zip_path in sys.argv[1:]: with zipfile.ZipFile(zip_path.strip()) as z: for info in z.infolist(): - print(f"{info.filename} {info.file_size}") + print(info.filename) diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index 8e54f02fd8..f082df2d87 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -37,13 +37,13 @@ prebuilt/other_test_code-0.0.1-py3-none-any.whl === Verify contents, there should be no new_module.py >>> find.py --expect 1 .databricks/.*my_test_code.*whl -src/__init__.py 48 -src/__main__.py 242 -my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/METADATA 313 -my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/WHEEL 91 -my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/entry_points.txt 34 -my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/top_level.txt 4 -my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD 657 +src/__init__.py +src/__main__.py +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/METADATA +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/WHEEL +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/entry_points.txt +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/top_level.txt +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD === Expecting 2 wheels in libraries section in /jobs/create >>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt @@ -85,14 +85,14 @@ Deployment complete! === Verify contents, there should have new_module.py >>> find.py --expect 1 .databricks/.*my_test_code.*whl -src/__init__.py 48 -src/__main__.py 242 -src/new_module.py 0 -my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/METADATA 313 -my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/WHEEL 91 -my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/entry_points.txt 34 -my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/top_level.txt 4 -my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD 728 +src/__init__.py +src/__main__.py +src/new_module.py +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/METADATA +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/WHEEL +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/entry_points.txt +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/top_level.txt +my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD === Expecting 2 wheels in libraries section in /jobs/reset >>> jq -s .[] | select(.path=="/api/2.1/jobs/reset") | .body.new_settings.tasks out.requests.txt From f6b03023dd2ea74f9551499a4daff3eb6ae5eb74 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 23 Mar 2025 12:12:42 +0100 Subject: [PATCH 08/20] restore interactive_cluster --- .../interactive_cluster/databricks.yml.tmpl | 8 -------- 1 file changed, 8 deletions(-) diff --git a/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl b/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl index ebc396aeb0..d59e9a4328 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl +++ b/acceptance/bundle/integration_whl/interactive_cluster/databricks.yml.tmpl @@ -4,14 +4,6 @@ bundle: workspace: root_path: "~/.bundle/$UNIQUE_NAME" -artifacts: - python_artifact: - type: whl - build: python setup.py bdist_wheel - files: - - source: dist/*.whl - dynamic_version: true - resources: clusters: test_cluster: From 1b36e6eda4faff9e57836ebdd798967f8f293205 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Sun, 23 Mar 2025 13:04:20 +0100 Subject: [PATCH 09/20] add interactive_cluster_dynamic_version --- .../databricks.yml.tmpl | 37 +++++++++++++++ .../output.txt | 45 +++++++++++++++++++ .../script | 10 +++++ .../test.toml | 5 +++ 4 files changed, 97 insertions(+) create mode 100644 acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/databricks.yml.tmpl create mode 100644 acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt create mode 100644 acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/script create mode 100644 acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/databricks.yml.tmpl b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/databricks.yml.tmpl new file mode 100644 index 0000000000..093e064f32 --- /dev/null +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/databricks.yml.tmpl @@ -0,0 +1,37 @@ +bundle: + name: wheel-task + +workspace: + root_path: "~/.bundle/$UNIQUE_NAME" + +artifacts: + python_artifact: + type: whl + build: uv build --wheel + files: + - source: dist/*.whl + dynamic_version: true + +resources: + clusters: + test_cluster: + cluster_name: "test-cluster-$UNIQUE_NAME" + spark_version: "$DEFAULT_SPARK_VERSION" + node_type_id: "$NODE_TYPE_ID" + num_workers: 1 + data_security_mode: $DATA_SECURITY_MODE + + jobs: + some_other_job: + name: "[${bundle.target}] Test Wheel Job $UNIQUE_NAME" + tasks: + - task_key: TestTask + existing_cluster_id: "${resources.clusters.test_cluster.cluster_id}" + python_wheel_task: + package_name: my_test_code + entry_point: run + parameters: + - "one" + - "two" + libraries: + - whl: ./dist/*.whl diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt new file mode 100644 index 0000000000..c3b514ebfc --- /dev/null +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt @@ -0,0 +1,45 @@ + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading .databricks/bundle/default/patched_wheels/python_artifact_my_test_code/my_test_code-0.0.1+[NUMID]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS +Hello from my func +Got arguments: +['my_test_code', 'one', 'two'] + +=== Make a change to code without version change and run the job again +>>> [CLI] bundle deploy +Building python_artifact... +Uploading .databricks/bundle/default/patched_wheels/python_artifact_my_test_code/my_test_code-0.0.1+[NUMID]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS +UPDATED MY FUNC +Got arguments: +['my_test_code', 'one', 'two'] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete cluster test_cluster + delete job some_other_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/script b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/script new file mode 100644 index 0000000000..e59d0c0dad --- /dev/null +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/script @@ -0,0 +1,10 @@ +envsubst < databricks.yml.tmpl > databricks.yml +cp -r $TESTDIR/../interactive_cluster/{setup.py,my_test_code} . +trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT +trace $CLI bundle deploy +trace $CLI bundle run some_other_job + +title "Make a change to code without version change and run the job again" +update_file.py my_test_code/__main__.py 'Hello from my func' 'UPDATED MY FUNC' +trace $CLI bundle deploy +trace $CLI bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml new file mode 100644 index 0000000000..35f395f3c4 --- /dev/null +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/test.toml @@ -0,0 +1,5 @@ +[EnvMatrix] +DATA_SECURITY_MODE = [ + "USER_ISOLATION", + "SINGLE_USER", +] From 9f9a54182b1821fda3fc3c1554743540446202dd Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 24 Mar 2025 13:51:37 +0100 Subject: [PATCH 10/20] support for_each_task --- .../artifacts/whl_dynamic/databricks.yml | 10 +++++ .../bundle/artifacts/whl_dynamic/output.txt | 40 +++++++++++++++++-- .../bundle/artifacts/whl_dynamic/script | 8 ++-- bundle/libraries/switch_to_patched_wheels.go | 15 +++++-- 4 files changed, 61 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_dynamic/databricks.yml b/acceptance/bundle/artifacts/whl_dynamic/databricks.yml index 7e31d36110..6be8a89830 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/databricks.yml +++ b/acceptance/bundle/artifacts/whl_dynamic/databricks.yml @@ -27,3 +27,13 @@ resources: libraries: - whl: ./my_test_code/dist/*.whl - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl + for_each_task: + inputs: "[1]" + task: + task_key: SubTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./my_test_code/dist/*.whl diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index f082df2d87..cecb4570ca 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -45,11 +45,27 @@ my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/entry_points.txt my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/top_level.txt my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD -=== Expecting 2 wheels in libraries section in /jobs/create +=== Expecting 2 patched wheels in libraries section in /jobs/create >>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt [ { "existing_cluster_id": "0717-132531-5opeqon1", + "for_each_task": { + "inputs": "[1]", + "task": { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "SubTask" + } + }, "libraries": [ { "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" @@ -66,7 +82,7 @@ my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD } ] -=== Expecting 2 wheels to be uploaded +=== Expecting 2 patched wheels to be uploaded >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" @@ -94,11 +110,27 @@ my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/entry_points.txt my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/top_level.txt my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD -=== Expecting 2 wheels in libraries section in /jobs/reset +=== Expecting 2 patched wheels in libraries section in /jobs/reset >>> jq -s .[] | select(.path=="/api/2.1/jobs/reset") | .body.new_settings.tasks out.requests.txt [ { "existing_cluster_id": "0717-132531-5opeqon1", + "for_each_task": { + "inputs": "[1]", + "task": { + "existing_cluster_id": "0717-132531-5opeqon1", + "libraries": [ + { + "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" + } + ], + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "SubTask" + } + }, "libraries": [ { "whl": "/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" @@ -115,7 +147,7 @@ my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD } ] -=== Expecting 2 wheels to be uploaded (Bad: it is currently uploaded twice) +=== Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice) >>> jq .path "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" "/api/2.0/workspace-files/import-file/Workspace/Users/[USERNAME]/.bundle/python-wheel/default/artifacts/.internal/other_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl" diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index dad02bcdc1..3bbfc07bb2 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -14,10 +14,10 @@ trace find.py --expect 4 whl title "Verify contents, there should be no new_module.py" trace find.py --expect 1 '.databricks/.*my_test_code.*whl' | xargs ziplist.py -title "Expecting 2 wheels in libraries section in /jobs/create" +title "Expecting 2 patched wheels in libraries section in /jobs/create" trace jq -s '.[] | select(.path=="/api/2.1/jobs/create") | .body.tasks' out.requests.txt -title "Expecting 2 wheels to be uploaded" +title "Expecting 2 patched wheels to be uploaded" trace jq .path < out.requests.txt | grep import | grep whl | sort rm out.requests.txt @@ -29,8 +29,8 @@ trace $CLI bundle deploy title "Verify contents, there should have new_module.py" trace find.py --expect 1 '.databricks/.*my_test_code.*whl' | xargs ziplist.py -title "Expecting 2 wheels in libraries section in /jobs/reset" +title "Expecting 2 patched wheels in libraries section in /jobs/reset" trace jq -s '.[] | select(.path=="/api/2.1/jobs/reset") | .body.new_settings.tasks' out.requests.txt -title "Expecting 2 wheels to be uploaded (Bad: it is currently uploaded twice)" +title "Expecting 2 pached wheels to be uploaded (Bad: it is currently uploaded twice)" trace jq .path < out.requests.txt | grep import | grep whl | sort diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 1a200ba97f..7fc5e239fc 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -33,13 +33,20 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag repl := replacements[lib.Whl] if repl != "" { log.Debugf(ctx, "Updating resources.jobs.%s.task[%d].libraries[%d].whl from %s to %s", jobName, taskInd, libInd, lib.Whl, repl) - task.Libraries[libInd].Whl = repl + job.Tasks[taskInd].Libraries[libInd].Whl = repl } } - } - // resources.jobs.*.task[*].*.for_each_task.task.libraries - // TODO + // resources.jobs.*.task[*].for_each_task.task.libraries + + for libInd, lib := range task.ForEachTask.Task.Libraries { + repl := replacements[lib.Whl] + if repl != "" { + log.Debugf(ctx, "Updating resources.jobs.%s.task[%d].for_each_task.task.libraries[%d].whl from %s to %s", jobName, taskInd, libInd, lib.Whl, repl) + job.Tasks[taskInd].ForEachTask.Task.Libraries[libInd].Whl = repl + } + } + } // resources.jobs.*.environments.*.spec.dependencies // TODO From f9bd32b132a8a379bb25ef81e57673445807516e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 24 Mar 2025 14:06:34 +0100 Subject: [PATCH 11/20] fix nil check for foreachtask --- bundle/libraries/switch_to_patched_wheels.go | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 7fc5e239fc..9fe101d014 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -39,17 +39,21 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag // resources.jobs.*.task[*].for_each_task.task.libraries - for libInd, lib := range task.ForEachTask.Task.Libraries { - repl := replacements[lib.Whl] - if repl != "" { - log.Debugf(ctx, "Updating resources.jobs.%s.task[%d].for_each_task.task.libraries[%d].whl from %s to %s", jobName, taskInd, libInd, lib.Whl, repl) - job.Tasks[taskInd].ForEachTask.Task.Libraries[libInd].Whl = repl + foreachptr := task.ForEachTask + + if foreachptr != nil { + for libInd, lib := range foreachptr.Task.Libraries { + repl := replacements[lib.Whl] + if repl != "" { + log.Debugf(ctx, "Updating resources.jobs.%s.task[%d].for_each_task.task.libraries[%d].whl from %s to %s", jobName, taskInd, libInd, lib.Whl, repl) + foreachptr.Task.Libraries[libInd].Whl = repl + } } } - } - // resources.jobs.*.environments.*.spec.dependencies - // TODO + // resources.jobs.*.environments.*.spec.dependencies + + } } return nil From 72d8f51f311f0e56fcba69c96e9c2a7907b46c4a Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 24 Mar 2025 14:35:02 +0100 Subject: [PATCH 12/20] patch environments.*.spec.dependencies. --- .../artifacts/whl_dynamic/databricks.yml | 11 +++++++++ .../bundle/artifacts/whl_dynamic/output.txt | 16 +++++++++++++ bundle/libraries/switch_to_patched_wheels.go | 23 ++++++++++++++----- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_dynamic/databricks.yml b/acceptance/bundle/artifacts/whl_dynamic/databricks.yml index 6be8a89830..f3cbfa4372 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/databricks.yml +++ b/acceptance/bundle/artifacts/whl_dynamic/databricks.yml @@ -37,3 +37,14 @@ resources: entry_point: "run" libraries: - whl: ./my_test_code/dist/*.whl + - task_key: ServerlessTestTask + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + environment_key: "test_env" + environments: + - environment_key: "test_env" + spec: + client: "1" + dependencies: + - ./my_test_code/dist/*.whl diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index cecb4570ca..3a12c930fd 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -48,6 +48,14 @@ my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD === Expecting 2 patched wheels in libraries section in /jobs/create >>> jq -s .[] | select(.path=="/api/2.1/jobs/create") | .body.tasks out.requests.txt [ + { + "environment_key": "test_env", + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "ServerlessTestTask" + }, { "existing_cluster_id": "0717-132531-5opeqon1", "for_each_task": { @@ -113,6 +121,14 @@ my_test_code-0.0.1+[TIMESTAMP_NS].dist-info/RECORD === Expecting 2 patched wheels in libraries section in /jobs/reset >>> jq -s .[] | select(.path=="/api/2.1/jobs/reset") | .body.new_settings.tasks out.requests.txt [ + { + "environment_key": "test_env", + "python_wheel_task": { + "entry_point": "run", + "package_name": "my_test_code" + }, + "task_key": "ServerlessTestTask" + }, { "existing_cluster_id": "0717-132531-5opeqon1", "for_each_task": { diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 9fe101d014..ae009fd5d1 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -26,9 +26,9 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag continue } - // resources.jobs.*.task[*].libraries[*] - for taskInd, task := range job.Tasks { + + // Update resources.jobs.*.task[*].libraries[*].whl for libInd, lib := range task.Libraries { repl := replacements[lib.Whl] if repl != "" { @@ -37,10 +37,9 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } - // resources.jobs.*.task[*].for_each_task.task.libraries + // Update resources.jobs.*.task[*].for_each_task.task.libraries[*].whl foreachptr := task.ForEachTask - if foreachptr != nil { for libInd, lib := range foreachptr.Task.Libraries { repl := replacements[lib.Whl] @@ -50,9 +49,21 @@ func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag } } } + } - // resources.jobs.*.environments.*.spec.dependencies - + // Update resources.jobs.*.environments.*.spec.dependencies[*] + for envInd, env := range job.Environments { + specptr := env.Spec + if specptr == nil { + continue + } + for depInd, dep := range specptr.Dependencies { + repl := replacements[dep] + if repl != "" { + log.Debugf(ctx, "Updating resources.jobs.%s.environments[%d].spec.dependencies[%d] from %s to %s", jobName, envInd, depInd, dep, repl) + specptr.Dependencies[depInd] = repl + } + } } } From 9bd1c9a1afae012447143396efb8cf99ed81ad80 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 24 Mar 2025 15:00:21 +0100 Subject: [PATCH 13/20] add serverless_dynamic_version --- .../databricks.yml.tmpl | 28 +++++++++++ .../serverless_dynamic_version/output.txt | 46 +++++++++++++++++++ .../serverless_dynamic_version/script | 10 ++++ .../serverless_dynamic_version/test.toml | 6 +++ 4 files changed, 90 insertions(+) create mode 100644 acceptance/bundle/integration_whl/serverless_dynamic_version/databricks.yml.tmpl create mode 100644 acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt create mode 100644 acceptance/bundle/integration_whl/serverless_dynamic_version/script create mode 100644 acceptance/bundle/integration_whl/serverless_dynamic_version/test.toml diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/databricks.yml.tmpl b/acceptance/bundle/integration_whl/serverless_dynamic_version/databricks.yml.tmpl new file mode 100644 index 0000000000..b6932428f0 --- /dev/null +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/databricks.yml.tmpl @@ -0,0 +1,28 @@ +bundle: + name: environment_key + +workspace: + root_path: "~/.bundle/$UNIQUE_NAME" + +artifacts: + my_test_code: + type: whl + build: python3 setup.py bdist_wheel + dynamic_version: true + +resources: + jobs: + some_other_job: + name: "My Wheel Job" + tasks: + - task_key: TestTask + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + environment_key: "test_env" + environments: + - environment_key: "test_env" + spec: + client: "1" + dependencies: + - ./dist/*.whl diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt new file mode 100644 index 0000000000..2ab6ce2dbe --- /dev/null +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt @@ -0,0 +1,46 @@ + +>>> cp -r [TESTROOT]/bundle/integration_whl/serverless_dynamic_version/../base/setup.py [TESTROOT]/bundle/integration_whl/serverless_dynamic_version/../base/my_test_code . + +>>> [CLI] bundle deploy +Building my_test_code... +Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[NUMID]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "My Wheel Job" RUNNING +[TIMESTAMP] "My Wheel Job" TERMINATED SUCCESS +Hello from my func +Got arguments: +['my_test_code'] + +=== Make a change to code without version change and run the job again +>>> [CLI] bundle deploy +Building my_test_code... +Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[NUMID]-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "My Wheel Job" RUNNING +[TIMESTAMP] "My Wheel Job" TERMINATED SUCCESS +UPDATED MY FUNC +Got arguments: +['my_test_code'] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete job some_other_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/script b/acceptance/bundle/integration_whl/serverless_dynamic_version/script new file mode 100644 index 0000000000..529be3ee7e --- /dev/null +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/script @@ -0,0 +1,10 @@ +envsubst < databricks.yml.tmpl > databricks.yml +trace cp -r $TESTDIR/../base/{setup.py,my_test_code} . +trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT +trace $CLI bundle deploy +trace $CLI bundle run some_other_job + +title "Make a change to code without version change and run the job again" +update_file.py my_test_code/__main__.py 'Hello from my func' 'UPDATED MY FUNC' +trace $CLI bundle deploy +trace $CLI bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/test.toml b/acceptance/bundle/integration_whl/serverless_dynamic_version/test.toml new file mode 100644 index 0000000000..01576446a6 --- /dev/null +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/test.toml @@ -0,0 +1,6 @@ +# serverless is only enabled if UC is enabled +RequiresUnityCatalog = true + +[[Repls]] +Old = '\\' +New = '/' From 6b5f5ed07e671883049bf2a9f15e0bc08cd7d627 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 24 Mar 2025 15:16:43 +0100 Subject: [PATCH 14/20] fix comments; use TIMESTAMP_NS replacement --- acceptance/bundle/artifacts/whl_dynamic/output.txt | 4 ++-- acceptance/bundle/artifacts/whl_dynamic/script | 4 ++-- acceptance/bundle/artifacts/whl_dynamic/test.toml | 4 ---- .../interactive_cluster_dynamic_version/output.txt | 4 ++-- .../integration_whl/serverless_dynamic_version/output.txt | 4 ++-- acceptance/bundle/test.toml | 4 ++++ 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_dynamic/output.txt b/acceptance/bundle/artifacts/whl_dynamic/output.txt index 3a12c930fd..049da0d81d 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/output.txt +++ b/acceptance/bundle/artifacts/whl_dynamic/output.txt @@ -35,7 +35,7 @@ Deployment complete! my_test_code/dist/my_test_code-0.0.1-py3-none-any.whl prebuilt/other_test_code-0.0.1-py3-none-any.whl -=== Verify contents, there should be no new_module.py +=== Verify contents of the zip file >>> find.py --expect 1 .databricks/.*my_test_code.*whl src/__init__.py src/__main__.py @@ -107,7 +107,7 @@ Deploying resources... Updating deployment state... Deployment complete! -=== Verify contents, there should have new_module.py +=== Verify contents, it should now have new_module.py >>> find.py --expect 1 .databricks/.*my_test_code.*whl src/__init__.py src/__main__.py diff --git a/acceptance/bundle/artifacts/whl_dynamic/script b/acceptance/bundle/artifacts/whl_dynamic/script index 3bbfc07bb2..51e995a069 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/script +++ b/acceptance/bundle/artifacts/whl_dynamic/script @@ -11,7 +11,7 @@ trace $CLI bundle deploy title "There are 2 original wheels and 2 patched ones" trace find.py --expect 4 whl -title "Verify contents, there should be no new_module.py" +title "Verify contents of the zip file" trace find.py --expect 1 '.databricks/.*my_test_code.*whl' | xargs ziplist.py title "Expecting 2 patched wheels in libraries section in /jobs/create" @@ -26,7 +26,7 @@ title "Updating the local wheel and deploying again" touch my_test_code/src/new_module.py trace $CLI bundle deploy -title "Verify contents, there should have new_module.py" +title "Verify contents, it should now have new_module.py" trace find.py --expect 1 '.databricks/.*my_test_code.*whl' | xargs ziplist.py title "Expecting 2 patched wheels in libraries section in /jobs/reset" diff --git a/acceptance/bundle/artifacts/whl_dynamic/test.toml b/acceptance/bundle/artifacts/whl_dynamic/test.toml index 86e9e54cce..7a953a580f 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/test.toml +++ b/acceptance/bundle/artifacts/whl_dynamic/test.toml @@ -2,10 +2,6 @@ Old = '\\\\' New = '/' -[[Repls]] -Old = '1\d{18}' -New = '[TIMESTAMP_NS]' - [[Server]] Pattern = "POST /api/2.1/jobs/reset" Response.Body = '{}' diff --git a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt index c3b514ebfc..b8d4e22ed0 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/interactive_cluster_dynamic_version/output.txt @@ -1,7 +1,7 @@ >>> [CLI] bundle deploy Building python_artifact... -Uploading .databricks/bundle/default/patched_wheels/python_artifact_my_test_code/my_test_code-0.0.1+[NUMID]-py3-none-any.whl... +Uploading .databricks/bundle/default/patched_wheels/python_artifact_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Deploying resources... Updating deployment state... @@ -19,7 +19,7 @@ Got arguments: === Make a change to code without version change and run the job again >>> [CLI] bundle deploy Building python_artifact... -Uploading .databricks/bundle/default/patched_wheels/python_artifact_my_test_code/my_test_code-0.0.1+[NUMID]-py3-none-any.whl... +Uploading .databricks/bundle/default/patched_wheels/python_artifact_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt index 2ab6ce2dbe..617bc70951 100644 --- a/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt +++ b/acceptance/bundle/integration_whl/serverless_dynamic_version/output.txt @@ -3,7 +3,7 @@ >>> [CLI] bundle deploy Building my_test_code... -Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[NUMID]-py3-none-any.whl... +Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Deploying resources... Updating deployment state... @@ -21,7 +21,7 @@ Got arguments: === Make a change to code without version change and run the job again >>> [CLI] bundle deploy Building my_test_code... -Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[NUMID]-py3-none-any.whl... +Uploading .databricks/bundle/default/patched_wheels/my_test_code_my_test_code/my_test_code-0.0.1+[TIMESTAMP_NS]-py3-none-any.whl... Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Deploying resources... Updating deployment state... diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index e919f997be..a33e028e16 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -4,3 +4,7 @@ Cloud = true [EnvMatrix] # The lowest Python version we support. Alternative to "uv run --python 3.10" UV_PYTHON = ["3.10"] + +[[Repls]] +Old = '1\d{18}' +New = '[TIMESTAMP_NS]' From 424212e312db64331f070a440f990e0e03cf8ebc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 24 Mar 2025 15:28:06 +0100 Subject: [PATCH 15/20] restore CloudSlow setting --- acceptance/bundle/integration_whl/test.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/integration_whl/test.toml b/acceptance/bundle/integration_whl/test.toml index 82e4d48d61..50228c95a1 100644 --- a/acceptance/bundle/integration_whl/test.toml +++ b/acceptance/bundle/integration_whl/test.toml @@ -1,5 +1,5 @@ Local = false -#CloudSlow = true +CloudSlow = true Ignore = [ ".databricks", "build", From 18fd31bed789294abc539cbfef17809b8f42d2dc Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 25 Mar 2025 09:27:08 +0100 Subject: [PATCH 16/20] fix debug message cache vs built --- bundle/artifacts/build.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 0ee7ee05a3..fecd1b4038 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -144,9 +144,9 @@ func mkPatchedWheel(ctx context.Context, cacheDir, artifactName, wheel string) ( } if isBuilt { - log.Infof(ctx, "Patched wheel (cache) %s -> %s", wheel, patchedWheel) - } else { log.Infof(ctx, "Patched wheel (built) %s -> %s", wheel, patchedWheel) + } else { + log.Infof(ctx, "Patched wheel (cache) %s -> %s", wheel, patchedWheel) } return patchedWheel, nil From d67d059f9f9275f99415865f78108211ef1e2ff0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 25 Mar 2025 09:46:40 +0100 Subject: [PATCH 17/20] add NEXT_CHANGELOG.md entry --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index fd164be98c..76d456d9fa 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -12,5 +12,6 @@ * Added support for dashboards in deployment bind/unbind commands ([#2516](https://github.com/databricks/cli/pull/2516)) * Added support for registered models in deployment bind/unbind commands ([#2556](https://github.com/databricks/cli/pull/2556)) * Added a mismatch check when host is defined in config and as an env variable ([#2549](https://github.com/databricks/cli/pull/2549)) +* New attribute on artifacts entries: `dynamic_version`. When set to true, it patches the wheel with dynamic version suffix so it is always used by Databricks environments, even if original wheel version is the same. Intended for development loop on interactive clusters. ([#2520](https://github.com/databricks/cli/pull/2520)) ### API Changes From 1dfde8a8eedce2b80c271307e35809168114dda5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 25 Mar 2025 15:02:46 +0100 Subject: [PATCH 18/20] mk -> make --- bundle/artifacts/build.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index fecd1b4038..fd0de6148f 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -81,7 +81,7 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if a.Type == "whl" && a.DynamicVersion && cacheDir != "" { for ind, artifactFile := range a.Files { - patchedWheel, extraDiags := mkPatchedWheel(ctx, cacheDir, artifactName, artifactFile.Source) + patchedWheel, extraDiags := makePatchedWheel(ctx, cacheDir, artifactName, artifactFile.Source) log.Debugf(ctx, "Patching ind=%d artifactName=%s Source=%s patchedWheel=%s", ind, artifactName, artifactFile.Source, patchedWheel) diags = append(diags, extraDiags...) if patchedWheel != "" { @@ -121,7 +121,7 @@ func doBuild(ctx context.Context, artifactName string, a *config.Artifact) error return nil } -func mkPatchedWheel(ctx context.Context, cacheDir, artifactName, wheel string) (string, diag.Diagnostics) { +func makePatchedWheel(ctx context.Context, cacheDir, artifactName, wheel string) (string, diag.Diagnostics) { msg := "Failed to patch wheel with dynamic version" info, err := patchwheel.ParseWheelFilename(wheel) if err != nil { From 9f5f06b28dfec08b186e536f46c3cab11a982154 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Mar 2025 10:56:46 +0100 Subject: [PATCH 19/20] 2 space indent --- .../artifacts/whl_dynamic/databricks.yml | 78 +++++++++---------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/acceptance/bundle/artifacts/whl_dynamic/databricks.yml b/acceptance/bundle/artifacts/whl_dynamic/databricks.yml index f3cbfa4372..c1cb6d137d 100644 --- a/acceptance/bundle/artifacts/whl_dynamic/databricks.yml +++ b/acceptance/bundle/artifacts/whl_dynamic/databricks.yml @@ -2,49 +2,49 @@ bundle: name: python-wheel artifacts: - my_test_code: - type: whl - path: "./my_test_code" - # using 'python' there because 'python3' does not exist in virtualenv on windows - build: python setup.py bdist_wheel - dynamic_version: true - my_prebuilt_whl: - type: whl - files: - - source: prebuilt/other_test_code-0.0.1-py3-none-any.whl - dynamic_version: true + my_test_code: + type: whl + path: "./my_test_code" + # using 'python' there because 'python3' does not exist in virtualenv on windows + build: python setup.py bdist_wheel + dynamic_version: true + my_prebuilt_whl: + type: whl + files: + - source: prebuilt/other_test_code-0.0.1-py3-none-any.whl + dynamic_version: true resources: jobs: test_job: name: "[${bundle.target}] My Wheel Job" tasks: - - task_key: TestTask - existing_cluster_id: "0717-132531-5opeqon1" - python_wheel_task: - package_name: "my_test_code" - entry_point: "run" - libraries: - - whl: ./my_test_code/dist/*.whl - - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl - for_each_task: - inputs: "[1]" - task: - task_key: SubTask - existing_cluster_id: "0717-132531-5opeqon1" - python_wheel_task: - package_name: "my_test_code" - entry_point: "run" - libraries: - - whl: ./my_test_code/dist/*.whl - - task_key: ServerlessTestTask - python_wheel_task: - package_name: "my_test_code" - entry_point: "run" - environment_key: "test_env" + - task_key: TestTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./my_test_code/dist/*.whl + - whl: prebuilt/other_test_code-0.0.1-py3-none-any.whl + for_each_task: + inputs: "[1]" + task: + task_key: SubTask + existing_cluster_id: "0717-132531-5opeqon1" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./my_test_code/dist/*.whl + - task_key: ServerlessTestTask + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + environment_key: "test_env" environments: - - environment_key: "test_env" - spec: - client: "1" - dependencies: - - ./my_test_code/dist/*.whl + - environment_key: "test_env" + spec: + client: "1" + dependencies: + - ./my_test_code/dist/*.whl From 7b2055791b4cfa4e02a80bdf0c8a4eebb52f1bd9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Mar 2025 11:01:53 +0100 Subject: [PATCH 20/20] local sv --- bundle/libraries/upload.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index c4edff4042..55ad847e81 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -111,8 +111,7 @@ func collectLocalLibraries(b *bundle.Bundle) (map[string][]configLocation, error return v, fmt.Errorf("expected string, got %s", v.Kind()) } - sv, ok = file.GetByString("patched") - if ok { + if sv, ok = file.GetByString("patched"); ok { patched, ok := sv.AsString() if ok && patched != "" { source = patched