From 02a4dc2e8cc9901e13552d90882f71a83e92eed0 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 28 Aug 2023 13:21:07 +0200 Subject: [PATCH 1/3] Allow referencing local Python wheels without artifacts section defined --- bundle/artifacts/autodetect.go | 1 + bundle/artifacts/infer.go | 6 +- bundle/artifacts/whl/from_libraries.go | 54 ++++++++++++++++++ .../.gitignore | 3 + .../bundle.yml | 22 +++++++ .../my_test_code-0.0.1-py3-none-any.whl | Bin 0 -> 1909 bytes bundle/tests/bundle/wheel_test.go | 26 +++++++++ 7 files changed, 111 insertions(+), 1 deletion(-) create mode 100644 bundle/artifacts/whl/from_libraries.go create mode 100644 bundle/tests/bundle/python_wheel_no_artifact_no_setup/.gitignore create mode 100644 bundle/tests/bundle/python_wheel_no_artifact_no_setup/bundle.yml create mode 100644 bundle/tests/bundle/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl diff --git a/bundle/artifacts/autodetect.go b/bundle/artifacts/autodetect.go index fa8126f973..6e80ef0b63 100644 --- a/bundle/artifacts/autodetect.go +++ b/bundle/artifacts/autodetect.go @@ -28,5 +28,6 @@ func (m *autodetect) Apply(ctx context.Context, b *bundle.Bundle) error { return bundle.Apply(ctx, b, bundle.Seq( whl.DetectPackage(), + whl.DefineArtifactsFromLibraries(), )) } diff --git a/bundle/artifacts/infer.go b/bundle/artifacts/infer.go index 233fbda86b..ade5def516 100644 --- a/bundle/artifacts/infer.go +++ b/bundle/artifacts/infer.go @@ -47,7 +47,11 @@ func (m *infer) Apply(ctx context.Context, b *bundle.Bundle) error { return fmt.Errorf("artifact doesn't exist: %s", m.name) } - if artifact.BuildCommand != "" { + // only try to infer command if it's not already defined + // and there is no explicitly files defined which means + // that the package is built outside of bundle cycles + // manually by customer + if artifact.BuildCommand != "" || len(artifact.Files) > 0 { return nil } diff --git a/bundle/artifacts/whl/from_libraries.go b/bundle/artifacts/whl/from_libraries.go new file mode 100644 index 0000000000..ff8018cda5 --- /dev/null +++ b/bundle/artifacts/whl/from_libraries.go @@ -0,0 +1,54 @@ +package whl + +import ( + "context" + "path/filepath" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/libraries" + "github.com/databricks/cli/libs/log" +) + +type fromLibraries struct{} + +func DefineArtifactsFromLibraries() bundle.Mutator { + return &fromLibraries{} +} + +func (m *fromLibraries) Name() string { + return "artifacts.whl.DefineArtifactsFromLibraries" +} + +func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) error { + if b.Config.Artifacts != nil { + log.Debugf(ctx, "Skipping defining artifacts from libraries because artifacts section is explicitly defined") + return nil + } + + tasks := libraries.FindAllWheelTasks(b) + for _, task := range tasks { + for _, lib := range task.Libraries { + matches, err := filepath.Glob(filepath.Join(b.Config.Path, lib.Whl)) + // File referenced from libraries section does not exists, skipping + if err != nil { + continue + } + + for _, match := range matches { + name := filepath.Base(match) + if b.Config.Artifacts == nil { + b.Config.Artifacts = make(map[string]*config.Artifact) + } + b.Config.Artifacts[name] = &config.Artifact{ + Files: []config.ArtifactFile{ + {Source: match}, + }, + Type: config.ArtifactPythonWheel, + } + } + } + } + + return nil +} diff --git a/bundle/tests/bundle/python_wheel_no_artifact_no_setup/.gitignore b/bundle/tests/bundle/python_wheel_no_artifact_no_setup/.gitignore new file mode 100644 index 0000000000..f03e23bc26 --- /dev/null +++ b/bundle/tests/bundle/python_wheel_no_artifact_no_setup/.gitignore @@ -0,0 +1,3 @@ +build/ +*.egg-info +.databricks diff --git a/bundle/tests/bundle/python_wheel_no_artifact_no_setup/bundle.yml b/bundle/tests/bundle/python_wheel_no_artifact_no_setup/bundle.yml new file mode 100644 index 0000000000..1bac4ebadf --- /dev/null +++ b/bundle/tests/bundle/python_wheel_no_artifact_no_setup/bundle.yml @@ -0,0 +1,22 @@ +bundle: + name: python-wheel-local + +resources: + jobs: + test_job: + name: "[${bundle.environment}] My Wheel Job" + tasks: + - task_key: TestTask + existing_cluster_id: "0717-aaaaa-bbbbbb" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./package/*.whl + - task_key: TestTask2 + existing_cluster_id: "0717-aaaaa-bbbbbb" + python_wheel_task: + package_name: "my_test_code" + entry_point: "run" + libraries: + - whl: ./non-existing/*.whl diff --git a/bundle/tests/bundle/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl b/bundle/tests/bundle/python_wheel_no_artifact_no_setup/package/my_test_code-0.0.1-py3-none-any.whl new file mode 100644 index 0000000000000000000000000000000000000000..14702281d87f6180c03799b19ec1ab6a10f95509 GIT binary patch literal 1909 zcmWIWW@Zs#U|`^2*jp?fe!}|sR6QWi0Ei`kIJYvsB(=CCJ~=-nRX;vHGcU6wK3=b& zvb$g3TUYDcne*NK;Tu8&j4l{oFb;Xt)$gyPd-8;S6UQ!scPxs+J%e^0VsBe z?D2WA7-;q<>~`cPX6AwI2<_X=cSu2`t$f}ie!eZ;{1=vN?&DMITQaeELIL+u(>;5) z$X+r2xBYYBxtn(dG^WH%T*S*7xY*=nSa&BQzw;A`e9N@juAK?%jFWvPw$0CZ@vCv+ zUC#34^olIq*6pQyn)$`I!%h`$?Gjn`C#t5}?EPHZ3!U1Q;tPwvd=RbT^uPH3*Yt>e zM<1Tg)_I<`wpMwQR8ZT=$x|fUIW*1fG>RX#;XTV4(hyusmi%+F9@wUD;JW4nu!?^$dy zymf_ll8nmFA_hkt!=|h^YuC#bJFA^;`|kbeX>&dRJ69 z$LIY8Z@pKnw6_ymom8!NL@#1rQkVUv_Vm}n6HeJ~oXB#=tVhD5W4+QnRF8HC7_Eu{ zdJu$_2zfNz!`0P?`@Dzm*^9i8G_yI#;EM5sBLB1Ax?Vab^*051c%9VI>(pmo8F<0q zg0b0k;|ou?PMz1k_*Lf`uZFJI*^@q-f;2R>e4lc8`fLxF(do6c`hOW@qmTJp+EAUA$0x@kV?}enEUrYFTOy$imY| zRy;-bO4j6ipW_%A7#x@w7?cTlCCJs;Kggx^!s((W1BTXzwewju6GS3P6a`)N*1GB} zo6R$|BhW*GGw|1!x5sAg%8Y;T`}~}=1Cuy9zwc4$mk4%KJyvC%@hBo<^2^E7%>*5v zoVe}L_+0g#4p+3F{uw5}TWgyqthzT(JRh{-dqHsOvazjxvS@7*ijT2DGQG1vS2$uXOsx>kEV}tyyM^QLgJ<_EQLtFYDR9(i4JT8T(!p zvsMQSZum7N^~wA&w+~NeY*}qm{HI*u>qBp=>s#9M1H2iTM3`}xmcTdygC&h13ON-Z zm!0UQpqGabGZ+|_G^Svh0xunrO~F?9AWX3ZCLv_E;4F&JjYQAp2qV>iDH5-d;7pHh z9(vY9n0JVfdFUAx-Dvb2h%nlWnSfJK@*}#r=vfeE?iR*gLgqp;WPmp-8%Ps75E=u0 Jsly850RXT?oKXM( literal 0 HcmV?d00001 diff --git a/bundle/tests/bundle/wheel_test.go b/bundle/tests/bundle/wheel_test.go index ee7457735c..f7f0e75e5f 100644 --- a/bundle/tests/bundle/wheel_test.go +++ b/bundle/tests/bundle/wheel_test.go @@ -60,3 +60,29 @@ func TestBundlePythonWheelWithDBFSLib(t *testing.T) { err = match.Apply(ctx, b) require.NoError(t, err) } + +func TestBundlePythonWheelBuildNoBuildJustUpload(t *testing.T) { + ctx := context.Background() + b, err := bundle.Load(ctx, "./python_wheel_no_artifact_no_setup") + require.NoError(t, err) + + m := phases.Build() + err = m.Apply(ctx, b) + require.NoError(t, err) + + match := libraries.MatchWithArtifacts() + err = match.Apply(ctx, b) + require.ErrorContains(t, err, "./non-existing/*.whl") + + require.NotZero(t, len(b.Config.Artifacts)) + + artifact := b.Config.Artifacts["my_test_code-0.0.1-py3-none-any.whl"] + require.NotNil(t, artifact) + require.Empty(t, artifact.BuildCommand) + require.Contains(t, artifact.Files[0].Source, filepath.Join( + b.Config.Path, + "package", + "my_test_code-0.0.1-py3-none-any.whl", + )) + require.True(t, artifact.Files[0].NeedsUpload()) +} From d25c6ad7de775185f7f1b0f4f65fbee9f5a3ec91 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 28 Aug 2023 17:39:04 +0200 Subject: [PATCH 2/3] Minor fixes --- bundle/artifacts/whl/from_libraries.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bundle/artifacts/whl/from_libraries.go b/bundle/artifacts/whl/from_libraries.go index ff8018cda5..aae9c104ba 100644 --- a/bundle/artifacts/whl/from_libraries.go +++ b/bundle/artifacts/whl/from_libraries.go @@ -21,7 +21,7 @@ func (m *fromLibraries) Name() string { } func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) error { - if b.Config.Artifacts != nil { + if len(b.Config.Artifacts) == 0 { log.Debugf(ctx, "Skipping defining artifacts from libraries because artifacts section is explicitly defined") return nil } @@ -40,6 +40,8 @@ func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) error { if b.Config.Artifacts == nil { b.Config.Artifacts = make(map[string]*config.Artifact) } + + log.Debugf(ctx, "Adding an artifact block for %s", match) b.Config.Artifacts[name] = &config.Artifact{ Files: []config.ArtifactFile{ {Source: match}, From 4b9e043c217d8aeaf4dd2452826f061291dcec06 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Mon, 28 Aug 2023 18:24:05 +0200 Subject: [PATCH 3/3] fixed typo --- bundle/artifacts/whl/from_libraries.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bundle/artifacts/whl/from_libraries.go b/bundle/artifacts/whl/from_libraries.go index aae9c104ba..855e5b9438 100644 --- a/bundle/artifacts/whl/from_libraries.go +++ b/bundle/artifacts/whl/from_libraries.go @@ -21,7 +21,7 @@ func (m *fromLibraries) Name() string { } func (*fromLibraries) Apply(ctx context.Context, b *bundle.Bundle) error { - if len(b.Config.Artifacts) == 0 { + if len(b.Config.Artifacts) != 0 { log.Debugf(ctx, "Skipping defining artifacts from libraries because artifacts section is explicitly defined") return nil }