From 0d04ff964d406f09fe9fd7f6bfa4f8555654ed25 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Mon, 10 Mar 2025 20:23:42 +0100 Subject: [PATCH 01/10] integration test wip add repl for TEST_INSTANCE_POOL_ID add uuid1 env/repl with dashes replaced with underscore - WIP - use randomString rm out.requests.txt update test fix the test mark my test as CloudLong update test.toml disable local runs split test in two for parallel run rename --- acceptance/acceptance_test.go | 7 ++++ .../databricks_template_schema.json | 0 .../python_wheel_task/input.json.tmpl | 8 +++++ .../python_wheel_task/output.txt | 35 +++++++++++++++++++ .../integration_whl/python_wheel_task/script | 8 +++++ .../template/databricks.yml.tmpl | 0 .../python_wheel_task/template/setup.py.tmpl | 0 .../template/{{.project_name}}/__init__.py | 0 .../template/{{.project_name}}/__main__.py | 0 .../output.txt | 35 +++++++++++++++++++ .../python_wheel_task_custom_params/script | 8 +++++ acceptance/bundle/integration_whl/test.toml | 20 +++++++++++ 12 files changed, 121 insertions(+) rename {integration/bundle/bundles => acceptance/bundle/integration_whl}/python_wheel_task/databricks_template_schema.json (100%) create mode 100644 acceptance/bundle/integration_whl/python_wheel_task/input.json.tmpl create mode 100644 acceptance/bundle/integration_whl/python_wheel_task/output.txt create mode 100644 acceptance/bundle/integration_whl/python_wheel_task/script rename {integration/bundle/bundles => acceptance/bundle/integration_whl}/python_wheel_task/template/databricks.yml.tmpl (100%) rename {integration/bundle/bundles => acceptance/bundle/integration_whl}/python_wheel_task/template/setup.py.tmpl (100%) rename {integration/bundle/bundles => acceptance/bundle/integration_whl}/python_wheel_task/template/{{.project_name}}/__init__.py (100%) rename {integration/bundle/bundles => acceptance/bundle/integration_whl}/python_wheel_task/template/{{.project_name}}/__main__.py (100%) create mode 100644 acceptance/bundle/integration_whl/python_wheel_task_custom_params/output.txt create mode 100644 acceptance/bundle/integration_whl/python_wheel_task_custom_params/script create mode 100644 acceptance/bundle/integration_whl/test.toml diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 6c97aca4c0..ffff75b779 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -159,6 +159,8 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { // do it last so that full paths match first: repls.SetPath(buildDir, "[BUILD_DIR]") + repls.Set(os.Getenv("TEST_INSTANCE_POOL_ID"), "[TEST_INSTANCE_POOL_ID]") + testdiff.PrepareReplacementsDevVersion(t, &repls) testdiff.PrepareReplacementSdkVersion(t, &repls) testdiff.PrepareReplacementsGoVersion(t, &repls) @@ -256,6 +258,10 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont } } + // XXX rename away from uuid1 since it's not uuid due to replacements + uuid1 := strings.ReplaceAll(uuid.NewString(), "-", "_") + repls.Set(uuid1, "[UUID1]") + var tmpDir string var err error if KeepTmp { @@ -281,6 +287,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont args := []string{"bash", "-euo", "pipefail", EntryPointScript} cmd := exec.Command(args[0], args[1:]...) cmd.Env = os.Environ() + cmd.Env = append(cmd.Env, "UUID1="+uuid1) var workspaceClient *databricks.WorkspaceClient var user iam.User diff --git a/integration/bundle/bundles/python_wheel_task/databricks_template_schema.json b/acceptance/bundle/integration_whl/python_wheel_task/databricks_template_schema.json similarity index 100% rename from integration/bundle/bundles/python_wheel_task/databricks_template_schema.json rename to acceptance/bundle/integration_whl/python_wheel_task/databricks_template_schema.json diff --git a/acceptance/bundle/integration_whl/python_wheel_task/input.json.tmpl b/acceptance/bundle/integration_whl/python_wheel_task/input.json.tmpl new file mode 100644 index 0000000000..292bca7d40 --- /dev/null +++ b/acceptance/bundle/integration_whl/python_wheel_task/input.json.tmpl @@ -0,0 +1,8 @@ +{ + "project_name": "my_test_code", + "spark_version": "$SPARK_VERSION", + "node_type_id": "$NODE_TYPE_ID", + "unique_id": "$UUID1", + "python_wheel_wrapper": $PYTHON_WHEEL_WRAPPER, + "instance_pool_id": "$TEST_INSTANCE_POOL_ID" +} diff --git a/acceptance/bundle/integration_whl/python_wheel_task/output.txt b/acceptance/bundle/integration_whl/python_wheel_task/output.txt new file mode 100644 index 0000000000..71eaea6a7c --- /dev/null +++ b/acceptance/bundle/integration_whl/python_wheel_task/output.txt @@ -0,0 +1,35 @@ +{ + "project_name": "my_test_code", + "spark_version": "13.3.x-snapshot-scala2.12", + "node_type_id": "", + "unique_id": "[UUID1]", + "python_wheel_wrapper": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]" +} +✨ Successfully initialized template + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UUID1]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UUID1]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UUID1]" TERMINATED SUCCESS +Hello from my func +Got arguments: +['my_test_code', 'one', 'two'] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete job some_other_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UUID1] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/integration_whl/python_wheel_task/script b/acceptance/bundle/integration_whl/python_wheel_task/script new file mode 100644 index 0000000000..a0d6af9890 --- /dev/null +++ b/acceptance/bundle/integration_whl/python_wheel_task/script @@ -0,0 +1,8 @@ +export SPARK_VERSION=13.3.x-snapshot-scala2.12 +export PYTHON_WHEEL_WRAPPER=false +envsubst < input.json.tmpl > input.json +cat input.json +$CLI bundle init . --config-file input.json +trace $CLI bundle deploy +trap "errcode trace $CLI bundle destroy --auto-approve" EXIT +trace $CLI bundle run some_other_job diff --git a/integration/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl b/acceptance/bundle/integration_whl/python_wheel_task/template/databricks.yml.tmpl similarity index 100% rename from integration/bundle/bundles/python_wheel_task/template/databricks.yml.tmpl rename to acceptance/bundle/integration_whl/python_wheel_task/template/databricks.yml.tmpl diff --git a/integration/bundle/bundles/python_wheel_task/template/setup.py.tmpl b/acceptance/bundle/integration_whl/python_wheel_task/template/setup.py.tmpl similarity index 100% rename from integration/bundle/bundles/python_wheel_task/template/setup.py.tmpl rename to acceptance/bundle/integration_whl/python_wheel_task/template/setup.py.tmpl diff --git a/integration/bundle/bundles/python_wheel_task/template/{{.project_name}}/__init__.py b/acceptance/bundle/integration_whl/python_wheel_task/template/{{.project_name}}/__init__.py similarity index 100% rename from integration/bundle/bundles/python_wheel_task/template/{{.project_name}}/__init__.py rename to acceptance/bundle/integration_whl/python_wheel_task/template/{{.project_name}}/__init__.py diff --git a/integration/bundle/bundles/python_wheel_task/template/{{.project_name}}/__main__.py b/acceptance/bundle/integration_whl/python_wheel_task/template/{{.project_name}}/__main__.py similarity index 100% rename from integration/bundle/bundles/python_wheel_task/template/{{.project_name}}/__main__.py rename to acceptance/bundle/integration_whl/python_wheel_task/template/{{.project_name}}/__main__.py diff --git a/acceptance/bundle/integration_whl/python_wheel_task_custom_params/output.txt b/acceptance/bundle/integration_whl/python_wheel_task_custom_params/output.txt new file mode 100644 index 0000000000..3dca29b3c1 --- /dev/null +++ b/acceptance/bundle/integration_whl/python_wheel_task_custom_params/output.txt @@ -0,0 +1,35 @@ +{ + "project_name": "my_test_code", + "spark_version": "13.3.x-snapshot-scala2.12", + "node_type_id": "", + "unique_id": "[UUID1]", + "python_wheel_wrapper": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]" +} +✨ Successfully initialized template + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UUID1]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job --python-params param1,param2 +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UUID1]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UUID1]" TERMINATED SUCCESS +Hello from my func +Got arguments: +['my_test_code', 'param1', 'param2'] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete job some_other_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UUID1] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/integration_whl/python_wheel_task_custom_params/script b/acceptance/bundle/integration_whl/python_wheel_task_custom_params/script new file mode 100644 index 0000000000..bfc7b8f305 --- /dev/null +++ b/acceptance/bundle/integration_whl/python_wheel_task_custom_params/script @@ -0,0 +1,8 @@ +export SPARK_VERSION=13.3.x-snapshot-scala2.12 +export PYTHON_WHEEL_WRAPPER=false +envsubst < $TESTDIR/../python_wheel_task/input.json.tmpl > input.json +cat input.json +$CLI bundle init $TESTDIR/../python_wheel_task --config-file input.json +trace $CLI bundle deploy +trap "errcode trace $CLI bundle destroy --auto-approve" EXIT +trace $CLI bundle run some_other_job --python-params param1,param2 diff --git a/acceptance/bundle/integration_whl/test.toml b/acceptance/bundle/integration_whl/test.toml new file mode 100644 index 0000000000..1fef7db8b5 --- /dev/null +++ b/acceptance/bundle/integration_whl/test.toml @@ -0,0 +1,20 @@ +Local = false +CloudSlow = true +Ignore = [ + ".databricks", + "build", + "dist", + "my_test_code", + "my_test_code.egg-info", + "setup.py", + "input.json", + "databricks.yml", +] + +[[Repls]] +Old = '2\d\d\d-\d\d-\d\d \d\d:\d\d:\d\d' +New = "[TIMESTAMP]" + +[[Repls]] +Old = '\d{5,}' +New = "[NUMID]" From 2b5daeafb186cc5f555710597a8189344ed819ac Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 11 Mar 2025 16:51:24 +0100 Subject: [PATCH 02/10] tmp comment out CloudSlow --- acceptance/bundle/integration_whl/test.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/integration_whl/test.toml b/acceptance/bundle/integration_whl/test.toml index 1fef7db8b5..2e53f7b535 100644 --- a/acceptance/bundle/integration_whl/test.toml +++ b/acceptance/bundle/integration_whl/test.toml @@ -1,5 +1,5 @@ Local = false -CloudSlow = true +#CloudSlow = true Ignore = [ ".databricks", "build", From 3005e9372d00cbb22d37a4526d6d1b0546f7fbd7 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 11 Mar 2025 16:56:06 +0100 Subject: [PATCH 03/10] update --- .../databricks_template_schema.json | 0 .../input.json.tmpl | 0 .../{python_wheel_task => base}/output.txt | 0 .../{python_wheel_task => base}/script | 0 .../template/databricks.yml.tmpl | 0 .../template/setup.py.tmpl | 0 .../template/{{.project_name}}/__init__.py | 0 .../template/{{.project_name}}/__main__.py | 0 .../output.txt | 0 .../script | 4 +-- .../bundle/integration_whl/wrapper/output.txt | 35 +++++++++++++++++++ .../bundle/integration_whl/wrapper/script | 8 +++++ .../wrapper_custom_params/output.txt | 35 +++++++++++++++++++ .../wrapper_custom_params/script | 8 +++++ 14 files changed, 88 insertions(+), 2 deletions(-) rename acceptance/bundle/integration_whl/{python_wheel_task => base}/databricks_template_schema.json (100%) rename acceptance/bundle/integration_whl/{python_wheel_task => base}/input.json.tmpl (100%) rename acceptance/bundle/integration_whl/{python_wheel_task => base}/output.txt (100%) rename acceptance/bundle/integration_whl/{python_wheel_task => base}/script (100%) rename acceptance/bundle/integration_whl/{python_wheel_task => base}/template/databricks.yml.tmpl (100%) rename acceptance/bundle/integration_whl/{python_wheel_task => base}/template/setup.py.tmpl (100%) rename acceptance/bundle/integration_whl/{python_wheel_task => base}/template/{{.project_name}}/__init__.py (100%) rename acceptance/bundle/integration_whl/{python_wheel_task => base}/template/{{.project_name}}/__main__.py (100%) rename acceptance/bundle/integration_whl/{python_wheel_task_custom_params => custom_params}/output.txt (100%) rename acceptance/bundle/integration_whl/{python_wheel_task_custom_params => custom_params}/script (63%) create mode 100644 acceptance/bundle/integration_whl/wrapper/output.txt create mode 100644 acceptance/bundle/integration_whl/wrapper/script create mode 100644 acceptance/bundle/integration_whl/wrapper_custom_params/output.txt create mode 100644 acceptance/bundle/integration_whl/wrapper_custom_params/script diff --git a/acceptance/bundle/integration_whl/python_wheel_task/databricks_template_schema.json b/acceptance/bundle/integration_whl/base/databricks_template_schema.json similarity index 100% rename from acceptance/bundle/integration_whl/python_wheel_task/databricks_template_schema.json rename to acceptance/bundle/integration_whl/base/databricks_template_schema.json diff --git a/acceptance/bundle/integration_whl/python_wheel_task/input.json.tmpl b/acceptance/bundle/integration_whl/base/input.json.tmpl similarity index 100% rename from acceptance/bundle/integration_whl/python_wheel_task/input.json.tmpl rename to acceptance/bundle/integration_whl/base/input.json.tmpl diff --git a/acceptance/bundle/integration_whl/python_wheel_task/output.txt b/acceptance/bundle/integration_whl/base/output.txt similarity index 100% rename from acceptance/bundle/integration_whl/python_wheel_task/output.txt rename to acceptance/bundle/integration_whl/base/output.txt diff --git a/acceptance/bundle/integration_whl/python_wheel_task/script b/acceptance/bundle/integration_whl/base/script similarity index 100% rename from acceptance/bundle/integration_whl/python_wheel_task/script rename to acceptance/bundle/integration_whl/base/script diff --git a/acceptance/bundle/integration_whl/python_wheel_task/template/databricks.yml.tmpl b/acceptance/bundle/integration_whl/base/template/databricks.yml.tmpl similarity index 100% rename from acceptance/bundle/integration_whl/python_wheel_task/template/databricks.yml.tmpl rename to acceptance/bundle/integration_whl/base/template/databricks.yml.tmpl diff --git a/acceptance/bundle/integration_whl/python_wheel_task/template/setup.py.tmpl b/acceptance/bundle/integration_whl/base/template/setup.py.tmpl similarity index 100% rename from acceptance/bundle/integration_whl/python_wheel_task/template/setup.py.tmpl rename to acceptance/bundle/integration_whl/base/template/setup.py.tmpl diff --git a/acceptance/bundle/integration_whl/python_wheel_task/template/{{.project_name}}/__init__.py b/acceptance/bundle/integration_whl/base/template/{{.project_name}}/__init__.py similarity index 100% rename from acceptance/bundle/integration_whl/python_wheel_task/template/{{.project_name}}/__init__.py rename to acceptance/bundle/integration_whl/base/template/{{.project_name}}/__init__.py diff --git a/acceptance/bundle/integration_whl/python_wheel_task/template/{{.project_name}}/__main__.py b/acceptance/bundle/integration_whl/base/template/{{.project_name}}/__main__.py similarity index 100% rename from acceptance/bundle/integration_whl/python_wheel_task/template/{{.project_name}}/__main__.py rename to acceptance/bundle/integration_whl/base/template/{{.project_name}}/__main__.py diff --git a/acceptance/bundle/integration_whl/python_wheel_task_custom_params/output.txt b/acceptance/bundle/integration_whl/custom_params/output.txt similarity index 100% rename from acceptance/bundle/integration_whl/python_wheel_task_custom_params/output.txt rename to acceptance/bundle/integration_whl/custom_params/output.txt diff --git a/acceptance/bundle/integration_whl/python_wheel_task_custom_params/script b/acceptance/bundle/integration_whl/custom_params/script similarity index 63% rename from acceptance/bundle/integration_whl/python_wheel_task_custom_params/script rename to acceptance/bundle/integration_whl/custom_params/script index bfc7b8f305..8148417e3a 100644 --- a/acceptance/bundle/integration_whl/python_wheel_task_custom_params/script +++ b/acceptance/bundle/integration_whl/custom_params/script @@ -1,8 +1,8 @@ export SPARK_VERSION=13.3.x-snapshot-scala2.12 export PYTHON_WHEEL_WRAPPER=false -envsubst < $TESTDIR/../python_wheel_task/input.json.tmpl > input.json +envsubst < $TESTDIR/../base/input.json.tmpl > input.json cat input.json -$CLI bundle init $TESTDIR/../python_wheel_task --config-file input.json +$CLI bundle init $TESTDIR/../base --config-file input.json trace $CLI bundle deploy trap "errcode trace $CLI bundle destroy --auto-approve" EXIT trace $CLI bundle run some_other_job --python-params param1,param2 diff --git a/acceptance/bundle/integration_whl/wrapper/output.txt b/acceptance/bundle/integration_whl/wrapper/output.txt new file mode 100644 index 0000000000..3dca29b3c1 --- /dev/null +++ b/acceptance/bundle/integration_whl/wrapper/output.txt @@ -0,0 +1,35 @@ +{ + "project_name": "my_test_code", + "spark_version": "13.3.x-snapshot-scala2.12", + "node_type_id": "", + "unique_id": "[UUID1]", + "python_wheel_wrapper": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]" +} +✨ Successfully initialized template + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UUID1]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job --python-params param1,param2 +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UUID1]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UUID1]" TERMINATED SUCCESS +Hello from my func +Got arguments: +['my_test_code', 'param1', 'param2'] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete job some_other_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UUID1] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/integration_whl/wrapper/script b/acceptance/bundle/integration_whl/wrapper/script new file mode 100644 index 0000000000..3c921b8880 --- /dev/null +++ b/acceptance/bundle/integration_whl/wrapper/script @@ -0,0 +1,8 @@ +export SPARK_VERSION=12.2.x-scala2.12 +export PYTHON_WHEEL_WRAPPER=true +envsubst < $TESTDIR/../base/input.json.tmpl > input.json +cat input.json +$CLI bundle init $TESTDIR/../base --config-file input.json +trace $CLI bundle deploy +trap "errcode trace $CLI bundle destroy --auto-approve" EXIT +trace $CLI bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt b/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt new file mode 100644 index 0000000000..3dca29b3c1 --- /dev/null +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt @@ -0,0 +1,35 @@ +{ + "project_name": "my_test_code", + "spark_version": "13.3.x-snapshot-scala2.12", + "node_type_id": "", + "unique_id": "[UUID1]", + "python_wheel_wrapper": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]" +} +✨ Successfully initialized template + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UUID1]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job --python-params param1,param2 +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UUID1]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UUID1]" TERMINATED SUCCESS +Hello from my func +Got arguments: +['my_test_code', 'param1', 'param2'] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete job some_other_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UUID1] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/script b/acceptance/bundle/integration_whl/wrapper_custom_params/script new file mode 100644 index 0000000000..28b0839c00 --- /dev/null +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/script @@ -0,0 +1,8 @@ +export SPARK_VERSION=12.2.x-scala2.12 +export PYTHON_WHEEL_WRAPPER=true +envsubst < $TESTDIR/../base/input.json.tmpl > input.json +cat input.json +$CLI bundle init $TESTDIR/../base --config-file input.json +trace $CLI bundle deploy +trap "errcode trace $CLI bundle destroy --auto-approve" EXIT +trace $CLI bundle run some_other_job --python-params param1,param2 From 01e8eb45a03fc254b0a725aa5d846617568cb609 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 11 Mar 2025 17:35:18 +0100 Subject: [PATCH 04/10] update --- acceptance/acceptance_test.go | 10 ++++++---- .../integration_whl/base/input.json.tmpl | 2 +- .../bundle/integration_whl/base/output.txt | 10 +++++----- .../integration_whl/custom_params/output.txt | 10 +++++----- .../bundle/integration_whl/wrapper/output.txt | 18 +++++++++--------- .../wrapper_custom_params/output.txt | 14 +++++++------- 6 files changed, 33 insertions(+), 31 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index ffff75b779..1e4e94d293 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -3,6 +3,7 @@ package acceptance_test import ( "bufio" "context" + "encoding/base32" "encoding/json" "errors" "flag" @@ -258,9 +259,10 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont } } - // XXX rename away from uuid1 since it's not uuid due to replacements - uuid1 := strings.ReplaceAll(uuid.NewString(), "-", "_") - repls.Set(uuid1, "[UUID1]") + id := uuid.New() + uniqueName := strings.Trim(base32.StdEncoding.EncodeToString(id[:]), "=") + require.Greater(t, len(uniqueName), 8, uniqueName) + repls.Set(uniqueName, "[UNIQUE_NAME]") var tmpDir string var err error @@ -287,7 +289,7 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont args := []string{"bash", "-euo", "pipefail", EntryPointScript} cmd := exec.Command(args[0], args[1:]...) cmd.Env = os.Environ() - cmd.Env = append(cmd.Env, "UUID1="+uuid1) + cmd.Env = append(cmd.Env, "UNIQUE_NAME="+uniqueName) var workspaceClient *databricks.WorkspaceClient var user iam.User diff --git a/acceptance/bundle/integration_whl/base/input.json.tmpl b/acceptance/bundle/integration_whl/base/input.json.tmpl index 292bca7d40..8d4255be28 100644 --- a/acceptance/bundle/integration_whl/base/input.json.tmpl +++ b/acceptance/bundle/integration_whl/base/input.json.tmpl @@ -2,7 +2,7 @@ "project_name": "my_test_code", "spark_version": "$SPARK_VERSION", "node_type_id": "$NODE_TYPE_ID", - "unique_id": "$UUID1", + "unique_id": "$UNIQUE_NAME", "python_wheel_wrapper": $PYTHON_WHEEL_WRAPPER, "instance_pool_id": "$TEST_INSTANCE_POOL_ID" } diff --git a/acceptance/bundle/integration_whl/base/output.txt b/acceptance/bundle/integration_whl/base/output.txt index 71eaea6a7c..a6aadac831 100644 --- a/acceptance/bundle/integration_whl/base/output.txt +++ b/acceptance/bundle/integration_whl/base/output.txt @@ -2,7 +2,7 @@ "project_name": "my_test_code", "spark_version": "13.3.x-snapshot-scala2.12", "node_type_id": "", - "unique_id": "[UUID1]", + "unique_id": "[UNIQUE_NAME]", "python_wheel_wrapper": false, "instance_pool_id": "[TEST_INSTANCE_POOL_ID]" } @@ -11,7 +11,7 @@ >>> [CLI] bundle deploy Building python_artifact... Uploading my_test_code-0.0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UUID1]/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Deploying resources... Updating deployment state... Deployment complete! @@ -19,8 +19,8 @@ Deployment complete! >>> [CLI] bundle run some_other_job Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] -[TIMESTAMP] "[default] Test Wheel Job [UUID1]" RUNNING -[TIMESTAMP] "[default] Test Wheel Job [UUID1]" TERMINATED SUCCESS +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS Hello from my func Got arguments: ['my_test_code', 'one', 'two'] @@ -29,7 +29,7 @@ Got arguments: The following resources will be deleted: delete job some_other_job -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UUID1] +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Deleting files... Destroy complete! diff --git a/acceptance/bundle/integration_whl/custom_params/output.txt b/acceptance/bundle/integration_whl/custom_params/output.txt index 3dca29b3c1..f4715eab79 100644 --- a/acceptance/bundle/integration_whl/custom_params/output.txt +++ b/acceptance/bundle/integration_whl/custom_params/output.txt @@ -2,7 +2,7 @@ "project_name": "my_test_code", "spark_version": "13.3.x-snapshot-scala2.12", "node_type_id": "", - "unique_id": "[UUID1]", + "unique_id": "[UNIQUE_NAME]", "python_wheel_wrapper": false, "instance_pool_id": "[TEST_INSTANCE_POOL_ID]" } @@ -11,7 +11,7 @@ >>> [CLI] bundle deploy Building python_artifact... Uploading my_test_code-0.0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UUID1]/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Deploying resources... Updating deployment state... Deployment complete! @@ -19,8 +19,8 @@ Deployment complete! >>> [CLI] bundle run some_other_job --python-params param1,param2 Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] -[TIMESTAMP] "[default] Test Wheel Job [UUID1]" RUNNING -[TIMESTAMP] "[default] Test Wheel Job [UUID1]" TERMINATED SUCCESS +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS Hello from my func Got arguments: ['my_test_code', 'param1', 'param2'] @@ -29,7 +29,7 @@ Got arguments: The following resources will be deleted: delete job some_other_job -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UUID1] +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Deleting files... Destroy complete! diff --git a/acceptance/bundle/integration_whl/wrapper/output.txt b/acceptance/bundle/integration_whl/wrapper/output.txt index 3dca29b3c1..ca1819f3c8 100644 --- a/acceptance/bundle/integration_whl/wrapper/output.txt +++ b/acceptance/bundle/integration_whl/wrapper/output.txt @@ -1,9 +1,9 @@ { "project_name": "my_test_code", - "spark_version": "13.3.x-snapshot-scala2.12", + "spark_version": "12.2.x-scala2.12", "node_type_id": "", - "unique_id": "[UUID1]", - "python_wheel_wrapper": false, + "unique_id": "[UNIQUE_NAME]", + "python_wheel_wrapper": true, "instance_pool_id": "[TEST_INSTANCE_POOL_ID]" } ✨ Successfully initialized template @@ -11,25 +11,25 @@ >>> [CLI] bundle deploy Building python_artifact... Uploading my_test_code-0.0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UUID1]/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Deploying resources... Updating deployment state... Deployment complete! ->>> [CLI] bundle run some_other_job --python-params param1,param2 +>>> [CLI] bundle run some_other_job Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] -[TIMESTAMP] "[default] Test Wheel Job [UUID1]" RUNNING -[TIMESTAMP] "[default] Test Wheel Job [UUID1]" TERMINATED SUCCESS +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS Hello from my func Got arguments: -['my_test_code', 'param1', 'param2'] +['my_test_code', 'one', 'two'] >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: delete job some_other_job -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UUID1] +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Deleting files... Destroy complete! diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt b/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt index 3dca29b3c1..b5f97d5bce 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/output.txt @@ -1,9 +1,9 @@ { "project_name": "my_test_code", - "spark_version": "13.3.x-snapshot-scala2.12", + "spark_version": "12.2.x-scala2.12", "node_type_id": "", - "unique_id": "[UUID1]", - "python_wheel_wrapper": false, + "unique_id": "[UNIQUE_NAME]", + "python_wheel_wrapper": true, "instance_pool_id": "[TEST_INSTANCE_POOL_ID]" } ✨ Successfully initialized template @@ -11,7 +11,7 @@ >>> [CLI] bundle deploy Building python_artifact... Uploading my_test_code-0.0.1-py3-none-any.whl... -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UUID1]/files... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Deploying resources... Updating deployment state... Deployment complete! @@ -19,8 +19,8 @@ Deployment complete! >>> [CLI] bundle run some_other_job --python-params param1,param2 Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] -[TIMESTAMP] "[default] Test Wheel Job [UUID1]" RUNNING -[TIMESTAMP] "[default] Test Wheel Job [UUID1]" TERMINATED SUCCESS +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS Hello from my func Got arguments: ['my_test_code', 'param1', 'param2'] @@ -29,7 +29,7 @@ Got arguments: The following resources will be deleted: delete job some_other_job -All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UUID1] +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] Deleting files... Destroy complete! From c55818a76ae95cd5c07e125d04e8a0f8029a6b07 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 11 Mar 2025 17:48:53 +0100 Subject: [PATCH 05/10] clean up python_wheel_task tests from integration --- integration/bundle/python_wheel_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/integration/bundle/python_wheel_test.go b/integration/bundle/python_wheel_test.go index 62846f7b5c..98d0423aaf 100644 --- a/integration/bundle/python_wheel_test.go +++ b/integration/bundle/python_wheel_test.go @@ -47,14 +47,6 @@ func runPythonWheelTest(t *testing.T, templateName, sparkVersion string, pythonW require.Contains(t, out, "['my_test_code', 'param1', 'param2']") } -func TestPythonWheelTaskDeployAndRunWithoutWrapper(t *testing.T) { - runPythonWheelTest(t, "python_wheel_task", "13.3.x-snapshot-scala2.12", false) -} - -func TestPythonWheelTaskDeployAndRunWithWrapper(t *testing.T) { - runPythonWheelTest(t, "python_wheel_task", "12.2.x-scala2.12", true) -} - func TestPythonWheelTaskDeployAndRunOnInteractiveCluster(t *testing.T) { if testutil.GetCloud(t) == testutil.AWS { t.Skip("Skipping test for AWS cloud because it is not permitted to create clusters") From 21f7bac87c04d171fe8231970034755897bc0521 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Tue, 11 Mar 2025 18:42:53 +0100 Subject: [PATCH 06/10] Add single quotes around $CLI --- acceptance/bundle/integration_whl/base/script | 2 +- acceptance/bundle/integration_whl/custom_params/script | 2 +- acceptance/bundle/integration_whl/wrapper/script | 2 +- acceptance/bundle/integration_whl/wrapper_custom_params/script | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/integration_whl/base/script b/acceptance/bundle/integration_whl/base/script index a0d6af9890..24398ecbab 100644 --- a/acceptance/bundle/integration_whl/base/script +++ b/acceptance/bundle/integration_whl/base/script @@ -4,5 +4,5 @@ envsubst < input.json.tmpl > input.json cat input.json $CLI bundle init . --config-file input.json trace $CLI bundle deploy -trap "errcode trace $CLI bundle destroy --auto-approve" EXIT +trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT trace $CLI bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/custom_params/script b/acceptance/bundle/integration_whl/custom_params/script index 8148417e3a..7fc3900f8f 100644 --- a/acceptance/bundle/integration_whl/custom_params/script +++ b/acceptance/bundle/integration_whl/custom_params/script @@ -4,5 +4,5 @@ envsubst < $TESTDIR/../base/input.json.tmpl > input.json cat input.json $CLI bundle init $TESTDIR/../base --config-file input.json trace $CLI bundle deploy -trap "errcode trace $CLI bundle destroy --auto-approve" EXIT +trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT trace $CLI bundle run some_other_job --python-params param1,param2 diff --git a/acceptance/bundle/integration_whl/wrapper/script b/acceptance/bundle/integration_whl/wrapper/script index 3c921b8880..f11a3246d0 100644 --- a/acceptance/bundle/integration_whl/wrapper/script +++ b/acceptance/bundle/integration_whl/wrapper/script @@ -4,5 +4,5 @@ envsubst < $TESTDIR/../base/input.json.tmpl > input.json cat input.json $CLI bundle init $TESTDIR/../base --config-file input.json trace $CLI bundle deploy -trap "errcode trace $CLI bundle destroy --auto-approve" EXIT +trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT trace $CLI bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/script b/acceptance/bundle/integration_whl/wrapper_custom_params/script index 28b0839c00..2911d9f861 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/script +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/script @@ -4,5 +4,5 @@ envsubst < $TESTDIR/../base/input.json.tmpl > input.json cat input.json $CLI bundle init $TESTDIR/../base --config-file input.json trace $CLI bundle deploy -trap "errcode trace $CLI bundle destroy --auto-approve" EXIT +trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT trace $CLI bundle run some_other_job --python-params param1,param2 From d533906f11bbaec68ba4602e47a806d17aa4d9e2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 12 Mar 2025 09:34:41 +0100 Subject: [PATCH 07/10] Migrate TestPythonWheelTaskDeployAndRunOnInteractiveCluster --- acceptance/acceptance_test.go | 3 + acceptance/bundle/integration_whl/base/script | 2 +- .../integration_whl/custom_params/script | 2 +- .../databricks_template_schema.json | 0 .../interactive_cluster/output.txt | 35 ++++++++++++ .../interactive_cluster/script | 8 +++ .../template/databricks.yml.tmpl | 0 .../template/setup.py.tmpl | 0 .../template/{{.project_name}}/__init__.py | 0 .../template/{{.project_name}}/__main__.py | 0 integration/bundle/python_wheel_test.go | 56 ------------------- 11 files changed, 48 insertions(+), 58 deletions(-) rename {integration/bundle/bundles/python_wheel_task_with_cluster => acceptance/bundle/integration_whl/interactive_cluster}/databricks_template_schema.json (100%) create mode 100644 acceptance/bundle/integration_whl/interactive_cluster/output.txt create mode 100644 acceptance/bundle/integration_whl/interactive_cluster/script rename {integration/bundle/bundles/python_wheel_task_with_cluster => acceptance/bundle/integration_whl/interactive_cluster}/template/databricks.yml.tmpl (100%) rename {integration/bundle/bundles/python_wheel_task_with_cluster => acceptance/bundle/integration_whl/interactive_cluster}/template/setup.py.tmpl (100%) rename {integration/bundle/bundles/python_wheel_task_with_cluster => acceptance/bundle/integration_whl/interactive_cluster}/template/{{.project_name}}/__init__.py (100%) rename {integration/bundle/bundles/python_wheel_task_with_cluster => acceptance/bundle/integration_whl/interactive_cluster}/template/{{.project_name}}/__main__.py (100%) delete mode 100644 integration/bundle/python_wheel_test.go diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 1e4e94d293..ac2648fa64 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -170,6 +170,9 @@ func testAccept(t *testing.T, InprocessMode bool, singleTest string) int { repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"}) + // Matches defaultSparkVersion in ../integration/bundle/helpers_test.go + t.Setenv("DEFAULT_SPARK_VERSION", "13.3.x-snapshot-scala2.12") + testDirs := getTests(t) require.NotEmpty(t, testDirs) diff --git a/acceptance/bundle/integration_whl/base/script b/acceptance/bundle/integration_whl/base/script index 24398ecbab..2af401c88f 100644 --- a/acceptance/bundle/integration_whl/base/script +++ b/acceptance/bundle/integration_whl/base/script @@ -1,4 +1,4 @@ -export SPARK_VERSION=13.3.x-snapshot-scala2.12 +export SPARK_VERSION=$DEFAULT_SPARK_VERSION export PYTHON_WHEEL_WRAPPER=false envsubst < input.json.tmpl > input.json cat input.json diff --git a/acceptance/bundle/integration_whl/custom_params/script b/acceptance/bundle/integration_whl/custom_params/script index 7fc3900f8f..21ac535a49 100644 --- a/acceptance/bundle/integration_whl/custom_params/script +++ b/acceptance/bundle/integration_whl/custom_params/script @@ -1,4 +1,4 @@ -export SPARK_VERSION=13.3.x-snapshot-scala2.12 +export SPARK_VERSION=$DEFAULT_SPARK_VERSION export PYTHON_WHEEL_WRAPPER=false envsubst < $TESTDIR/../base/input.json.tmpl > input.json cat input.json diff --git a/integration/bundle/bundles/python_wheel_task_with_cluster/databricks_template_schema.json b/acceptance/bundle/integration_whl/interactive_cluster/databricks_template_schema.json similarity index 100% rename from integration/bundle/bundles/python_wheel_task_with_cluster/databricks_template_schema.json rename to acceptance/bundle/integration_whl/interactive_cluster/databricks_template_schema.json diff --git a/acceptance/bundle/integration_whl/interactive_cluster/output.txt b/acceptance/bundle/integration_whl/interactive_cluster/output.txt new file mode 100644 index 0000000000..a6aadac831 --- /dev/null +++ b/acceptance/bundle/integration_whl/interactive_cluster/output.txt @@ -0,0 +1,35 @@ +{ + "project_name": "my_test_code", + "spark_version": "13.3.x-snapshot-scala2.12", + "node_type_id": "", + "unique_id": "[UNIQUE_NAME]", + "python_wheel_wrapper": false, + "instance_pool_id": "[TEST_INSTANCE_POOL_ID]" +} +✨ Successfully initialized template + +>>> [CLI] bundle deploy +Building python_artifact... +Uploading my_test_code-0.0.1-py3-none-any.whl... +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle run some_other_job +Run URL: [DATABRICKS_URL]/?o=[NUMID]#job/[NUMID]/run/[NUMID] + +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" RUNNING +[TIMESTAMP] "[default] Test Wheel Job [UNIQUE_NAME]" TERMINATED SUCCESS +Hello from my func +Got arguments: +['my_test_code', 'one', 'two'] + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete job some_other_job + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME] + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/integration_whl/interactive_cluster/script b/acceptance/bundle/integration_whl/interactive_cluster/script new file mode 100644 index 0000000000..2af401c88f --- /dev/null +++ b/acceptance/bundle/integration_whl/interactive_cluster/script @@ -0,0 +1,8 @@ +export SPARK_VERSION=$DEFAULT_SPARK_VERSION +export PYTHON_WHEEL_WRAPPER=false +envsubst < input.json.tmpl > input.json +cat input.json +$CLI bundle init . --config-file input.json +trace $CLI bundle deploy +trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT +trace $CLI bundle run some_other_job diff --git a/integration/bundle/bundles/python_wheel_task_with_cluster/template/databricks.yml.tmpl b/acceptance/bundle/integration_whl/interactive_cluster/template/databricks.yml.tmpl similarity index 100% rename from integration/bundle/bundles/python_wheel_task_with_cluster/template/databricks.yml.tmpl rename to acceptance/bundle/integration_whl/interactive_cluster/template/databricks.yml.tmpl diff --git a/integration/bundle/bundles/python_wheel_task_with_cluster/template/setup.py.tmpl b/acceptance/bundle/integration_whl/interactive_cluster/template/setup.py.tmpl similarity index 100% rename from integration/bundle/bundles/python_wheel_task_with_cluster/template/setup.py.tmpl rename to acceptance/bundle/integration_whl/interactive_cluster/template/setup.py.tmpl diff --git a/integration/bundle/bundles/python_wheel_task_with_cluster/template/{{.project_name}}/__init__.py b/acceptance/bundle/integration_whl/interactive_cluster/template/{{.project_name}}/__init__.py similarity index 100% rename from integration/bundle/bundles/python_wheel_task_with_cluster/template/{{.project_name}}/__init__.py rename to acceptance/bundle/integration_whl/interactive_cluster/template/{{.project_name}}/__init__.py diff --git a/integration/bundle/bundles/python_wheel_task_with_cluster/template/{{.project_name}}/__main__.py b/acceptance/bundle/integration_whl/interactive_cluster/template/{{.project_name}}/__main__.py similarity index 100% rename from integration/bundle/bundles/python_wheel_task_with_cluster/template/{{.project_name}}/__main__.py rename to acceptance/bundle/integration_whl/interactive_cluster/template/{{.project_name}}/__main__.py diff --git a/integration/bundle/python_wheel_test.go b/integration/bundle/python_wheel_test.go deleted file mode 100644 index 98d0423aaf..0000000000 --- a/integration/bundle/python_wheel_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package bundle_test - -import ( - "testing" - - "github.com/databricks/cli/integration/internal/acc" - "github.com/databricks/cli/internal/testutil" - "github.com/databricks/cli/libs/env" - "github.com/google/uuid" - "github.com/stretchr/testify/require" -) - -func runPythonWheelTest(t *testing.T, templateName, sparkVersion string, pythonWheelWrapper bool) { - ctx, _ := acc.WorkspaceTest(t) - - nodeTypeId := testutil.GetCloud(t).NodeTypeID() - instancePoolId := env.Get(ctx, "TEST_INSTANCE_POOL_ID") - bundleRoot := initTestTemplate(t, ctx, templateName, map[string]any{ - "node_type_id": nodeTypeId, - "unique_id": uuid.New().String(), - "spark_version": sparkVersion, - "python_wheel_wrapper": pythonWheelWrapper, - "instance_pool_id": instancePoolId, - }) - - deployBundle(t, ctx, bundleRoot) - - t.Cleanup(func() { - destroyBundle(t, ctx, bundleRoot) - }) - - if testing.Short() { - t.Log("Skip the job run in short mode") - return - } - - out, err := runResource(t, ctx, bundleRoot, "some_other_job") - require.NoError(t, err) - require.Contains(t, out, "Hello from my func") - require.Contains(t, out, "Got arguments:") - require.Contains(t, out, "['my_test_code', 'one', 'two']") - - out, err = runResourceWithParams(t, ctx, bundleRoot, "some_other_job", "--python-params=param1,param2") - require.NoError(t, err) - require.Contains(t, out, "Hello from my func") - require.Contains(t, out, "Got arguments:") - require.Contains(t, out, "['my_test_code', 'param1', 'param2']") -} - -func TestPythonWheelTaskDeployAndRunOnInteractiveCluster(t *testing.T) { - if testutil.GetCloud(t) == testutil.AWS { - t.Skip("Skipping test for AWS cloud because it is not permitted to create clusters") - } - - runPythonWheelTest(t, "python_wheel_task_with_cluster", defaultSparkVersion, false) -} From a549009e61c14846adc9ede448120579909f3794 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 12 Mar 2025 09:54:39 +0100 Subject: [PATCH 08/10] fix script --- acceptance/bundle/integration_whl/interactive_cluster/script | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/integration_whl/interactive_cluster/script b/acceptance/bundle/integration_whl/interactive_cluster/script index 2af401c88f..790f84d89e 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/script +++ b/acceptance/bundle/integration_whl/interactive_cluster/script @@ -1,8 +1,8 @@ export SPARK_VERSION=$DEFAULT_SPARK_VERSION export PYTHON_WHEEL_WRAPPER=false -envsubst < input.json.tmpl > input.json +envsubst < $TESTDIR/../base/input.json.tmpl > input.json cat input.json -$CLI bundle init . --config-file input.json +$CLI bundle init $TESTDIR/../base --config-file input.json trace $CLI bundle deploy trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT trace $CLI bundle run some_other_job From b080fa8776c583108ca6d96ee47c0109b1847cda Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 12 Mar 2025 10:47:24 +0100 Subject: [PATCH 09/10] move trap before deploy --- acceptance/bundle/integration_whl/base/script | 2 +- acceptance/bundle/integration_whl/custom_params/script | 2 +- acceptance/bundle/integration_whl/interactive_cluster/script | 2 +- acceptance/bundle/integration_whl/wrapper/script | 2 +- acceptance/bundle/integration_whl/wrapper_custom_params/script | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/integration_whl/base/script b/acceptance/bundle/integration_whl/base/script index 2af401c88f..06c3bffdc5 100644 --- a/acceptance/bundle/integration_whl/base/script +++ b/acceptance/bundle/integration_whl/base/script @@ -3,6 +3,6 @@ export PYTHON_WHEEL_WRAPPER=false envsubst < input.json.tmpl > input.json cat input.json $CLI bundle init . --config-file input.json -trace $CLI bundle deploy trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT +trace $CLI bundle deploy trace $CLI bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/custom_params/script b/acceptance/bundle/integration_whl/custom_params/script index 21ac535a49..3abb7aafa9 100644 --- a/acceptance/bundle/integration_whl/custom_params/script +++ b/acceptance/bundle/integration_whl/custom_params/script @@ -3,6 +3,6 @@ export PYTHON_WHEEL_WRAPPER=false envsubst < $TESTDIR/../base/input.json.tmpl > input.json cat input.json $CLI bundle init $TESTDIR/../base --config-file input.json -trace $CLI bundle deploy trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT +trace $CLI bundle deploy trace $CLI bundle run some_other_job --python-params param1,param2 diff --git a/acceptance/bundle/integration_whl/interactive_cluster/script b/acceptance/bundle/integration_whl/interactive_cluster/script index 790f84d89e..7e10d6299d 100644 --- a/acceptance/bundle/integration_whl/interactive_cluster/script +++ b/acceptance/bundle/integration_whl/interactive_cluster/script @@ -3,6 +3,6 @@ export PYTHON_WHEEL_WRAPPER=false envsubst < $TESTDIR/../base/input.json.tmpl > input.json cat input.json $CLI bundle init $TESTDIR/../base --config-file input.json -trace $CLI bundle deploy trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT +trace $CLI bundle deploy trace $CLI bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/wrapper/script b/acceptance/bundle/integration_whl/wrapper/script index f11a3246d0..ffbfece606 100644 --- a/acceptance/bundle/integration_whl/wrapper/script +++ b/acceptance/bundle/integration_whl/wrapper/script @@ -3,6 +3,6 @@ export PYTHON_WHEEL_WRAPPER=true envsubst < $TESTDIR/../base/input.json.tmpl > input.json cat input.json $CLI bundle init $TESTDIR/../base --config-file input.json -trace $CLI bundle deploy trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT +trace $CLI bundle deploy trace $CLI bundle run some_other_job diff --git a/acceptance/bundle/integration_whl/wrapper_custom_params/script b/acceptance/bundle/integration_whl/wrapper_custom_params/script index 2911d9f861..c92f7162db 100644 --- a/acceptance/bundle/integration_whl/wrapper_custom_params/script +++ b/acceptance/bundle/integration_whl/wrapper_custom_params/script @@ -3,6 +3,6 @@ export PYTHON_WHEEL_WRAPPER=true envsubst < $TESTDIR/../base/input.json.tmpl > input.json cat input.json $CLI bundle init $TESTDIR/../base --config-file input.json -trace $CLI bundle deploy trap "errcode trace '$CLI' bundle destroy --auto-approve" EXIT +trace $CLI bundle deploy trace $CLI bundle run some_other_job --python-params param1,param2 From 060b8f50898fdc4fd1032f75e8fc0490c6221f49 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 12 Mar 2025 10:59:04 +0100 Subject: [PATCH 10/10] add a comment from Andrew wrapper --- acceptance/bundle/integration_whl/wrapper/script | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/acceptance/bundle/integration_whl/wrapper/script b/acceptance/bundle/integration_whl/wrapper/script index ffbfece606..ee57bc783e 100644 --- a/acceptance/bundle/integration_whl/wrapper/script +++ b/acceptance/bundle/integration_whl/wrapper/script @@ -1,3 +1,7 @@ +# Installing wheels from Workspace file system is only supported starting from DBR 13.1+. +# But before users used older DBRs and python wheel tasks but installed it from DBFS. +# We still want to support older DBRs and did the trampoline workaround (https://github.com/databricks/cli/pull/635) +# Hence this is to test that python wheel tasks in DABs are working for older DBRs export SPARK_VERSION=12.2.x-scala2.12 export PYTHON_WHEEL_WRAPPER=true envsubst < $TESTDIR/../base/input.json.tmpl > input.json