diff --git a/job_with_python_wheel_task/.gitignore b/job_with_python_wheel_task/.gitignore new file mode 100644 index 00000000..f9c4a055 --- /dev/null +++ b/job_with_python_wheel_task/.gitignore @@ -0,0 +1,5 @@ +.databricks +dist +build +*.egg-info +*.pyc \ No newline at end of file diff --git a/job_with_python_wheel_task/common/lib/__init__.py b/job_with_python_wheel_task/common/lib/__init__.py new file mode 100644 index 00000000..a19e8f00 --- /dev/null +++ b/job_with_python_wheel_task/common/lib/__init__.py @@ -0,0 +1,6 @@ +__version__ = "0.0.1" +__author__ = "Databricks" + + +def some_lib_method(): + print("Hello from common lib") diff --git a/job_with_python_wheel_task/common/setup.py b/job_with_python_wheel_task/common/setup.py new file mode 100644 index 00000000..ae095336 --- /dev/null +++ b/job_with_python_wheel_task/common/setup.py @@ -0,0 +1,14 @@ +from setuptools import setup, find_packages + +import lib + +setup( + name="lib", + version=lib.__version__, + author=lib.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my lib wheel", + packages=find_packages(include=["lib"]), + install_requires=["setuptools"], +) diff --git a/job_with_python_wheel_task/example/databricks.yml b/job_with_python_wheel_task/example/databricks.yml new file mode 100644 index 00000000..6b8b8d56 --- /dev/null +++ b/job_with_python_wheel_task/example/databricks.yml @@ -0,0 +1,29 @@ +bundle: + name: job_with_python_wheel_task + +artifacts: + common: + type: whl + path: "../common" + example: + type: whl + +workspace: + host: https://e2-dogfood.staging.cloud.databricks.com/ + +resources: + jobs: + test_job: + name: "[${bundle.environment}] Job With Wheel Task" + tasks: + - task_key: TestTask + new_cluster: + num_workers: 1 + spark_version: 13.2.x-snapshot-scala2.12 + node_type_id: i3.xlarge + python_wheel_task: + package_name: "example" + entry_point: "run" + libraries: + - whl: ../common/dist/*.whl + - whl: ./dist/*.whl diff --git a/job_with_python_wheel_task/example/setup.py b/job_with_python_wheel_task/example/setup.py new file mode 100644 index 00000000..9d4eec5a --- /dev/null +++ b/job_with_python_wheel_task/example/setup.py @@ -0,0 +1,15 @@ +from setuptools import setup, find_packages + +import src + +setup( + name="example", + version=src.__version__, + author=src.__author__, + url="https://databricks.com", + author_email="john.doe@databricks.com", + description="my example wheel", + packages=find_packages(include=["src"]), + entry_points={"group_1": "run=src.__main__:main"}, + install_requires=["setuptools"], +) diff --git a/job_with_python_wheel_task/example/src/__init__.py b/job_with_python_wheel_task/example/src/__init__.py new file mode 100644 index 00000000..bb32a876 --- /dev/null +++ b/job_with_python_wheel_task/example/src/__init__.py @@ -0,0 +1,6 @@ +__version__ = "0.0.1" +__author__ = "Databricks" + +import sys, os + +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..")) diff --git a/job_with_python_wheel_task/example/src/__main__.py b/job_with_python_wheel_task/example/src/__main__.py new file mode 100644 index 00000000..06e144cc --- /dev/null +++ b/job_with_python_wheel_task/example/src/__main__.py @@ -0,0 +1,19 @@ +""" +The entry point of the Python Wheel +""" + +import sys, os + +sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "common")) + +from lib import some_lib_method + + +def main(): + # This method will print the provided arguments + print("Hello from my example") + some_lib_method() + + +if __name__ == "__main__": + main()