Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions job_with_python_wheel_task/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
.databricks
dist
build
*.egg-info
*.pyc
6 changes: 6 additions & 0 deletions job_with_python_wheel_task/common/lib/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
__version__ = "0.0.1"
__author__ = "Databricks"


def some_lib_method():
print("Hello from common lib")
14 changes: 14 additions & 0 deletions job_with_python_wheel_task/common/setup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
from setuptools import setup, find_packages

import lib

setup(
name="lib",
version=lib.__version__,
author=lib.__author__,
url="https://databricks.com",
author_email="john.doe@databricks.com",
description="my lib wheel",
packages=find_packages(include=["lib"]),
install_requires=["setuptools"],
)
29 changes: 29 additions & 0 deletions job_with_python_wheel_task/example/databricks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
bundle:
name: job_with_python_wheel_task

artifacts:
common:
type: whl
path: "../common"
example:
type: whl

workspace:
host: https://e2-dogfood.staging.cloud.databricks.com/

resources:
jobs:
test_job:
name: "[${bundle.environment}] Job With Wheel Task"
tasks:
- task_key: TestTask
new_cluster:
num_workers: 1
spark_version: 13.2.x-snapshot-scala2.12
node_type_id: i3.xlarge
python_wheel_task:
package_name: "example"
entry_point: "run"
libraries:
- whl: ../common/dist/*.whl
- whl: ./dist/*.whl
15 changes: 15 additions & 0 deletions job_with_python_wheel_task/example/setup.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
from setuptools import setup, find_packages

import src

setup(
name="example",
version=src.__version__,
author=src.__author__,
url="https://databricks.com",
author_email="john.doe@databricks.com",
description="my example wheel",
packages=find_packages(include=["src"]),
entry_points={"group_1": "run=src.__main__:main"},
install_requires=["setuptools"],
)
6 changes: 6 additions & 0 deletions job_with_python_wheel_task/example/src/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
__version__ = "0.0.1"
__author__ = "Databricks"

import sys, os

sys.path.append(os.path.join(os.path.dirname(__file__), "..", ".."))
19 changes: 19 additions & 0 deletions job_with_python_wheel_task/example/src/__main__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
"""
The entry point of the Python Wheel
"""

import sys, os

sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "common"))
Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This shouldn't be necessary during execution, because the wheels are all available already, right?

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is to make sure code works both on Databricks and locally

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should figure out a way to make this work without having to do this.

Would a virtualenv + installing these in development mode work?

https://packaging.python.org/en/latest/guides/distributing-packages-using-setuptools/#working-in-development-mode


from lib import some_lib_method


def main():
# This method will print the provided arguments
print("Hello from my example")
some_lib_method()


if __name__ == "__main__":
main()