From 8057da70b6d21d420337e47a0782b5df16187119 Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Tue, 5 Sep 2023 11:48:34 +0200 Subject: [PATCH 01/14] new feature --- .../cli_commands/generate/databricks_job.py | 71 +++++++++++++++++++ .../cli_commands/generate/generate.py | 6 +- 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 data_pipelines_cli/cli_commands/generate/databricks_job.py diff --git a/data_pipelines_cli/cli_commands/generate/databricks_job.py b/data_pipelines_cli/cli_commands/generate/databricks_job.py new file mode 100644 index 0000000..25a10a3 --- /dev/null +++ b/data_pipelines_cli/cli_commands/generate/databricks_job.py @@ -0,0 +1,71 @@ +import click + +from dbt_databricks_factory.cli import create_job_cli +from dbt_databricks_factory.config import GitProvider + +@click.command("databricks-job", help="Generate a Databricks job") +@click.argument( + "manifest-file", + type=click.Path(exists=True, file_okay=True, dir_okay=False, readable=True), +) +@click.option("--job-name", required=True, help="Name of the job to create.") +@click.option("--project-dir", required=True, help="Path to dbt project directory.") +@click.option("--profiles-dir", required=True, help="Path to dbt profiles directory.") +@click.option("--cron-schedule", help="Cron schedule for the job.") +@click.option("--job-cluster", multiple=True, type=click.Tuple([str, str]), help="Job cluster config.") +@click.option( + "--task-cluster", + multiple=True, + type=click.Tuple([str, str]), + help="Job cluster name or existing cluster id.", +) +@click.option("--default-task-cluster", help="Default task cluster name or existing cluster id.") +@click.option("--library", multiple=True, type=str, help="Libraries config.") +@click.option("--git-url", required=True, help="Git url.") +@click.option("--git-branch", help="Git branch.") +@click.option("--git-commit", help="Git commit.") +@click.option("--git-tag", help="Git tag.") +@click.option( + "--git-provider", + required=True, + help="Git provider.", + type=click.Choice([provider.value for provider in GitProvider]), +) +@click.option("--pretty", is_flag=True, help="Pretty print the output.") +@click.option("--output-file", help="Output file path.", type=click.Path(file_okay=True, dir_okay=False, writable=True)) +def generate_databricks_job_command( + job_name: str, + manifest_file: str, + project_dir: str, + profiles_dir: str, + cron_schedule: str | None, + job_cluster: list[tuple[str, str]], + task_cluster: list[tuple[str, str]], + default_task_cluster: str | None, + library: list[str], + git_url: str, + git_branch: str | None, + git_commit: str | None, + git_tag: str | None, + git_provider: str, + pretty: bool, + output_file: str, +) -> None: + create_job_cli( + job_name, + manifest_file, + project_dir, + profiles_dir, + cron_schedule, + job_cluster, + task_cluster, + default_task_cluster, + library, + git_url, + git_branch, + git_commit, + git_tag, + git_provider, + pretty, + output_file, + ) diff --git a/data_pipelines_cli/cli_commands/generate/generate.py b/data_pipelines_cli/cli_commands/generate/generate.py index fa2f706..981c18e 100644 --- a/data_pipelines_cli/cli_commands/generate/generate.py +++ b/data_pipelines_cli/cli_commands/generate/generate.py @@ -3,13 +3,17 @@ from .model_yaml import generate_model_yamls_command from .source_sql import generate_source_sqls_command from .source_yaml import generate_source_yamls_command - +from dbt_databricks_factory.cli import create_job_cli @click.group(name="generate", help="Generate additional dbt files") def generate_group() -> None: pass +@click.command("databricks-job", help="Generate a Databricks job") +def generate_databricks_job_command() -> None: + create_job_cli() + generate_group.add_command(generate_model_yamls_command) generate_group.add_command(generate_source_sqls_command) generate_group.add_command(generate_source_yamls_command) From 3b2d5d8f2cf29431ab00e22ba68de0ece460114c Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Tue, 5 Sep 2023 11:49:50 +0200 Subject: [PATCH 02/14] small fixes --- data_pipelines_cli/cli_commands/generate/generate.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/data_pipelines_cli/cli_commands/generate/generate.py b/data_pipelines_cli/cli_commands/generate/generate.py index 981c18e..a35d764 100644 --- a/data_pipelines_cli/cli_commands/generate/generate.py +++ b/data_pipelines_cli/cli_commands/generate/generate.py @@ -3,17 +3,14 @@ from .model_yaml import generate_model_yamls_command from .source_sql import generate_source_sqls_command from .source_yaml import generate_source_yamls_command -from dbt_databricks_factory.cli import create_job_cli +from .databricks_job import generate_databricks_job_command @click.group(name="generate", help="Generate additional dbt files") def generate_group() -> None: pass -@click.command("databricks-job", help="Generate a Databricks job") -def generate_databricks_job_command() -> None: - create_job_cli() - generate_group.add_command(generate_model_yamls_command) generate_group.add_command(generate_source_sqls_command) generate_group.add_command(generate_source_yamls_command) +generate_group.add_command(generate_databricks_job_command) From 534032b78f79654453b3f9629a91a0cc76dfd7b0 Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Wed, 6 Sep 2023 08:23:53 +0200 Subject: [PATCH 03/14] dbt databricks factory requirement --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 6dfbcd0..09bcf06 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ "colorama==0.4.5", "dbt-core==1.5.4", "pydantic<2", + "dbt-databricks-factory>=0.1.1", ] EXTRA_FILESYSTEMS_REQUIRE = { From 4fd2a12a34e566f06d1e6ebfa2e372aa518a1bc5 Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 09:42:13 +0200 Subject: [PATCH 04/14] python 3.9 onwards --- .github/workflows/prepare-release.yml | 2 +- .github/workflows/python-package.yml | 2 +- README.md | 2 +- data_pipelines_cli/cli_commands/generate/generate.py | 10 +++++++--- docs/index.rst | 2 +- setup.py | 8 ++++---- tox.ini | 3 +-- 7 files changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml index 8b67cdc..b08f46f 100644 --- a/.github/workflows/prepare-release.yml +++ b/.github/workflows/prepare-release.yml @@ -13,7 +13,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.8] + python-version: [3.9] env: PYTHON_PACKAGE: data_pipelines_cli steps: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index d88932d..1abd470 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10"] steps: - uses: actions/checkout@v3 diff --git a/README.md b/README.md index 2f9a568..3693d58 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # data-pipelines-cli -[![Python Version](https://img.shields.io/badge/python-3.8%20%7C%203.9%20%7C%203.10-blue.svg)](https://github.com/getindata/data-pipelines-cli) +[![Python Version](https://img.shields.io/badge/python-3.9%20%7C%203.10-blue.svg)](https://github.com/getindata/data-pipelines-cli) [![PyPI Version](https://badge.fury.io/py/data-pipelines-cli.svg)](https://pypi.org/project/data-pipelines-cli/) [![Downloads](https://pepy.tech/badge/data-pipelines-cli)](https://pepy.tech/project/data-pipelines-cli) [![Maintainability](https://api.codeclimate.com/v1/badges/e44ed9383a42b59984f6/maintainability)](https://codeclimate.com/github/getindata/data-pipelines-cli/maintainability) diff --git a/data_pipelines_cli/cli_commands/generate/generate.py b/data_pipelines_cli/cli_commands/generate/generate.py index a35d764..c6dbcbf 100644 --- a/data_pipelines_cli/cli_commands/generate/generate.py +++ b/data_pipelines_cli/cli_commands/generate/generate.py @@ -1,9 +1,8 @@ import click - +import logging from .model_yaml import generate_model_yamls_command from .source_sql import generate_source_sqls_command from .source_yaml import generate_source_yamls_command -from .databricks_job import generate_databricks_job_command @click.group(name="generate", help="Generate additional dbt files") def generate_group() -> None: @@ -13,4 +12,9 @@ def generate_group() -> None: generate_group.add_command(generate_model_yamls_command) generate_group.add_command(generate_source_sqls_command) generate_group.add_command(generate_source_yamls_command) -generate_group.add_command(generate_databricks_job_command) + +try: + from .databricks_job import generate_databricks_job_command + generate_group.add_command(generate_databricks_job_command) +except ImportError: + logging.info("Databricks CLI not installed") diff --git a/docs/index.rst b/docs/index.rst index 920c27f..3f077aa 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,7 +1,7 @@ ``Data Pipelines CLI``: CLI for data platform ============================================== -.. image:: https://img.shields.io/badge/python-3.7%20%7C%203.8%20%7C%203.9-blue.svg +.. image:: https://img.shields.io/badge/python-3.9%20%7C%203.10-blue.svg :target: https://github.com/getindata/data-pipelines-cli :alt: Python Version diff --git a/setup.py b/setup.py index 09bcf06..8b0df1d 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,6 @@ "colorama==0.4.5", "dbt-core==1.5.4", "pydantic<2", - "dbt-databricks-factory>=0.1.1", ] EXTRA_FILESYSTEMS_REQUIRE = { @@ -32,12 +31,14 @@ "postgres": ["dbt-postgres==1.5.4"], "snowflake": ["dbt-snowflake==1.5.2"], "redshift": ["dbt-redshift==1.5.9"], + "databricks": ["dbt-databricks-factory>=0.1.1"], "dbt-all": [ "dbt-bigquery==1.5.5", "dbt-postgres==1.5.4", "dbt-snowflake==1.5.2", "dbt-redshift==1.5.9", - ], + "dbt-databricks-factory>=0.1.1", + ], # --- "docker": ["docker==6.0.1"], "datahub": ["acryl-datahub[dbt]==0.10.4"], @@ -76,10 +77,9 @@ long_description_content_type="text/markdown", license="Apache Software License (Apache 2.0)", license_files=("LICENSE",), - python_requires=">=3.8", + python_requires=">=3.9", classifiers=[ "Development Status :: 1 - Planning", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", ], diff --git a/tox.ini b/tox.ini index 75ba379..3d14329 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,9 @@ [tox] -envlist = py38, py39, py310 +envlist = py39, py310 [gh-actions] python = - 3.8: py38 3.9: py39 3.10: py310 From 7f63d96d99b6d9da836f4e9e3e198e3a56a5d793 Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 10:01:15 +0200 Subject: [PATCH 05/14] testing --- .pre-commit-config.yaml | 4 ++-- .../cli_commands/generate/databricks_job.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2570c97..89fcf4b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ repos: -- repo: https://github.com/pre-commit/mirrors-isort - rev: v5.10.1 +- repo: https://github.com/PyCQA/isort + rev: 5.12.0 hooks: - id: isort diff --git a/data_pipelines_cli/cli_commands/generate/databricks_job.py b/data_pipelines_cli/cli_commands/generate/databricks_job.py index 25a10a3..68e6277 100644 --- a/data_pipelines_cli/cli_commands/generate/databricks_job.py +++ b/data_pipelines_cli/cli_commands/generate/databricks_job.py @@ -12,7 +12,12 @@ @click.option("--project-dir", required=True, help="Path to dbt project directory.") @click.option("--profiles-dir", required=True, help="Path to dbt profiles directory.") @click.option("--cron-schedule", help="Cron schedule for the job.") -@click.option("--job-cluster", multiple=True, type=click.Tuple([str, str]), help="Job cluster config.") +@click.option( + "--job-cluster", + multiple=True, + type=click.Tuple([str, str]), + help="Job cluster config." +) @click.option( "--task-cluster", multiple=True, @@ -32,7 +37,11 @@ type=click.Choice([provider.value for provider in GitProvider]), ) @click.option("--pretty", is_flag=True, help="Pretty print the output.") -@click.option("--output-file", help="Output file path.", type=click.Path(file_okay=True, dir_okay=False, writable=True)) +@click.option( + "--output-file", + help="Output file path.", + type=click.Path(file_okay=True, dir_okay=False, writable=True) +) def generate_databricks_job_command( job_name: str, manifest_file: str, From f2692fadabecdb53ff40f05e5ea08b1ba838154f Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 10:01:37 +0200 Subject: [PATCH 06/14] linting --- .../cli_commands/generate/databricks_job.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/data_pipelines_cli/cli_commands/generate/databricks_job.py b/data_pipelines_cli/cli_commands/generate/databricks_job.py index 68e6277..775d1f3 100644 --- a/data_pipelines_cli/cli_commands/generate/databricks_job.py +++ b/data_pipelines_cli/cli_commands/generate/databricks_job.py @@ -1,8 +1,8 @@ import click - from dbt_databricks_factory.cli import create_job_cli from dbt_databricks_factory.config import GitProvider + @click.command("databricks-job", help="Generate a Databricks job") @click.argument( "manifest-file", @@ -13,10 +13,7 @@ @click.option("--profiles-dir", required=True, help="Path to dbt profiles directory.") @click.option("--cron-schedule", help="Cron schedule for the job.") @click.option( - "--job-cluster", - multiple=True, - type=click.Tuple([str, str]), - help="Job cluster config." + "--job-cluster", multiple=True, type=click.Tuple([str, str]), help="Job cluster config." ) @click.option( "--task-cluster", @@ -40,7 +37,7 @@ @click.option( "--output-file", help="Output file path.", - type=click.Path(file_okay=True, dir_okay=False, writable=True) + type=click.Path(file_okay=True, dir_okay=False, writable=True), ) def generate_databricks_job_command( job_name: str, From 60d4a591027a3094675f32785126dc09db73ed28 Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 10:06:17 +0200 Subject: [PATCH 07/14] annotations fix --- data_pipelines_cli/cli_commands/generate/databricks_job.py | 3 +++ data_pipelines_cli/cli_commands/generate/generate.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/data_pipelines_cli/cli_commands/generate/databricks_job.py b/data_pipelines_cli/cli_commands/generate/databricks_job.py index 775d1f3..8ac4f75 100644 --- a/data_pipelines_cli/cli_commands/generate/databricks_job.py +++ b/data_pipelines_cli/cli_commands/generate/databricks_job.py @@ -1,3 +1,5 @@ +"""Generate a Databricks job module.""" +from __future__ import annotations import click from dbt_databricks_factory.cli import create_job_cli from dbt_databricks_factory.config import GitProvider @@ -57,6 +59,7 @@ def generate_databricks_job_command( pretty: bool, output_file: str, ) -> None: + """Generate a Databricks job.""" create_job_cli( job_name, manifest_file, diff --git a/data_pipelines_cli/cli_commands/generate/generate.py b/data_pipelines_cli/cli_commands/generate/generate.py index c6dbcbf..bca0199 100644 --- a/data_pipelines_cli/cli_commands/generate/generate.py +++ b/data_pipelines_cli/cli_commands/generate/generate.py @@ -1,5 +1,5 @@ -import click import logging +import click from .model_yaml import generate_model_yamls_command from .source_sql import generate_source_sqls_command from .source_yaml import generate_source_yamls_command From 2fffc6ba913fc55cf334de463cd06cb963ee4085 Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 10:06:38 +0200 Subject: [PATCH 08/14] linting --- data_pipelines_cli/cli_commands/generate/databricks_job.py | 1 + data_pipelines_cli/cli_commands/generate/generate.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/data_pipelines_cli/cli_commands/generate/databricks_job.py b/data_pipelines_cli/cli_commands/generate/databricks_job.py index 8ac4f75..90f4857 100644 --- a/data_pipelines_cli/cli_commands/generate/databricks_job.py +++ b/data_pipelines_cli/cli_commands/generate/databricks_job.py @@ -1,5 +1,6 @@ """Generate a Databricks job module.""" from __future__ import annotations + import click from dbt_databricks_factory.cli import create_job_cli from dbt_databricks_factory.config import GitProvider diff --git a/data_pipelines_cli/cli_commands/generate/generate.py b/data_pipelines_cli/cli_commands/generate/generate.py index bca0199..4cf1359 100644 --- a/data_pipelines_cli/cli_commands/generate/generate.py +++ b/data_pipelines_cli/cli_commands/generate/generate.py @@ -1,9 +1,12 @@ import logging + import click + from .model_yaml import generate_model_yamls_command from .source_sql import generate_source_sqls_command from .source_yaml import generate_source_yamls_command + @click.group(name="generate", help="Generate additional dbt files") def generate_group() -> None: pass @@ -15,6 +18,7 @@ def generate_group() -> None: try: from .databricks_job import generate_databricks_job_command + generate_group.add_command(generate_databricks_job_command) except ImportError: logging.info("Databricks CLI not installed") From ca553443a028d517882006a11993d2cc571f4ac5 Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 10:28:25 +0200 Subject: [PATCH 09/14] no docs --- data_pipelines_cli/cli_commands/generate/databricks_job.py | 1 - 1 file changed, 1 deletion(-) diff --git a/data_pipelines_cli/cli_commands/generate/databricks_job.py b/data_pipelines_cli/cli_commands/generate/databricks_job.py index 90f4857..7d86294 100644 --- a/data_pipelines_cli/cli_commands/generate/databricks_job.py +++ b/data_pipelines_cli/cli_commands/generate/databricks_job.py @@ -1,4 +1,3 @@ -"""Generate a Databricks job module.""" from __future__ import annotations import click From 895565628afc5088ba340945bea9628e98b76db0 Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 10:36:00 +0200 Subject: [PATCH 10/14] linting --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 8b0df1d..0630007 100644 --- a/setup.py +++ b/setup.py @@ -38,7 +38,7 @@ "dbt-snowflake==1.5.2", "dbt-redshift==1.5.9", "dbt-databricks-factory>=0.1.1", - ], + ], # --- "docker": ["docker==6.0.1"], "datahub": ["acryl-datahub[dbt]==0.10.4"], From 78cadbf7edd0e17ccfb5cd691bd7000ee6ffa2c1 Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 11:57:09 +0200 Subject: [PATCH 11/14] update docs --- docs/source/data_pipelines_cli.cli_commands.generate.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/data_pipelines_cli.cli_commands.generate.rst b/docs/source/data_pipelines_cli.cli_commands.generate.rst index 9cbf127..9feb682 100644 --- a/docs/source/data_pipelines_cli.cli_commands.generate.rst +++ b/docs/source/data_pipelines_cli.cli_commands.generate.rst @@ -36,6 +36,14 @@ data\_pipelines\_cli.cli\_commands.generate.source\_sql module data\_pipelines\_cli.cli\_commands.generate.source\_yaml module --------------------------------------------------------------- +.. automodule:: data_pipelines_cli.cli_commands.generate.source_yaml + :members: + :undoc-members: + :show-inheritance: + +data\_pipelines\_cli.cli\_commands.generate.databricks\_job module +--------------------------------------------------------------- + .. automodule:: data_pipelines_cli.cli_commands.generate.source_yaml :members: :undoc-members: @@ -48,4 +56,3 @@ data\_pipelines\_cli.cli\_commands.generate.utils module :members: :undoc-members: :show-inheritance: - From e7eee80a8586d44555a1dc4163cbbd893f27e87e Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 12:48:27 +0200 Subject: [PATCH 12/14] testing --- tests/cli_commands/test_generate.py | 94 +++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/tests/cli_commands/test_generate.py b/tests/cli_commands/test_generate.py index 6451884..58685fe 100644 --- a/tests/cli_commands/test_generate.py +++ b/tests/cli_commands/test_generate.py @@ -323,3 +323,97 @@ def test_is_ephemeral_model(self): self.assertFalse(_is_ephemeral_model(example_dict, "c")) with self.assertRaises(DataPipelinesError): _is_ephemeral_model(example_dict, "d") + + +def test_generate_databricks_job() -> None: + runner = CliRunner() + result = runner.invoke( + _cli, + [ + "generate", + "databricks-job", + "--project-dir", + "/project/dir", + "--profiles-dir", + "/dbfs/profiles", + "--job-name", + "my-job", + "--library", + "my-library==1.0.0", + "--git-url", + "https://my-git.url.com", + "--git-provider", + "gitHub", + "--git-branch", + "my-branch", + "--default-task-cluster", + "my-cluster", + "--pretty", + str(GOLDENS_DIR_PATH / "target" / "manifest.json"), + ], + ) + expected = { + "name": "my-job", + "tasks": [ + { + "task_key": "model_my_new_project_my_first_dbt_model-run", + "dbt_task": { + "project_directory": "/project/dir", + "commands": [ + "dbt deps", + "dbt run --profiles-dir /dbfs/profiles --select my_first_dbt_model", + ], + }, + "libraries": [{"pypi": {"package": "my-library==1.0.0"}}], + "existing_cluster_id": "my-cluster", + }, + { + "task_key": "model_my_new_project_my_first_dbt_model-test", + "dbt_task": { + "project_directory": "/project/dir", + "commands": [ + "dbt deps", + "dbt test --profiles-dir /dbfs/profiles --select my_first_dbt_model", + ], + }, + "depends_on": [{"task_key": "model_my_new_project_my_first_dbt_model-run"}], + "libraries": [{"pypi": {"package": "my-library==1.0.0"}}], + "existing_cluster_id": "my-cluster", + }, + { + "task_key": "model_my_new_project_my_second_dbt_model-run", + "dbt_task": { + "project_directory": "/project/dir", + "commands": [ + "dbt deps", + "dbt run --profiles-dir /dbfs/profiles --select my_second_dbt_model", + ], + }, + "depends_on": [{"task_key": "model_my_new_project_my_first_dbt_model-test"}], + "libraries": [{"pypi": {"package": "my-library==1.0.0"}}], + "existing_cluster_id": "my-cluster", + }, + { + "task_key": "model_my_new_project_my_second_dbt_model-test", + "dbt_task": { + "project_directory": "/project/dir", + "commands": [ + "dbt deps", + "dbt test --profiles-dir /dbfs/profiles --select my_second_dbt_model", + ], + }, + "depends_on": [{"task_key": "model_my_new_project_my_second_dbt_model-run"}], + "libraries": [{"pypi": {"package": "my-library==1.0.0"}}], + "existing_cluster_id": "my-cluster", + }, + ], + "job_clusters": [], + "git_source": { + "git_url": "https://my-git.url.com", + "git_provider": "gitHub", + "git_branch": "my-branch", + }, + "format": "MULTI_TASK", + } + assert result.output == json.dumps(expected, indent=2) + "\n" + assert result.exit_code == 0 From ddc1b76bc7a31ac4515411bcc3518f8794b3019e Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 12:57:00 +0200 Subject: [PATCH 13/14] tests with databricks --- .github/workflows/python-package.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 1abd470..fad8bf7 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -32,7 +32,7 @@ jobs: - name: Check pre-commit status run: | - pip install .[tests] + pip install .[tests,databricks] pip freeze pipdeptree pre-commit run --all-files From 2172afe4bcdcaa509b9e4c7b454d54a8c9c704ea Mon Sep 17 00:00:00 2001 From: Piotr Tutak Date: Thu, 7 Sep 2023 13:07:06 +0200 Subject: [PATCH 14/14] testenv --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index 3d14329..f35d5c0 100644 --- a/tox.ini +++ b/tox.ini @@ -10,6 +10,7 @@ python = [testenv] extras = tests + databricks commands= python -m pytest --cov data_pipelines_cli --cov-report xml --cov-report term-missing --ignore=venv