From 3bbfd0180bec44a1d61830bc99945fbe4020c130 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 27 Aug 2025 10:59:08 +0200 Subject: [PATCH 1/3] Fixed bundle init not working on Standard tier --- .../templates/default-python/no-uc/input.json | 7 ++ .../default-python/no-uc/out.test.toml | 5 ++ .../templates/default-python/no-uc/output.txt | 30 +++++++ .../.vscode/__builtins__.pyi | 3 + .../my_default_python/.vscode/extensions.json | 7 ++ .../my_default_python/.vscode/settings.json | 16 ++++ .../no-uc/output/my_default_python/README.md | 51 +++++++++++ .../output/my_default_python/databricks.yml | 42 +++++++++ .../my_default_python/fixtures/.gitkeep | 22 +++++ .../output/my_default_python/out.gitignore | 8 ++ .../output/my_default_python/pyproject.toml | 41 +++++++++ .../resources/my_default_python.job.yml | 50 +++++++++++ .../resources/my_default_python.pipeline.yml | 14 +++ .../my_default_python/scratch/README.md | 4 + .../scratch/exploration.ipynb | 61 +++++++++++++ .../src/my_default_python/__init__.py | 0 .../src/my_default_python/main.py | 25 ++++++ .../my_default_python/src/notebook.ipynb | 75 ++++++++++++++++ .../my_default_python/src/pipeline.ipynb | 90 +++++++++++++++++++ .../my_default_python/tests/main_test.py | 6 ++ .../templates/default-python/no-uc/script | 9 ++ .../templates/default-python/no-uc/test.toml | 4 + libs/template/helpers.go | 2 +- 23 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 acceptance/bundle/templates/default-python/no-uc/input.json create mode 100644 acceptance/bundle/templates/default-python/no-uc/out.test.toml create mode 100644 acceptance/bundle/templates/default-python/no-uc/output.txt create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/__builtins__.pyi create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/extensions.json create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/settings.json create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/README.md create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/databricks.yml create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/fixtures/.gitkeep create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/out.gitignore create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/pyproject.toml create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.job.yml create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.pipeline.yml create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/README.md create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/exploration.ipynb create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/__init__.py create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/main.py create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/notebook.ipynb create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/pipeline.ipynb create mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/tests/main_test.py create mode 100644 acceptance/bundle/templates/default-python/no-uc/script create mode 100644 acceptance/bundle/templates/default-python/no-uc/test.toml diff --git a/acceptance/bundle/templates/default-python/no-uc/input.json b/acceptance/bundle/templates/default-python/no-uc/input.json new file mode 100644 index 0000000000..2c4416c00c --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/input.json @@ -0,0 +1,7 @@ +{ + "project_name": "my_default_python", + "include_notebook": "yes", + "include_dlt": "yes", + "include_python": "yes", + "serverless": "no" +} diff --git a/acceptance/bundle/templates/default-python/no-uc/out.test.toml b/acceptance/bundle/templates/default-python/no-uc/out.test.toml new file mode 100644 index 0000000000..8f3575be7b --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_CLI_DEPLOYMENT = ["terraform", "direct-exp"] diff --git a/acceptance/bundle/templates/default-python/no-uc/output.txt b/acceptance/bundle/templates/default-python/no-uc/output.txt new file mode 100644 index 0000000000..930e756de7 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output.txt @@ -0,0 +1,30 @@ + +>>> [CLI] bundle init default-python --config-file ./input.json --output-dir output + +Welcome to the default Python template for Databricks Asset Bundles! +Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [DATABRICKS_URL] + +✨ Your new project has been created in the 'my_default_python' directory! + +Please refer to the README.md file for "getting started" instructions. +See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. + +>>> [CLI] bundle validate -t dev +Name: my_default_python +Target: dev +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev + +Validation OK! + +>>> [CLI] bundle validate -t prod +Name: my_default_python +Target: prod +Workspace: + Host: [DATABRICKS_URL] + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod + +Validation OK! diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/__builtins__.pyi b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/__builtins__.pyi new file mode 100644 index 0000000000..0edd5181bc --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/__builtins__.pyi @@ -0,0 +1,3 @@ +# Typings for Pylance in Visual Studio Code +# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md +from databricks.sdk.runtime import * diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/extensions.json new file mode 100644 index 0000000000..5d15eba363 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/extensions.json @@ -0,0 +1,7 @@ +{ + "recommendations": [ + "databricks.databricks", + "ms-python.vscode-pylance", + "redhat.vscode-yaml" + ] +} diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/settings.json b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/settings.json new file mode 100644 index 0000000000..8ee87c30d4 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/settings.json @@ -0,0 +1,16 @@ +{ + "python.analysis.stubPath": ".vscode", + "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", + "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", + "python.testing.pytestArgs": [ + "." + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true, + "python.analysis.extraPaths": ["src"], + "files.exclude": { + "**/*.egg-info": true, + "**/__pycache__": true, + ".pytest_cache": true, + }, +} diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/README.md new file mode 100644 index 0000000000..f3b5d153b2 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/README.md @@ -0,0 +1,51 @@ +# my_default_python + +The 'my_default_python' project was generated by using the default-python template. + +## Getting started + +0. Install UV: https://docs.astral.sh/uv/getting-started/installation/ + +1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/databricks-cli.html + +2. Authenticate to your Databricks workspace, if you have not done so already: + ``` + $ databricks configure + ``` + +3. To deploy a development copy of this project, type: + ``` + $ databricks bundle deploy --target dev + ``` + (Note that "dev" is the default target, so the `--target` parameter + is optional here.) + + This deploys everything that's defined for this project. + For example, the default template would deploy a job called + `[dev yourname] my_default_python_job` to your workspace. + You can find that job by opening your workpace and clicking on **Workflows**. + +4. Similarly, to deploy a production copy, type: + ``` + $ databricks bundle deploy --target prod + ``` + + Note that the default job from the template has a schedule that runs every day + (defined in resources/my_default_python.job.yml). The schedule + is paused when deploying in development mode (see + https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). + +5. To run a job or pipeline, use the "run" command: + ``` + $ databricks bundle run + ``` +6. Optionally, install the Databricks extension for Visual Studio code for local development from + https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your + virtual environment and setup Databricks Connect for running unit tests locally. + When not using these tools, consult your development environment's documentation + and/or the documentation for Databricks Connect for manually setting up your environment + (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html). + +7. For documentation on the Databricks asset bundles format used + for this project, and for CI/CD configuration, see + https://docs.databricks.com/dev-tools/bundles/index.html. diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/databricks.yml new file mode 100644 index 0000000000..ed3d53b999 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/databricks.yml @@ -0,0 +1,42 @@ +# This is a Databricks asset bundle definition for my_default_python. +# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. +bundle: + name: my_default_python + uuid: [UUID] + +artifacts: + python_artifact: + type: whl + build: uv build --wheel + +include: + - resources/*.yml + - resources/*/*.yml + +targets: + dev: + # The default target uses 'mode: development' to create a development copy. + # - Deployed resources get prefixed with '[dev my_user_name]' + # - Any job schedules and triggers are paused by default. + # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. + mode: development + default: true + workspace: + host: [DATABRICKS_URL] + + presets: + # Set dynamic_version: true on all artifacts of type "whl". + # This makes "bundle deploy" add a timestamp to wheel's version before uploading, + # new wheel takes over the previous installation even if actual wheel version is unchanged. + # See https://docs.databricks.com/aws/en/dev-tools/bundles/settings + artifacts_dynamic_version: true + + prod: + mode: production + workspace: + host: [DATABRICKS_URL] + # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy. + root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} + permissions: + - user_name: [USERNAME] + level: CAN_MANAGE diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/fixtures/.gitkeep b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/fixtures/.gitkeep new file mode 100644 index 0000000000..fa25d2745e --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/fixtures/.gitkeep @@ -0,0 +1,22 @@ +# Fixtures + +This folder is reserved for fixtures, such as CSV files. + +Below is an example of how to load fixtures as a data frame: + +``` +import pandas as pd +import os + +def get_absolute_path(*relative_parts): + if 'dbutils' in globals(): + base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore + path = os.path.normpath(os.path.join(base_dir, *relative_parts)) + return path if path.startswith("/Workspace") else "/Workspace" + path + else: + return os.path.join(*relative_parts) + +csv_file = get_absolute_path("..", "fixtures", "mycsv.csv") +df = pd.read_csv(csv_file) +display(df) +``` diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/out.gitignore b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/out.gitignore new file mode 100644 index 0000000000..0dab7f4995 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/out.gitignore @@ -0,0 +1,8 @@ +.databricks/ +build/ +dist/ +__pycache__/ +*.egg-info +.venv/ +scratch/** +!scratch/README.md diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/pyproject.toml new file mode 100644 index 0000000000..5049f8a3ea --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/pyproject.toml @@ -0,0 +1,41 @@ +[project] +name = "my_default_python" +version = "0.0.1" +authors = [{ name = "[USERNAME]" }] +requires-python = ">= 3.11" + +[project.optional-dependencies] +dev = [ + "pytest", + + # Code completion support for Lakeflow Declarative Pipelines, also install databricks-connect + "databricks-dlt", + + # databricks-connect can be used to run parts of this project locally. + # See https://docs.databricks.com/dev-tools/databricks-connect.html. + # + # Note, databricks-connect is automatically installed if you're using Databricks + # extension for Visual Studio Code + # (https://docs.databricks.com/dev-tools/vscode-ext/dev-tasks/databricks-connect.html). + # + # To manually install databricks-connect, uncomment the line below to install a version + # of db-connect that corresponds to the Databricks Runtime version used for this project. + # See https://docs.databricks.com/dev-tools/databricks-connect.html + # "databricks-connect>=15.4,<15.5", +] + +[tool.pytest.ini_options] +pythonpath = "src" +testpaths = [ + "tests", +] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/my_default_python"] + +[project.scripts] +main = "my_default_python.main:main" diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.job.yml new file mode 100644 index 0000000000..30b579f500 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.job.yml @@ -0,0 +1,50 @@ +# The main job for my_default_python. +resources: + jobs: + my_default_python_job: + name: my_default_python_job + + trigger: + # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger + periodic: + interval: 1 + unit: DAYS + + #email_notifications: + # on_failure: + # - your_email@example.com + + tasks: + - task_key: notebook_task + job_cluster_key: job_cluster + notebook_task: + notebook_path: ../src/notebook.ipynb + + - task_key: refresh_pipeline + depends_on: + - task_key: notebook_task + pipeline_task: + pipeline_id: ${resources.pipelines.my_default_python_pipeline.id} + + - task_key: main_task + depends_on: + - task_key: refresh_pipeline + job_cluster_key: job_cluster + python_wheel_task: + package_name: my_default_python + entry_point: main + libraries: + # By default we just include the .whl file generated for the my_default_python package. + # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html + # for more information on how to add other libraries. + - whl: ../dist/*.whl + + job_clusters: + - job_cluster_key: job_cluster + new_cluster: + spark_version: 15.4.x-scala2.12 + node_type_id: [NODE_TYPE_ID] + data_security_mode: SINGLE_USER + autoscale: + min_workers: 1 + max_workers: 4 diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.pipeline.yml new file mode 100644 index 0000000000..6e49947083 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.pipeline.yml @@ -0,0 +1,14 @@ +# The main pipeline for my_default_python +resources: + pipelines: + my_default_python_pipeline: + name: my_default_python_pipeline + ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: + # catalog: catalog_name + schema: my_default_python_${bundle.target} + libraries: + - notebook: + path: ../src/pipeline.ipynb + + configuration: + bundle.sourcePath: ${workspace.file_path}/src diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/README.md b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/README.md new file mode 100644 index 0000000000..e6cfb81b46 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/README.md @@ -0,0 +1,4 @@ +# scratch + +This folder is reserved for personal, exploratory notebooks. +By default these are not committed to Git, as 'scratch' is listed in .gitignore. diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/exploration.ipynb b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/exploration.ipynb new file mode 100644 index 0000000000..a12773d4e8 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/exploration.ipynb @@ -0,0 +1,61 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "sys.path.append(\"../src\")\n", + "from my_default_python import main\n", + "\n", + "main.get_taxis(spark).show(10)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2 + }, + "notebookName": "ipynb-notebook", + "widgets": {} + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/__init__.py b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/main.py b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/main.py new file mode 100644 index 0000000000..5ae344c7e2 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/main.py @@ -0,0 +1,25 @@ +from pyspark.sql import SparkSession, DataFrame + + +def get_taxis(spark: SparkSession) -> DataFrame: + return spark.read.table("samples.nyctaxi.trips") + + +# Create a new Databricks Connect session. If this fails, +# check that you have configured Databricks Connect correctly. +# See https://docs.databricks.com/dev-tools/databricks-connect.html. +def get_spark() -> SparkSession: + try: + from databricks.connect import DatabricksSession + + return DatabricksSession.builder.getOrCreate() + except ImportError: + return SparkSession.builder.getOrCreate() + + +def main(): + get_taxis(get_spark()).show(5) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/notebook.ipynb new file mode 100644 index 0000000000..472ccb2190 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/notebook.ipynb @@ -0,0 +1,75 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "source": [ + "# Default notebook\n", + "\n", + "This default notebook is executed using Databricks Workflows as defined in resources/my_default_python.job.yml." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": { + "byteLimit": 2048000, + "rowLimit": 10000 + }, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "from my_default_python import main\n", + "\n", + "main.get_taxis(spark).show(10)" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2 + }, + "notebookName": "notebook", + "widgets": {} + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/pipeline.ipynb b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/pipeline.ipynb new file mode 100644 index 0000000000..8845c185df --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/pipeline.ipynb @@ -0,0 +1,90 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "source": [ + "# Lakeflow Declarative Pipeline\n", + "\n", + "This Lakeflow Declarative Pipeline (LDP) definition is executed using a pipeline defined in resources/my_default_python.pipeline.yml." + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "# Import DLT and src/my_default_python\n", + "import dlt\n", + "import sys\n", + "\n", + "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n", + "from pyspark.sql.functions import expr\n", + "from my_default_python import main" + ] + }, + { + "cell_type": "code", + "execution_count": 0, + "metadata": { + "application/vnd.databricks.v1+cell": { + "cellMetadata": {}, + "inputWidgets": {}, + "nuid": "[UUID]", + "showTitle": false, + "title": "" + } + }, + "outputs": [], + "source": [ + "@dlt.view\n", + "def taxi_raw():\n", + " return main.get_taxis(spark)\n", + "\n", + "\n", + "@dlt.table\n", + "def filtered_taxis():\n", + " return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))" + ] + } + ], + "metadata": { + "application/vnd.databricks.v1+notebook": { + "dashboards": [], + "language": "python", + "notebookMetadata": { + "pythonIndentUnit": 2 + }, + "notebookName": "pipeline", + "widgets": {} + }, + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.4" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/tests/main_test.py b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/tests/main_test.py new file mode 100644 index 0000000000..dc449154a6 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/tests/main_test.py @@ -0,0 +1,6 @@ +from my_default_python.main import get_taxis, get_spark + + +def test_main(): + taxis = get_taxis(get_spark()) + assert taxis.count() > 5 diff --git a/acceptance/bundle/templates/default-python/no-uc/script b/acceptance/bundle/templates/default-python/no-uc/script new file mode 100644 index 0000000000..d7b047fec8 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/script @@ -0,0 +1,9 @@ +trace $CLI bundle init default-python --config-file ./input.json --output-dir output + +cd output/my_default_python +trace $CLI bundle validate -t dev +trace $CLI bundle validate -t prod + +# Do not affect this repository's git behaviour #2318 +mv .gitignore out.gitignore +rm .databricks/.gitignore diff --git a/acceptance/bundle/templates/default-python/no-uc/test.toml b/acceptance/bundle/templates/default-python/no-uc/test.toml new file mode 100644 index 0000000000..3f795d5c48 --- /dev/null +++ b/acceptance/bundle/templates/default-python/no-uc/test.toml @@ -0,0 +1,4 @@ +[[Server]] +Pattern = "GET /api/2.1/unity-catalog/current-metastore-assignment" +Response.StatusCode = 404 +Response.Body = "{\"error_code\": \"FEATURE_DISABLED\", \"message\": \"Unity Catalog is not available for feature tier STANDARD_TIER.\"}" diff --git a/libs/template/helpers.go b/libs/template/helpers.go index c258775a43..dde372b4da 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -141,7 +141,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { metastore, err := w.Metastores.Current(ctx) if err != nil { var aerr *apierr.APIError - if errors.As(err, &aerr) && (aerr.ErrorCode == "PERMISSION_DENIED" || aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST") { + if errors.As(err, &aerr) && (aerr.ErrorCode == "PERMISSION_DENIED" || aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST" || aerr.ErrorCode == "FEATURE_DISABLED") { // Ignore: access denied or workspace doesn't have a metastore assigned empty_default := "" cachedCatalog = &empty_default From 75aa1e919f92b57a00883d8af174989acf312243 Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 27 Aug 2025 11:31:57 +0200 Subject: [PATCH 2/3] remove output --- .../templates/default-python/no-uc/output.txt | 20 ----- .../.vscode/__builtins__.pyi | 3 - .../my_default_python/.vscode/extensions.json | 7 -- .../my_default_python/.vscode/settings.json | 16 ---- .../no-uc/output/my_default_python/README.md | 51 ----------- .../output/my_default_python/databricks.yml | 42 --------- .../my_default_python/fixtures/.gitkeep | 22 ----- .../output/my_default_python/out.gitignore | 8 -- .../output/my_default_python/pyproject.toml | 41 --------- .../resources/my_default_python.job.yml | 50 ----------- .../resources/my_default_python.pipeline.yml | 14 --- .../my_default_python/scratch/README.md | 4 - .../scratch/exploration.ipynb | 61 ------------- .../src/my_default_python/__init__.py | 0 .../src/my_default_python/main.py | 25 ------ .../my_default_python/src/notebook.ipynb | 75 ---------------- .../my_default_python/src/pipeline.ipynb | 90 ------------------- .../my_default_python/tests/main_test.py | 6 -- .../templates/default-python/no-uc/script | 8 +- libs/template/helpers.go | 9 +- 20 files changed, 9 insertions(+), 543 deletions(-) delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/__builtins__.pyi delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/extensions.json delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/settings.json delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/README.md delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/databricks.yml delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/fixtures/.gitkeep delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/out.gitignore delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/pyproject.toml delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.job.yml delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.pipeline.yml delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/README.md delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/exploration.ipynb delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/__init__.py delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/main.py delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/notebook.ipynb delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/pipeline.ipynb delete mode 100644 acceptance/bundle/templates/default-python/no-uc/output/my_default_python/tests/main_test.py diff --git a/acceptance/bundle/templates/default-python/no-uc/output.txt b/acceptance/bundle/templates/default-python/no-uc/output.txt index 930e756de7..6abf52cf09 100644 --- a/acceptance/bundle/templates/default-python/no-uc/output.txt +++ b/acceptance/bundle/templates/default-python/no-uc/output.txt @@ -8,23 +8,3 @@ Workspace to use (auto-detected, edit in 'my_default_python/databricks.yml'): [D Please refer to the README.md file for "getting started" instructions. See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. - ->>> [CLI] bundle validate -t dev -Name: my_default_python -Target: dev -Workspace: - Host: [DATABRICKS_URL] - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/dev - -Validation OK! - ->>> [CLI] bundle validate -t prod -Name: my_default_python -Target: prod -Workspace: - Host: [DATABRICKS_URL] - User: [USERNAME] - Path: /Workspace/Users/[USERNAME]/.bundle/my_default_python/prod - -Validation OK! diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/__builtins__.pyi b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/__builtins__.pyi deleted file mode 100644 index 0edd5181bc..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/__builtins__.pyi +++ /dev/null @@ -1,3 +0,0 @@ -# Typings for Pylance in Visual Studio Code -# see https://github.com/microsoft/pyright/blob/main/docs/builtins.md -from databricks.sdk.runtime import * diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/extensions.json b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/extensions.json deleted file mode 100644 index 5d15eba363..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/extensions.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "recommendations": [ - "databricks.databricks", - "ms-python.vscode-pylance", - "redhat.vscode-yaml" - ] -} diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/settings.json b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/settings.json deleted file mode 100644 index 8ee87c30d4..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/.vscode/settings.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "python.analysis.stubPath": ".vscode", - "jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])", - "jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------", - "python.testing.pytestArgs": [ - "." - ], - "python.testing.unittestEnabled": false, - "python.testing.pytestEnabled": true, - "python.analysis.extraPaths": ["src"], - "files.exclude": { - "**/*.egg-info": true, - "**/__pycache__": true, - ".pytest_cache": true, - }, -} diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/README.md b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/README.md deleted file mode 100644 index f3b5d153b2..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/README.md +++ /dev/null @@ -1,51 +0,0 @@ -# my_default_python - -The 'my_default_python' project was generated by using the default-python template. - -## Getting started - -0. Install UV: https://docs.astral.sh/uv/getting-started/installation/ - -1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/databricks-cli.html - -2. Authenticate to your Databricks workspace, if you have not done so already: - ``` - $ databricks configure - ``` - -3. To deploy a development copy of this project, type: - ``` - $ databricks bundle deploy --target dev - ``` - (Note that "dev" is the default target, so the `--target` parameter - is optional here.) - - This deploys everything that's defined for this project. - For example, the default template would deploy a job called - `[dev yourname] my_default_python_job` to your workspace. - You can find that job by opening your workpace and clicking on **Workflows**. - -4. Similarly, to deploy a production copy, type: - ``` - $ databricks bundle deploy --target prod - ``` - - Note that the default job from the template has a schedule that runs every day - (defined in resources/my_default_python.job.yml). The schedule - is paused when deploying in development mode (see - https://docs.databricks.com/dev-tools/bundles/deployment-modes.html). - -5. To run a job or pipeline, use the "run" command: - ``` - $ databricks bundle run - ``` -6. Optionally, install the Databricks extension for Visual Studio code for local development from - https://docs.databricks.com/dev-tools/vscode-ext.html. It can configure your - virtual environment and setup Databricks Connect for running unit tests locally. - When not using these tools, consult your development environment's documentation - and/or the documentation for Databricks Connect for manually setting up your environment - (https://docs.databricks.com/en/dev-tools/databricks-connect/python/index.html). - -7. For documentation on the Databricks asset bundles format used - for this project, and for CI/CD configuration, see - https://docs.databricks.com/dev-tools/bundles/index.html. diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/databricks.yml b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/databricks.yml deleted file mode 100644 index ed3d53b999..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/databricks.yml +++ /dev/null @@ -1,42 +0,0 @@ -# This is a Databricks asset bundle definition for my_default_python. -# See https://docs.databricks.com/dev-tools/bundles/index.html for documentation. -bundle: - name: my_default_python - uuid: [UUID] - -artifacts: - python_artifact: - type: whl - build: uv build --wheel - -include: - - resources/*.yml - - resources/*/*.yml - -targets: - dev: - # The default target uses 'mode: development' to create a development copy. - # - Deployed resources get prefixed with '[dev my_user_name]' - # - Any job schedules and triggers are paused by default. - # See also https://docs.databricks.com/dev-tools/bundles/deployment-modes.html. - mode: development - default: true - workspace: - host: [DATABRICKS_URL] - - presets: - # Set dynamic_version: true on all artifacts of type "whl". - # This makes "bundle deploy" add a timestamp to wheel's version before uploading, - # new wheel takes over the previous installation even if actual wheel version is unchanged. - # See https://docs.databricks.com/aws/en/dev-tools/bundles/settings - artifacts_dynamic_version: true - - prod: - mode: production - workspace: - host: [DATABRICKS_URL] - # We explicitly deploy to /Workspace/Users/[USERNAME] to make sure we only have a single copy. - root_path: /Workspace/Users/[USERNAME]/.bundle/${bundle.name}/${bundle.target} - permissions: - - user_name: [USERNAME] - level: CAN_MANAGE diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/fixtures/.gitkeep b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/fixtures/.gitkeep deleted file mode 100644 index fa25d2745e..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/fixtures/.gitkeep +++ /dev/null @@ -1,22 +0,0 @@ -# Fixtures - -This folder is reserved for fixtures, such as CSV files. - -Below is an example of how to load fixtures as a data frame: - -``` -import pandas as pd -import os - -def get_absolute_path(*relative_parts): - if 'dbutils' in globals(): - base_dir = os.path.dirname(dbutils.notebook.entry_point.getDbutils().notebook().getContext().notebookPath().get()) # type: ignore - path = os.path.normpath(os.path.join(base_dir, *relative_parts)) - return path if path.startswith("/Workspace") else "/Workspace" + path - else: - return os.path.join(*relative_parts) - -csv_file = get_absolute_path("..", "fixtures", "mycsv.csv") -df = pd.read_csv(csv_file) -display(df) -``` diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/out.gitignore b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/out.gitignore deleted file mode 100644 index 0dab7f4995..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/out.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -.databricks/ -build/ -dist/ -__pycache__/ -*.egg-info -.venv/ -scratch/** -!scratch/README.md diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/pyproject.toml b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/pyproject.toml deleted file mode 100644 index 5049f8a3ea..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/pyproject.toml +++ /dev/null @@ -1,41 +0,0 @@ -[project] -name = "my_default_python" -version = "0.0.1" -authors = [{ name = "[USERNAME]" }] -requires-python = ">= 3.11" - -[project.optional-dependencies] -dev = [ - "pytest", - - # Code completion support for Lakeflow Declarative Pipelines, also install databricks-connect - "databricks-dlt", - - # databricks-connect can be used to run parts of this project locally. - # See https://docs.databricks.com/dev-tools/databricks-connect.html. - # - # Note, databricks-connect is automatically installed if you're using Databricks - # extension for Visual Studio Code - # (https://docs.databricks.com/dev-tools/vscode-ext/dev-tasks/databricks-connect.html). - # - # To manually install databricks-connect, uncomment the line below to install a version - # of db-connect that corresponds to the Databricks Runtime version used for this project. - # See https://docs.databricks.com/dev-tools/databricks-connect.html - # "databricks-connect>=15.4,<15.5", -] - -[tool.pytest.ini_options] -pythonpath = "src" -testpaths = [ - "tests", -] - -[build-system] -requires = ["hatchling"] -build-backend = "hatchling.build" - -[tool.hatch.build.targets.wheel] -packages = ["src/my_default_python"] - -[project.scripts] -main = "my_default_python.main:main" diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.job.yml b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.job.yml deleted file mode 100644 index 30b579f500..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.job.yml +++ /dev/null @@ -1,50 +0,0 @@ -# The main job for my_default_python. -resources: - jobs: - my_default_python_job: - name: my_default_python_job - - trigger: - # Run this job every day, exactly one day from the last run; see https://docs.databricks.com/api/workspace/jobs/create#trigger - periodic: - interval: 1 - unit: DAYS - - #email_notifications: - # on_failure: - # - your_email@example.com - - tasks: - - task_key: notebook_task - job_cluster_key: job_cluster - notebook_task: - notebook_path: ../src/notebook.ipynb - - - task_key: refresh_pipeline - depends_on: - - task_key: notebook_task - pipeline_task: - pipeline_id: ${resources.pipelines.my_default_python_pipeline.id} - - - task_key: main_task - depends_on: - - task_key: refresh_pipeline - job_cluster_key: job_cluster - python_wheel_task: - package_name: my_default_python - entry_point: main - libraries: - # By default we just include the .whl file generated for the my_default_python package. - # See https://docs.databricks.com/dev-tools/bundles/library-dependencies.html - # for more information on how to add other libraries. - - whl: ../dist/*.whl - - job_clusters: - - job_cluster_key: job_cluster - new_cluster: - spark_version: 15.4.x-scala2.12 - node_type_id: [NODE_TYPE_ID] - data_security_mode: SINGLE_USER - autoscale: - min_workers: 1 - max_workers: 4 diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.pipeline.yml b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.pipeline.yml deleted file mode 100644 index 6e49947083..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/resources/my_default_python.pipeline.yml +++ /dev/null @@ -1,14 +0,0 @@ -# The main pipeline for my_default_python -resources: - pipelines: - my_default_python_pipeline: - name: my_default_python_pipeline - ## Specify the 'catalog' field to configure this pipeline to make use of Unity Catalog: - # catalog: catalog_name - schema: my_default_python_${bundle.target} - libraries: - - notebook: - path: ../src/pipeline.ipynb - - configuration: - bundle.sourcePath: ${workspace.file_path}/src diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/README.md b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/README.md deleted file mode 100644 index e6cfb81b46..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/README.md +++ /dev/null @@ -1,4 +0,0 @@ -# scratch - -This folder is reserved for personal, exploratory notebooks. -By default these are not committed to Git, as 'scratch' is listed in .gitignore. diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/exploration.ipynb b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/exploration.ipynb deleted file mode 100644 index a12773d4e8..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/scratch/exploration.ipynb +++ /dev/null @@ -1,61 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "import sys\n", - "\n", - "sys.path.append(\"../src\")\n", - "from my_default_python import main\n", - "\n", - "main.get_taxis(spark).show(10)" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "ipynb-notebook", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/__init__.py b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/main.py b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/main.py deleted file mode 100644 index 5ae344c7e2..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/my_default_python/main.py +++ /dev/null @@ -1,25 +0,0 @@ -from pyspark.sql import SparkSession, DataFrame - - -def get_taxis(spark: SparkSession) -> DataFrame: - return spark.read.table("samples.nyctaxi.trips") - - -# Create a new Databricks Connect session. If this fails, -# check that you have configured Databricks Connect correctly. -# See https://docs.databricks.com/dev-tools/databricks-connect.html. -def get_spark() -> SparkSession: - try: - from databricks.connect import DatabricksSession - - return DatabricksSession.builder.getOrCreate() - except ImportError: - return SparkSession.builder.getOrCreate() - - -def main(): - get_taxis(get_spark()).show(5) - - -if __name__ == "__main__": - main() diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/notebook.ipynb b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/notebook.ipynb deleted file mode 100644 index 472ccb2190..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/notebook.ipynb +++ /dev/null @@ -1,75 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# Default notebook\n", - "\n", - "This default notebook is executed using Databricks Workflows as defined in resources/my_default_python.job.yml." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "%load_ext autoreload\n", - "%autoreload 2" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": { - "byteLimit": 2048000, - "rowLimit": 10000 - }, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "from my_default_python import main\n", - "\n", - "main.get_taxis(spark).show(10)" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "notebook", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/pipeline.ipynb b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/pipeline.ipynb deleted file mode 100644 index 8845c185df..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/src/pipeline.ipynb +++ /dev/null @@ -1,90 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "source": [ - "# Lakeflow Declarative Pipeline\n", - "\n", - "This Lakeflow Declarative Pipeline (LDP) definition is executed using a pipeline defined in resources/my_default_python.pipeline.yml." - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "# Import DLT and src/my_default_python\n", - "import dlt\n", - "import sys\n", - "\n", - "sys.path.append(spark.conf.get(\"bundle.sourcePath\", \".\"))\n", - "from pyspark.sql.functions import expr\n", - "from my_default_python import main" - ] - }, - { - "cell_type": "code", - "execution_count": 0, - "metadata": { - "application/vnd.databricks.v1+cell": { - "cellMetadata": {}, - "inputWidgets": {}, - "nuid": "[UUID]", - "showTitle": false, - "title": "" - } - }, - "outputs": [], - "source": [ - "@dlt.view\n", - "def taxi_raw():\n", - " return main.get_taxis(spark)\n", - "\n", - "\n", - "@dlt.table\n", - "def filtered_taxis():\n", - " return dlt.read(\"taxi_raw\").filter(expr(\"fare_amount < 30\"))" - ] - } - ], - "metadata": { - "application/vnd.databricks.v1+notebook": { - "dashboards": [], - "language": "python", - "notebookMetadata": { - "pythonIndentUnit": 2 - }, - "notebookName": "pipeline", - "widgets": {} - }, - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.4" - } - }, - "nbformat": 4, - "nbformat_minor": 0 -} diff --git a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/tests/main_test.py b/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/tests/main_test.py deleted file mode 100644 index dc449154a6..0000000000 --- a/acceptance/bundle/templates/default-python/no-uc/output/my_default_python/tests/main_test.py +++ /dev/null @@ -1,6 +0,0 @@ -from my_default_python.main import get_taxis, get_spark - - -def test_main(): - taxis = get_taxis(get_spark()) - assert taxis.count() > 5 diff --git a/acceptance/bundle/templates/default-python/no-uc/script b/acceptance/bundle/templates/default-python/no-uc/script index d7b047fec8..d6195ee17b 100644 --- a/acceptance/bundle/templates/default-python/no-uc/script +++ b/acceptance/bundle/templates/default-python/no-uc/script @@ -1,9 +1,3 @@ trace $CLI bundle init default-python --config-file ./input.json --output-dir output -cd output/my_default_python -trace $CLI bundle validate -t dev -trace $CLI bundle validate -t prod - -# Do not affect this repository's git behaviour #2318 -mv .gitignore out.gitignore -rm .databricks/.gitignore +rm -rf output diff --git a/libs/template/helpers.go b/libs/template/helpers.go index dde372b4da..27bb0d0432 100644 --- a/libs/template/helpers.go +++ b/libs/template/helpers.go @@ -8,6 +8,7 @@ import ( "net/url" "os" "regexp" + "slices" "strings" "text/template" @@ -38,6 +39,12 @@ var ( cachedCatalog *string ) +var metastoreDisabledErrorCodes = []string{ + "PERMISSION_DENIED", + "METASTORE_DOES_NOT_EXIST", // Default metastore is not assigned to the workspace. + "FEATURE_DISABLED", // Unity Catalog is not available for feature tier STANDARD_TIER. +} + // UUID that is stable for the duration of the template execution. This can be used // to populate the `bundle.uuid` field in databricks.yml by template authors. // @@ -141,7 +148,7 @@ func loadHelpers(ctx context.Context) template.FuncMap { metastore, err := w.Metastores.Current(ctx) if err != nil { var aerr *apierr.APIError - if errors.As(err, &aerr) && (aerr.ErrorCode == "PERMISSION_DENIED" || aerr.ErrorCode == "METASTORE_DOES_NOT_EXIST" || aerr.ErrorCode == "FEATURE_DISABLED") { + if errors.As(err, &aerr) && slices.Contains(metastoreDisabledErrorCodes, aerr.ErrorCode) { // Ignore: access denied or workspace doesn't have a metastore assigned empty_default := "" cachedCatalog = &empty_default From 5255744fe395229d4f463e4842cf1a9aad3459bc Mon Sep 17 00:00:00 2001 From: Andrew Nester Date: Wed, 27 Aug 2025 11:34:29 +0200 Subject: [PATCH 3/3] added changelog --- NEXT_CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index d6b5dde4f9..27c922279d 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -18,5 +18,6 @@ See more details here: ([#3225](https://github.com/databricks/cli/pull/3225)) * [Breaking Change] Remove deprecated path fallback mechanism for jobs and pipelines ([#3225](https://github.com/databricks/cli/pull/3225)) * Add support for Lakebase synced database tables in DABs ([#3467](https://github.com/databricks/cli/pull/3467)) * Rename Delta Live Tables to Lakeflow Declarative Pipelines in the default-python template ([#3476](https://github.com/databricks/cli/pull/3476)). +* Fixed bundle init not working on Standard tier ([#3496](https://github.com/databricks/cli/pull/3496)) ### API Changes