From d06f8c5c225a4f02904cf811059d1b4b977f2a72 Mon Sep 17 00:00:00 2001 From: Gleb Kanterov Date: Wed, 22 Oct 2025 19:30:38 +0200 Subject: [PATCH] Add pydabs template --- .../pydabs/check-consistency/output.txt | 4 +- .../templates/pydabs/check-consistency/script | 2 +- .../pydabs/check-formatting/input.json.tmpl | 1 - .../pydabs/check-formatting/output.txt | 2 +- .../templates/pydabs/check-formatting/script | 2 +- .../pydabs/deploy-classic/input.json | 3 +- .../pydabs/deploy-classic/output.txt | 2 +- .../templates/pydabs/deploy-classic/script | 2 +- .../templates/pydabs/init-classic/input.json | 3 +- .../templates/pydabs/init-classic/output.txt | 2 +- .../templates/pydabs/init-classic/script | 2 +- libs/template/reader.go | 27 +++++++++++++ libs/template/reader_test.go | 38 +++++++++++++++++++ libs/template/template.go | 18 ++++++--- libs/template/template_test.go | 2 +- 15 files changed, 90 insertions(+), 20 deletions(-) diff --git a/acceptance/bundle/templates/pydabs/check-consistency/output.txt b/acceptance/bundle/templates/pydabs/check-consistency/output.txt index 300891a691..5f2ee17e58 100644 --- a/acceptance/bundle/templates/pydabs/check-consistency/output.txt +++ b/acceptance/bundle/templates/pydabs/check-consistency/output.txt @@ -1,6 +1,6 @@ == Collecting resources into ./enable_pydabs=yes ->>> [CLI] bundle init default-python --config-file ./input.json +>>> [CLI] bundle init pydabs --config-file ./input.json Welcome to the default Python template for Databricks Asset Bundles! Workspace to use (auto-detected, edit in 'my_pydabs/databricks.yml'): [DATABRICKS_URL] @@ -17,7 +17,7 @@ See also the documentation at https://docs.databricks.com/dev-tools/bundles/inde >>> [CLI] bundle validate --output json == Collecting resources into ./enable_pydabs=no ->>> [CLI] bundle init default-python --config-file ./input.json +>>> [CLI] bundle init pydabs --config-file ./input.json Welcome to the default Python template for Databricks Asset Bundles! Workspace to use (auto-detected, edit in 'my_pydabs/databricks.yml'): [DATABRICKS_URL] diff --git a/acceptance/bundle/templates/pydabs/check-consistency/script b/acceptance/bundle/templates/pydabs/check-consistency/script index e1c6a32cb4..e7ee4f1e87 100644 --- a/acceptance/bundle/templates/pydabs/check-consistency/script +++ b/acceptance/bundle/templates/pydabs/check-consistency/script @@ -3,7 +3,7 @@ for enable_pydabs in yes no; do enable_pydabs=$enable_pydabs envsubst < input.json.tmpl > input.json - trace $CLI bundle init default-python --config-file ./input.json + trace $CLI bundle init pydabs --config-file ./input.json ( cd my_pydabs diff --git a/acceptance/bundle/templates/pydabs/check-formatting/input.json.tmpl b/acceptance/bundle/templates/pydabs/check-formatting/input.json.tmpl index f4532e3c14..0890152d2a 100644 --- a/acceptance/bundle/templates/pydabs/check-formatting/input.json.tmpl +++ b/acceptance/bundle/templates/pydabs/check-formatting/input.json.tmpl @@ -3,6 +3,5 @@ "include_notebook": "$INCLUDE_NOTEBOOK", "include_dlt": "$INCLUDE_DLT", "include_python": "$INCLUDE_PYTHON", - "enable_pydabs": "yes", "serverless": "$SERVERLESS" } diff --git a/acceptance/bundle/templates/pydabs/check-formatting/output.txt b/acceptance/bundle/templates/pydabs/check-formatting/output.txt index 86c87c9a77..57050fbec5 100644 --- a/acceptance/bundle/templates/pydabs/check-formatting/output.txt +++ b/acceptance/bundle/templates/pydabs/check-formatting/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle init default-python --config-file ./input.json +>>> [CLI] bundle init pydabs --config-file ./input.json Welcome to the default Python template for Databricks Asset Bundles! Workspace to use (auto-detected, edit in 'my_pydabs/databricks.yml'): [DATABRICKS_URL] diff --git a/acceptance/bundle/templates/pydabs/check-formatting/script b/acceptance/bundle/templates/pydabs/check-formatting/script index 8c1c6f37d7..540e84ceb7 100644 --- a/acceptance/bundle/templates/pydabs/check-formatting/script +++ b/acceptance/bundle/templates/pydabs/check-formatting/script @@ -1,6 +1,6 @@ envsubst < input.json.tmpl > input.json -trace $CLI bundle init default-python --config-file ./input.json +trace $CLI bundle init pydabs --config-file ./input.json # only keep relevant files for snapshots trace find my_pydabs -mindepth 1 ! -name 'pyproject.toml' ! -regex '.*/resources.*' -delete diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/input.json b/acceptance/bundle/templates/pydabs/deploy-classic/input.json index d052eea5de..461e9811d6 100644 --- a/acceptance/bundle/templates/pydabs/deploy-classic/input.json +++ b/acceptance/bundle/templates/pydabs/deploy-classic/input.json @@ -3,6 +3,5 @@ "include_notebook": "yes", "include_dlt": "yes", "include_python": "yes", - "serverless": "no", - "enable_pydabs": "yes" + "serverless": "no" } diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/output.txt b/acceptance/bundle/templates/pydabs/deploy-classic/output.txt index 5fd59ca998..0db6beb700 100644 --- a/acceptance/bundle/templates/pydabs/deploy-classic/output.txt +++ b/acceptance/bundle/templates/pydabs/deploy-classic/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle init default-python --config-file ./input.json +>>> [CLI] bundle init pydabs --config-file ./input.json Welcome to the default Python template for Databricks Asset Bundles! Workspace to use (auto-detected, edit in 'my_pydabs/databricks.yml'): [DATABRICKS_URL] diff --git a/acceptance/bundle/templates/pydabs/deploy-classic/script b/acceptance/bundle/templates/pydabs/deploy-classic/script index d01a9b4499..3ccd5aaabe 100644 --- a/acceptance/bundle/templates/pydabs/deploy-classic/script +++ b/acceptance/bundle/templates/pydabs/deploy-classic/script @@ -1,4 +1,4 @@ -trace $CLI bundle init default-python --config-file ./input.json +trace $CLI bundle init pydabs --config-file ./input.json ( cd my_pydabs diff --git a/acceptance/bundle/templates/pydabs/init-classic/input.json b/acceptance/bundle/templates/pydabs/init-classic/input.json index d052eea5de..461e9811d6 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/input.json +++ b/acceptance/bundle/templates/pydabs/init-classic/input.json @@ -3,6 +3,5 @@ "include_notebook": "yes", "include_dlt": "yes", "include_python": "yes", - "serverless": "no", - "enable_pydabs": "yes" + "serverless": "no" } diff --git a/acceptance/bundle/templates/pydabs/init-classic/output.txt b/acceptance/bundle/templates/pydabs/init-classic/output.txt index a4b5e3aa08..d924c24631 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/output.txt +++ b/acceptance/bundle/templates/pydabs/init-classic/output.txt @@ -1,5 +1,5 @@ ->>> [CLI] bundle init default-python --config-file ./input.json --output-dir output +>>> [CLI] bundle init pydabs --config-file ./input.json --output-dir output Welcome to the default Python template for Databricks Asset Bundles! Workspace to use (auto-detected, edit in 'my_pydabs/databricks.yml'): [DATABRICKS_URL] diff --git a/acceptance/bundle/templates/pydabs/init-classic/script b/acceptance/bundle/templates/pydabs/init-classic/script index d5306f4344..1d4076927a 100644 --- a/acceptance/bundle/templates/pydabs/init-classic/script +++ b/acceptance/bundle/templates/pydabs/init-classic/script @@ -1,4 +1,4 @@ -trace $CLI bundle init default-python --config-file ./input.json --output-dir output +trace $CLI bundle init pydabs --config-file ./input.json --output-dir output # only keep relevant files for snapshots trace find output/my_pydabs -mindepth 1 ! -name 'pyproject.toml' ! -regex '.*/resources.*' -delete diff --git a/libs/template/reader.go b/libs/template/reader.go index 0289701396..67966d29da 100644 --- a/libs/template/reader.go +++ b/libs/template/reader.go @@ -72,6 +72,33 @@ func (r *builtinReader) LoadSchemaAndTemplateFS(ctx context.Context) (*jsonschem func (r *builtinReader) Cleanup(ctx context.Context) {} +// overridingReader wraps another Reader and overrides their properties +type overridingReader struct { + underlying Reader + propertyDefaultOverrides map[string]any +} + +func (r *overridingReader) LoadSchemaAndTemplateFS(ctx context.Context) (*jsonschema.Schema, fs.FS, error) { + schema, schemaFS, err := r.underlying.LoadSchemaAndTemplateFS(ctx) + if err != nil { + return nil, nil, err + } + + for propName, newDefault := range r.propertyDefaultOverrides { + if propSchema, ok := schema.Properties[propName]; ok { + propSchema.Default = newDefault + } else { + return nil, nil, fmt.Errorf("cannot override default for non-existing property %s", propName) + } + } + + return schema, schemaFS, nil +} + +func (r *overridingReader) Cleanup(ctx context.Context) { + r.underlying.Cleanup(ctx) +} + // gitReader reads a template from a git repository. type gitReader struct { gitUrl string diff --git a/libs/template/reader_test.go b/libs/template/reader_test.go index 6d8419cdf4..f21e1ebe25 100644 --- a/libs/template/reader_test.go +++ b/libs/template/reader_test.go @@ -6,6 +6,8 @@ import ( "path/filepath" "testing" + "github.com/databricks/cli/libs/jsonschema" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/cmdio" "github.com/stretchr/testify/assert" @@ -215,3 +217,39 @@ func TestGitReaderWithTemplateDir(t *testing.T) { // Cleanup r.Cleanup(ctx) } + +func TestOverridingReader_LoadSchemaAndTemplateFS(t *testing.T) { + tmpDir := t.TempDir() + content := `{ + "welcome_message": "test", + "properties": { + "enable_feature_x": { + "type": "string", + "default": "no" + } + } + }` + + testutil.WriteFile(t, filepath.Join(tmpDir, "databricks_template_schema.json"), content) + ctx := context.Background() + + r := overridingReader{ + underlying: &localReader{path: tmpDir}, + propertyDefaultOverrides: map[string]any{ + "enable_feature_x": "yes", + }, + } + + schema, fsys, err := r.LoadSchemaAndTemplateFS(ctx) + + require.NoError(t, err) + require.NotNil(t, schema) + require.NotNil(t, fsys) + + assert.Equal(t, map[string]*jsonschema.Schema{ + "enable_feature_x": { + Type: "string", + Default: "yes", + }, + }, schema.Properties) +} diff --git a/libs/template/template.go b/libs/template/template.go index 74de09a857..7f392bb217 100644 --- a/libs/template/template.go +++ b/libs/template/template.go @@ -31,7 +31,7 @@ const ( CLIPipelines TemplateName = "cli-pipelines" DbtSql TemplateName = "dbt-sql" MlopsStacks TemplateName = "mlops-stacks" - DefaultPydabs TemplateName = "default-pydabs" + Pydabs TemplateName = "pydabs" Custom TemplateName = "custom" ExperimentalJobsAsCode TemplateName = "experimental-jobs-as-code" Default TemplateName = "default" @@ -85,11 +85,19 @@ var databricksTemplates = []Template{ Writer: &writerWithFullTelemetry{defaultWriter: defaultWriter{name: MlopsStacks}}, }, { - name: DefaultPydabs, + name: Pydabs, hidden: true, - description: "The default PyDABs template", - Reader: &gitReader{gitUrl: "https://databricks.github.io/workflows-authoring-toolkit/pydabs-template.git", cloneFunc: git.Clone}, - Writer: &writerWithFullTelemetry{defaultWriter: defaultWriter{name: DefaultPydabs}}, + description: "A variant of 'default-python' template that defines resources in Python instead of YAML", + Reader: &overridingReader{ + underlying: &builtinReader{name: string(DefaultPython)}, + propertyDefaultOverrides: map[string]any{ + // In 'default-python' template 'enable_pydabs' prompt is hidden with default of 'no'. + // We want a separate template called 'pydabs' to reuse 'default-python' but have + // the 'enable_pydabs' property enabled by default that switches from YAML to Python. + "enable_pydabs": "yes", + }, + }, + Writer: &writerWithFullTelemetry{defaultWriter: defaultWriter{name: Pydabs}}, }, { name: ExperimentalJobsAsCode, diff --git a/libs/template/template_test.go b/libs/template/template_test.go index c738bac801..48772e0f45 100644 --- a/libs/template/template_test.go +++ b/libs/template/template_test.go @@ -56,7 +56,7 @@ func TestTemplateGetDatabricksTemplate(t *testing.T) { DefaultSql, DbtSql, MlopsStacks, - DefaultPydabs, + Pydabs, } for _, name := range names {