From a773a79470e0a2117dafe0d3d2925f25e5d5b9b0 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Jun 2025 11:32:34 +0200 Subject: [PATCH 1/9] acc: yaml check all combinations of default-python; add $TESTROOT --- acceptance/acceptance_test.go | 1 + .../bundle/templates/default-python/combinations/output.txt | 2 ++ acceptance/bundle/templates/default-python/combinations/script | 1 + 3 files changed, 4 insertions(+) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index e8448ee8ed..ed4091957b 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -201,6 +201,7 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { testdiff.PrepareReplacementSdkVersion(t, &repls) testdiff.PrepareReplacementsGoVersion(t, &repls) + t.Setenv("TESTROOT", cwd) repls.SetPath(cwd, "[TESTROOT]") repls.Repls = append(repls.Repls, testdiff.Replacement{Old: regexp.MustCompile("dbapi[0-9a-f]+"), New: "[DATABRICKS_TOKEN]"}) diff --git a/acceptance/bundle/templates/default-python/combinations/output.txt b/acceptance/bundle/templates/default-python/combinations/output.txt index 8715bcbb7f..70ff4ec443 100644 --- a/acceptance/bundle/templates/default-python/combinations/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/output.txt @@ -9,6 +9,8 @@ Workspace to use (auto-detected, edit in 'X[UNIQUE_NAME]/databricks.yml'): [DATA Please refer to the README.md file for "getting started" instructions. See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. +>>> yamlfmt -conf=[TESTROOT]/../yamlfmt.yml . + >>> [CLI] bundle validate -t dev Name: X[UNIQUE_NAME] Target: dev diff --git a/acceptance/bundle/templates/default-python/combinations/script b/acceptance/bundle/templates/default-python/combinations/script index 8e117f5cd2..b53360b3e2 100644 --- a/acceptance/bundle/templates/default-python/combinations/script +++ b/acceptance/bundle/templates/default-python/combinations/script @@ -2,6 +2,7 @@ envsubst < input.json.tmpl > input.json trace $CLI bundle init default-python --config-file ./input.json cd ./X* +trace yamlfmt -conf=$TESTROOT/../yamlfmt.yml . trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod From fd78260886c9163cebe50a6d059ec514f7fed8c3 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Jun 2025 11:36:28 +0200 Subject: [PATCH 2/9] dry/verbose --- .../bundle/templates/default-python/combinations/output.txt | 3 ++- acceptance/bundle/templates/default-python/combinations/script | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/acceptance/bundle/templates/default-python/combinations/output.txt b/acceptance/bundle/templates/default-python/combinations/output.txt index 70ff4ec443..df1d4804c1 100644 --- a/acceptance/bundle/templates/default-python/combinations/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/output.txt @@ -9,7 +9,8 @@ Workspace to use (auto-detected, edit in 'X[UNIQUE_NAME]/databricks.yml'): [DATA Please refer to the README.md file for "getting started" instructions. See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. ->>> yamlfmt -conf=[TESTROOT]/../yamlfmt.yml . +>>> yamlfmt -conf=[TESTROOT]/../yamlfmt.yml -dry -verbose . +No files will be changed. >>> [CLI] bundle validate -t dev Name: X[UNIQUE_NAME] diff --git a/acceptance/bundle/templates/default-python/combinations/script b/acceptance/bundle/templates/default-python/combinations/script index b53360b3e2..da7b674a94 100644 --- a/acceptance/bundle/templates/default-python/combinations/script +++ b/acceptance/bundle/templates/default-python/combinations/script @@ -2,7 +2,7 @@ envsubst < input.json.tmpl > input.json trace $CLI bundle init default-python --config-file ./input.json cd ./X* -trace yamlfmt -conf=$TESTROOT/../yamlfmt.yml . +trace yamlfmt -conf=$TESTROOT/../yamlfmt.yml -dry -verbose . trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod From 11dc6a8112da4bd531f61feefa1ce48633e489be Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Jun 2025 11:49:31 +0200 Subject: [PATCH 3/9] add yamlcheck.py --- acceptance/bin/yamlcheck.py | 40 +++++++++++++++++++ .../default-python/combinations/script | 2 +- 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100755 acceptance/bin/yamlcheck.py diff --git a/acceptance/bin/yamlcheck.py b/acceptance/bin/yamlcheck.py new file mode 100755 index 0000000000..d47a6ebf02 --- /dev/null +++ b/acceptance/bin/yamlcheck.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys +from pathlib import Path +from difflib import unified_diff + + +def main(): + current_dir = Path.cwd() + yaml_files = sorted(current_dir.glob("**/*.yml")) + sorted(current_dir.glob("**/*.yaml")) + + yamlfmt_conf = Path(os.environ["TESTROOT"]) / ".." / "yamlfmt.yml" + + has_changes = False + + for yaml_file in yaml_files: + # Read original content into memory + original_content = yaml_file.read_text().splitlines(keepends=True) + + # Run yamlfmt + subprocess.run(["yamlfmt", f"-conf={yamlfmt_conf}", str(yaml_file)], check=True, capture_output=True) + + # Compare with formatted content + formatted_content = yaml_file.read_text().splitlines(keepends=True) + + if original_content != formatted_content: + has_changes = True + diff = unified_diff(original_content, formatted_content, fromfile=str(yaml_file), tofile=str(yaml_file), lineterm="") + print("".join(diff)) + else: + print(f"{yaml_file} OK") + + if has_changes: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bundle/templates/default-python/combinations/script b/acceptance/bundle/templates/default-python/combinations/script index da7b674a94..ff1e5f4f00 100644 --- a/acceptance/bundle/templates/default-python/combinations/script +++ b/acceptance/bundle/templates/default-python/combinations/script @@ -2,7 +2,7 @@ envsubst < input.json.tmpl > input.json trace $CLI bundle init default-python --config-file ./input.json cd ./X* -trace yamlfmt -conf=$TESTROOT/../yamlfmt.yml -dry -verbose . +trace yamlcheck.py trace $CLI bundle validate -t dev trace $CLI bundle validate -t prod From 6981c5d8e83167d04d95c22a27f831a8d7b5ffae Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Jun 2025 11:54:03 +0200 Subject: [PATCH 4/9] change for stable output --- acceptance/bin/yamlcheck.py | 9 +++------ .../templates/default-python/combinations/output.txt | 3 +-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/acceptance/bin/yamlcheck.py b/acceptance/bin/yamlcheck.py index d47a6ebf02..a263bdee20 100755 --- a/acceptance/bin/yamlcheck.py +++ b/acceptance/bin/yamlcheck.py @@ -10,30 +10,27 @@ def main(): current_dir = Path.cwd() yaml_files = sorted(current_dir.glob("**/*.yml")) + sorted(current_dir.glob("**/*.yaml")) + if not yaml_files: + sys.exit("No YAML files found") yamlfmt_conf = Path(os.environ["TESTROOT"]) / ".." / "yamlfmt.yml" has_changes = False for yaml_file in yaml_files: - # Read original content into memory original_content = yaml_file.read_text().splitlines(keepends=True) - # Run yamlfmt subprocess.run(["yamlfmt", f"-conf={yamlfmt_conf}", str(yaml_file)], check=True, capture_output=True) - # Compare with formatted content formatted_content = yaml_file.read_text().splitlines(keepends=True) if original_content != formatted_content: has_changes = True diff = unified_diff(original_content, formatted_content, fromfile=str(yaml_file), tofile=str(yaml_file), lineterm="") print("".join(diff)) - else: - print(f"{yaml_file} OK") if has_changes: - sys.exit(1) + sys.exit("UNEXPECTED: YAML formatting issues") if __name__ == "__main__": diff --git a/acceptance/bundle/templates/default-python/combinations/output.txt b/acceptance/bundle/templates/default-python/combinations/output.txt index df1d4804c1..adaec99e94 100644 --- a/acceptance/bundle/templates/default-python/combinations/output.txt +++ b/acceptance/bundle/templates/default-python/combinations/output.txt @@ -9,8 +9,7 @@ Workspace to use (auto-detected, edit in 'X[UNIQUE_NAME]/databricks.yml'): [DATA Please refer to the README.md file for "getting started" instructions. See also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html. ->>> yamlfmt -conf=[TESTROOT]/../yamlfmt.yml -dry -verbose . -No files will be changed. +>>> yamlcheck.py >>> [CLI] bundle validate -t dev Name: X[UNIQUE_NAME] From 6ae47d587b9ec06f51f9dda4c268a6d6f74264df Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Jun 2025 12:01:09 +0200 Subject: [PATCH 5/9] trailing whitespace and filenames --- acceptance/bin/yamlcheck.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/acceptance/bin/yamlcheck.py b/acceptance/bin/yamlcheck.py index a263bdee20..ab5706eb57 100755 --- a/acceptance/bin/yamlcheck.py +++ b/acceptance/bin/yamlcheck.py @@ -15,7 +15,7 @@ def main(): yamlfmt_conf = Path(os.environ["TESTROOT"]) / ".." / "yamlfmt.yml" - has_changes = False + has_changes = [] for yaml_file in yaml_files: original_content = yaml_file.read_text().splitlines(keepends=True) @@ -25,12 +25,15 @@ def main(): formatted_content = yaml_file.read_text().splitlines(keepends=True) if original_content != formatted_content: - has_changes = True - diff = unified_diff(original_content, formatted_content, fromfile=str(yaml_file), tofile=str(yaml_file), lineterm="") + has_changes.append(str(yaml_file)) + # Add $ markers for trailing whitespace + original_with_markers = [line.rstrip("\n") + ("$" if line.rstrip() != line.rstrip("\n") else "") + "\n" for line in original_content] + formatted_with_markers = [line.rstrip("\n") + ("$" if line.rstrip() != line.rstrip("\n") else "") + "\n" for line in formatted_content] + diff = unified_diff(original_with_markers, formatted_with_markers, fromfile=str(yaml_file), tofile=str(yaml_file), lineterm="") print("".join(diff)) if has_changes: - sys.exit("UNEXPECTED: YAML formatting issues") + sys.exit("UNEXPECTED: YAML formatting issues in " + " ".join(has_changes)) if __name__ == "__main__": From 0bff1134f253e1bd978e1ba400579321e0ef2167 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Jun 2025 12:05:16 +0200 Subject: [PATCH 6/9] fix trailing whitespace --- .../{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl index 23a89c5e3c..9dea88dede 100644 --- a/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl +++ b/libs/template/templates/default-python/template/{{.project_name}}/resources/{{.project_name}}.job.yml.tmpl @@ -49,7 +49,7 @@ resources: {{- else if (eq .include_notebook "yes" )}} depends_on: - task_key: notebook_task - {{end}} +{{end}} {{- if $with_serverless }} environment_key: default {{- else }} From 5218c71236bbac852cab82b16be83a27fb227a0b Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 11 Jun 2025 12:32:59 +0200 Subject: [PATCH 7/9] build yamlfmt --- acceptance/acceptance_test.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index ed4091957b..cc77e4be51 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -146,6 +146,8 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { execPath = BuildCLI(t, buildDir, coverDir) } + BuildYamlfmt(t) + t.Setenv("CLI", execPath) repls.SetPath(execPath, "[CLI]") @@ -1146,3 +1148,10 @@ func isSameYAMLContent(str1, str2 string) bool { return reflect.DeepEqual(obj1, obj2) } + +func BuildYamlfmt(t *testing.T) { + args := []string{ + "make", "-s", "tools/yamlfmt", + } + RunCommand(t, args, "..") +} From 2797a28f6bab0266201b39aae138bfd7b588192d Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Jun 2025 10:31:30 +0200 Subject: [PATCH 8/9] add .exe on windows --- acceptance/acceptance_test.go | 14 +++++++++----- acceptance/bin/yamlcheck.py | 8 +++++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index cc77e4be51..5fcd3a1e08 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -81,6 +81,13 @@ const ( EnvFilterVar = "ENVFILTER" ) +var exeSuffix = func() string { + if runtime.GOOS == "windows" { + return ".exe" + } + return "" +}() + var Scripts = map[string]bool{ EntryPointScript: true, CleanupScript: true, @@ -186,10 +193,7 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { t.Setenv("DATABRICKS_TF_CLI_CONFIG_FILE", terraformrcPath) repls.SetPath(terraformrcPath, "[DATABRICKS_TF_CLI_CONFIG_FILE]") - terraformExecPath := filepath.Join(buildDir, "terraform") - if runtime.GOOS == "windows" { - terraformExecPath += ".exe" - } + terraformExecPath := filepath.Join(buildDir, "terraform") + exeSuffix t.Setenv("DATABRICKS_TF_EXEC_PATH", terraformExecPath) t.Setenv("TERRAFORM", terraformExecPath) repls.SetPath(terraformExecPath, "[TERRAFORM]") @@ -1151,7 +1155,7 @@ func isSameYAMLContent(str1, str2 string) bool { func BuildYamlfmt(t *testing.T) { args := []string{ - "make", "-s", "tools/yamlfmt", + "make", "-s", "tools/yamlfmt" + exeSuffix, } RunCommand(t, args, "..") } diff --git a/acceptance/bin/yamlcheck.py b/acceptance/bin/yamlcheck.py index ab5706eb57..eca62517e8 100755 --- a/acceptance/bin/yamlcheck.py +++ b/acceptance/bin/yamlcheck.py @@ -7,6 +7,11 @@ from difflib import unified_diff +NAME = "yamlfmt" +if sys.platform.startswith("win"): + NAME += ".exe" + + def main(): current_dir = Path.cwd() yaml_files = sorted(current_dir.glob("**/*.yml")) + sorted(current_dir.glob("**/*.yaml")) @@ -14,13 +19,14 @@ def main(): sys.exit("No YAML files found") yamlfmt_conf = Path(os.environ["TESTROOT"]) / ".." / "yamlfmt.yml" + yamlfmt = Path(os.environ["TESTROOT"]) / ".." / "tools" / NAME has_changes = [] for yaml_file in yaml_files: original_content = yaml_file.read_text().splitlines(keepends=True) - subprocess.run(["yamlfmt", f"-conf={yamlfmt_conf}", str(yaml_file)], check=True, capture_output=True) + subprocess.run([yamlfmt, f"-conf={yamlfmt_conf}", str(yaml_file)], check=True, capture_output=True) formatted_content = yaml_file.read_text().splitlines(keepends=True) From 8b2d9299f7e837415592e35145fa01ad46e8c7c5 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Thu, 12 Jun 2025 11:50:37 +0200 Subject: [PATCH 9/9] fix --- Makefile | 3 +++ acceptance/acceptance_test.go | 1 + 2 files changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 137f9f86b1..0ea45eba93 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,9 @@ fmt: tools/yamlfmt tools/yamlfmt: go.mod go build -o tools/yamlfmt github.com/google/yamlfmt/cmd/yamlfmt +tools/yamlfmt.exe: go.mod + go build -o tools/yamlfmt.exe github.com/google/yamlfmt/cmd/yamlfmt + ws: ./tools/validate_whitespace.py diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 5fcd3a1e08..034737e102 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -1154,6 +1154,7 @@ func isSameYAMLContent(str1, str2 string) bool { } func BuildYamlfmt(t *testing.T) { + // Using make here instead of "go build" directly cause it's faster when it's already built args := []string{ "make", "-s", "tools/yamlfmt" + exeSuffix, }