From 12067ee95a746236e5c9aac49ab2b6f33420050b Mon Sep 17 00:00:00 2001 From: Yong Date: Thu, 30 Jul 2026 01:03:19 -0500 Subject: [PATCH 1/2] Automate enum sync for spec files --- core-spec/spec.yaml | 4 +- scripts/generate-spec-types.py | 195 +++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+), 2 deletions(-) create mode 100755 scripts/generate-spec-types.py diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 32fbb3e1..2cd94d58 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -29,6 +29,7 @@ version: 0.2.0.dev0 # Standard enums used throughout the specification # Supported expression language dialects +# Auto-generated from osi-schema.json ($defs.Dialect.enum). dialects: - "ANSI_SQL" # Standard SQL dialect - "SNOWFLAKE" # Snowflake @@ -39,8 +40,7 @@ dialects: - "BIGQUERY" # Google BigQuery GoogleSQL # Supported logical data types for fields and metrics -# TODO: Generate this list from the authoritative DataType enum in -# osi-schema.json ($defs.DataType.enum); until then, keep them in sync. +# Auto-generated from osi-schema.json ($defs.DataType.enum). datatypes: - "String" # Variable-length Unicode character data - "Integer" # Exact integral number diff --git a/scripts/generate-spec-types.py b/scripts/generate-spec-types.py new file mode 100755 index 00000000..b3d7ba18 --- /dev/null +++ b/scripts/generate-spec-types.py @@ -0,0 +1,195 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Generate enums for spec.yaml from osi-schema.json. + +Usage: + python3 scripts/generate-spec-types.py --check # exit 1 if spec.yaml is out of sync + python3 scripts/generate-spec-types.py --apply # update spec.yaml in-place +""" + +import json +import sys +import argparse +from pathlib import Path + +try: + import yaml +except ImportError: + print("Missing dependency. Install with: pip install PyYAML") + sys.exit(1) + +REPO = Path(__file__).resolve().parent.parent +SCHEMA = REPO / "core-spec" / "osi-schema.json" +SPEC = REPO / "core-spec" / "spec.yaml" + +SECTIONS = [ + { + "key": "dialects", + "schema_path": ["$defs", "Dialect", "enum"], + "comment_col": 28, + "header": "# Supported expression language dialects", + "descriptions": { + "ANSI_SQL": "Standard SQL dialect", + "SNOWFLAKE": "Snowflake", + "MDX": "Multi-Dimensional Expressions", + "TABLEAU": "Tableau", + "DATABRICKS": "Databricks SQL", + "MAQL": "GoodData MAQL (Multi-Dimensional Analytical Query Language)", + "BIGQUERY": "Google BigQuery GoogleSQL", + }, + }, + { + "key": "datatypes", + "schema_path": ["$defs", "DataType", "enum"], + "comment_col": 28, + "header": "# Supported logical data types for fields and metrics", + "descriptions": { + "String": "Variable-length Unicode character data", + "Integer": "Exact integral number", + "Decimal": "Exact base-10 number", + "Float": "Approximate floating-point number", + "Boolean": "Logical two-valued truth type", + "Date": "Calendar date without time of day", + "Time": "Time of day without a date or timezone", + "DateTime": "Date and time without a timezone or offset", + "DateTimeTz": "Instant identified using offset or timezone context", + "Opaque": "Known type outside the portable vocabulary", + }, + }, +] + + +def _schema_values(section: dict) -> list[str]: + path = section["schema_path"] + with open(SCHEMA) as f: + data = json.load(f) + for part in path: + data = data[part] + return list(data) + + +def _spec_values(key: str) -> list[str]: + with open(SPEC) as f: + try: + docs = yaml.safe_load_all(f) + except yaml.YAMLError as e: + print(f"Error: Invalid YAML in {SPEC}: {e}", file=sys.stderr) + sys.exit(1) + for doc in docs: + if doc and key in doc: + return list(doc[key]) + return [] + + +def _generate_block(section: dict, values: list[str]) -> str: + key = section["key"] + comment_col = section["comment_col"] + descs = section["descriptions"] + ref = ".".join(section["schema_path"]) + + lines = [ + section["header"], + "# Auto-generated from osi-schema.json ({}).".format(ref), + "{}:".format(key), + ] + for v in values: + desc = descs.get(v) + entry = ' - "{}"'.format(v) + if desc: + lines.append('{}{}# {}'.format(entry, " " * (comment_col - len(entry)), desc)) + else: + lines.append(entry) + + return "\n".join(lines) + "\n\n" + + +def _replace_section(text: str, key: str, block: str) -> str: + lines = text.splitlines(keepends=True) + + key_idx = None + for idx, line in enumerate(lines): + if line.rstrip() == f"{key}:": + key_idx = idx + break + + if key_idx is None: + print(f"Error: '{key}:' not found in spec.yaml", file=sys.stderr) + sys.exit(1) + + start = key_idx + idx = key_idx - 1 + while idx >= 0 and lines[idx].strip() == "": + start = idx + idx -= 1 + while idx >= 0 and lines[idx].lstrip().startswith("#"): + start = idx + idx -= 1 + + end = key_idx + 1 + while end < len(lines) and (lines[end].strip() == "" or lines[end][0].isspace()): + end += 1 + + return "".join(lines[:start]) + block + "".join(lines[end:]) + + +def _apply(): + text = SPEC.read_text() + for section in SECTIONS: + values = _schema_values(section) + block = _generate_block(section, values) + text = _replace_section(text, section["key"], block) + + # parse and validate before write + try: + list(yaml.safe_load_all(text)) + except yaml.YAMLError as e: + print(f"Error: Generated YAML is invalid: {e}", file=sys.stderr) + sys.exit(1) + SPEC.write_text(text) + print(f"Wrote {SPEC}", file=sys.stderr) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--check", action="store_true", help="Check spec.yaml is in sync") + parser.add_argument("--apply", action="store_true", help="Update spec.yaml in-place") + args = parser.parse_args() + + if args.apply: + _apply() + return + + if args.check: + diff = False + for section in SECTIONS: + expected = _schema_values(section) + current = _spec_values(section["key"]) + if current != expected: + print( + f"spec.yaml '{section['key']}' is out of sync with osi-schema.json", + file=sys.stderr, + ) + diff = True + sys.exit(1 if diff else 0) + + parser.print_help() + + +if __name__ == "__main__": + main() From 3df75b9241d53368630174da5fd6059b7bafbdae Mon Sep 17 00:00:00 2001 From: Yong Date: Fri, 31 Jul 2026 02:53:11 -0500 Subject: [PATCH 2/2] Enhance script --- scripts/generate-spec-types.py | 67 +++++++++++++++++----------------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/scripts/generate-spec-types.py b/scripts/generate-spec-types.py index b3d7ba18..3cde2edc 100755 --- a/scripts/generate-spec-types.py +++ b/scripts/generate-spec-types.py @@ -23,6 +23,7 @@ python3 scripts/generate-spec-types.py --apply # update spec.yaml in-place """ +import difflib import json import sys import argparse @@ -84,19 +85,6 @@ def _schema_values(section: dict) -> list[str]: return list(data) -def _spec_values(key: str) -> list[str]: - with open(SPEC) as f: - try: - docs = yaml.safe_load_all(f) - except yaml.YAMLError as e: - print(f"Error: Invalid YAML in {SPEC}: {e}", file=sys.stderr) - sys.exit(1) - for doc in docs: - if doc and key in doc: - return list(doc[key]) - return [] - - def _generate_block(section: dict, values: list[str]) -> str: key = section["key"] comment_col = section["comment_col"] @@ -110,9 +98,12 @@ def _generate_block(section: dict, values: list[str]) -> str: ] for v in values: desc = descs.get(v) + if desc is None: + print(f"Warning: no description for '{key}' value '{v}'", file=sys.stderr) entry = ' - "{}"'.format(v) if desc: - lines.append('{}{}# {}'.format(entry, " " * (comment_col - len(entry)), desc)) + pad = max(1, comment_col - len(entry)) + lines.append('{}{}# {}'.format(entry, " " * pad, desc)) else: lines.append(entry) @@ -148,46 +139,56 @@ def _replace_section(text: str, key: str, block: str) -> str: return "".join(lines[:start]) + block + "".join(lines[end:]) -def _apply(): - text = SPEC.read_text() +def _render(text: str) -> str: for section in SECTIONS: values = _schema_values(section) block = _generate_block(section, values) text = _replace_section(text, section["key"], block) - # parse and validate before write + # parse and validate before returning try: list(yaml.safe_load_all(text)) except yaml.YAMLError as e: print(f"Error: Generated YAML is invalid: {e}", file=sys.stderr) sys.exit(1) - SPEC.write_text(text) + return text + + +def _apply(): + rendered = _render(SPEC.read_text()) + SPEC.write_text(rendered) print(f"Wrote {SPEC}", file=sys.stderr) +def _check() -> list: + current = SPEC.read_text() + rendered = _render(current) + if current == rendered: + return 0 + + diff = difflib.unified_diff( + current.splitlines(keepends=True), + rendered.splitlines(keepends=True), + fromfile=f"{SPEC} (current)", + tofile=f"{SPEC} (expected)" + ) + sys.stderr.writelines(diff) + print(f"{SPEC} is out of sync with {SCHEMA}. Run --apply to fix.", file=sys.stderr) + return 1 + + def main(): parser = argparse.ArgumentParser() - parser.add_argument("--check", action="store_true", help="Check spec.yaml is in sync") - parser.add_argument("--apply", action="store_true", help="Update spec.yaml in-place") + group = parser.add_mutually_exclusive_group() + group.add_argument("--check", action="store_true", help="Check spec.yaml is in sync") + group.add_argument("--apply", action="store_true", help="Update spec.yaml in-place") args = parser.parse_args() if args.apply: _apply() return - if args.check: - diff = False - for section in SECTIONS: - expected = _schema_values(section) - current = _spec_values(section["key"]) - if current != expected: - print( - f"spec.yaml '{section['key']}' is out of sync with osi-schema.json", - file=sys.stderr, - ) - diff = True - sys.exit(1 if diff else 0) - + sys.exit(_check()) parser.print_help()