From 518165e78ffcf88383c67f83aa99dd82369e06d3 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Wed, 22 Jul 2026 08:54:10 -0500 Subject: [PATCH 1/6] Add bidirectional GSF semantic model converter Add offline Ossie-to-GSF conversion with a CLI, round-trip metadata preservation, checked-in fixtures, and official Ossie schema validation. --- converters/README.md | 1 + converters/gsf/README.md | 131 +++ converters/gsf/pyproject.toml | 63 ++ converters/gsf/src/ossie_gsf/__init__.py | 30 + converters/gsf/src/ossie_gsf/converter.py | 889 ++++++++++++++++++ converters/gsf/tests/fixtures/sales.gsf.yaml | 96 ++ .../gsf/tests/fixtures/sales.ossie.yaml | 80 ++ converters/gsf/tests/test_converter.py | 296 ++++++ converters/gsf/uv.lock | 335 +++++++ 9 files changed, 1921 insertions(+) create mode 100644 converters/gsf/README.md create mode 100644 converters/gsf/pyproject.toml create mode 100644 converters/gsf/src/ossie_gsf/__init__.py create mode 100644 converters/gsf/src/ossie_gsf/converter.py create mode 100644 converters/gsf/tests/fixtures/sales.gsf.yaml create mode 100644 converters/gsf/tests/fixtures/sales.ossie.yaml create mode 100644 converters/gsf/tests/test_converter.py create mode 100644 converters/gsf/uv.lock diff --git a/converters/README.md b/converters/README.md index ac31d856..be287179 100644 --- a/converters/README.md +++ b/converters/README.md @@ -75,6 +75,7 @@ The Ossie specification currently defines extensions for the following vendors: | `DATABRICKS` | Databricks semantic layer | | `OMNI` | Omni semantic model | | `WISDOM` | WisdomAI domain | +| `GSF` | NVIDIA Generative Semantic Fabric standalone YAML | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/gsf/README.md b/converters/gsf/README.md new file mode 100644 index 00000000..e493fb10 --- /dev/null +++ b/converters/gsf/README.md @@ -0,0 +1,131 @@ + + +# Apache Ossie ↔ NVIDIA GSF Converter + +Offline conversion between Apache Ossie YAML and the standalone semantic-model +YAML format supported by [NVIDIA GSF](https://github.com/NVIDIA/GSF). No GSF, +Neo4j, database, or network connection is required. + +## Mapping + +| Apache Ossie | Standalone GSF YAML | +|---|---| +| Semantic model | `model` | +| Dataset | `terms[]` | +| Physical field | `column_attributes[]` | +| Computed field | `sql_attributes[]` with `kind: field` | +| Metric | `sql_attributes[]` with `kind: metric` | +| Relationship | `semantic_foreign_keys[]` | +| Expression dialects | `expressions[]` | +| Dataset source | Term `source` mapping | + +Computed fields and metrics include generated full SQL and explicit +`table_refs`, which lets GSF validate and attach them to its ingested catalog. + +## Setup + +```bash +cd converters/gsf +uv sync +``` + +## Ossie → GSF + +```bash +uv run ossie-gsf export \ + --input ../../examples/tpcds_semantic_model.yaml \ + --output tpcds.gsf.yaml \ + --database-name tpcds +``` + +`--database-name` supplies the database for sources written as `schema.table`. +Fully qualified `database.schema.table` sources do not require it. + +Python: + +```python +from ossie_gsf import convert_ossie_to_gsf + +gsf_yaml = convert_ossie_to_gsf( + ossie_yaml, + database_name="tpcds", +) +``` + +## GSF → Ossie + +```bash +uv run ossie-gsf import \ + --input tpcds.gsf.yaml \ + --output semantic_model.yaml +``` + +Use `--name` to override the exported Ossie semantic-model name. + +Python: + +```python +from ossie_gsf import convert_gsf_to_ossie + +ossie_yaml = convert_gsf_to_ossie(gsf_yaml) +``` + +## Loading the converted model into GSF + +Run the native GSF importer from the GSF repository: + +```bash +uv run python -m gsf.semantic import \ + --database-name tpcds \ + --input tpcds.gsf.yaml +``` + +GSF resolves the file against its existing catalog, writes the graph +transactionally, validates SQL, and refreshes semantic embeddings. + +## Conversion behavior and limitations + +- Apache Ossie `0.2.0.dev0` and GSF model-file `1.0` are supported. +- One semantic model is converted per document. +- GSF term sources must resolve to physical `database.schema.table` names. +- Simple field expressions become `ColumnAttribute` mappings. Other field + expressions become SQL attributes. +- Multi-dataset metric SQL is joined through declared Ossie relationships. + Disconnected datasets fail conversion instead of producing a cross join. +- GSF SQL attribute names are global, so duplicate computed-field or metric + names fail conversion. +- Native GSF SQL attributes with `kind: attribute` have no unambiguous Ossie + field/metric equivalent and fail GSF → Ossie conversion. +- Metrics are attached to the first referenced dataset. Unqualified metrics in + multi-dataset models need a `GSF` custom extension containing + `{"term": "dataset_name"}`. +- Expression dialect variants are preserved in the standalone GSF file. +- Ossie custom extensions and GSF metadata are preserved through metadata and + `GSF` custom-extension payloads. + +## Tests + +```bash +uv run pytest +``` + +The suite includes a checked-in Ossie/GSF fixture pair, bidirectional +round-trip checks, validation failures, CLI coverage, and verification of +generated Ossie YAML with the repository's official validator. diff --git a/converters/gsf/pyproject.toml b/converters/gsf/pyproject.toml new file mode 100644 index 00000000..c50c99ad --- /dev/null +++ b/converters/gsf/pyproject.toml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[dependency-groups] +dev = [ + "jsonschema>=4.26.0", + "pytest>=8.0", + "sqlglot>=30.12.0", +] + +[project] +name = "apache-ossie-gsf" +version = "0.1.0.dev0" +description = "NVIDIA GSF <> Apache Ossie offline YAML converter" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Open Semantic Interchange", + "NVIDIA GSF", + "semantic model", +] +dependencies = [ + "PyYAML>=6.0", +] + +[project.scripts] +ossie-gsf = "ossie_gsf.converter:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_gsf"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = ["dev"] diff --git a/converters/gsf/src/ossie_gsf/__init__.py b/converters/gsf/src/ossie_gsf/__init__.py new file mode 100644 index 00000000..96522e87 --- /dev/null +++ b/converters/gsf/src/ossie_gsf/__init__.py @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bidirectional Apache Ossie and NVIDIA GSF converter.""" + +from .converter import ( + GSFConversionError, + convert_gsf_to_ossie, + convert_ossie_to_gsf, +) + +__all__ = [ + "GSFConversionError", + "convert_gsf_to_ossie", + "convert_ossie_to_gsf", +] diff --git a/converters/gsf/src/ossie_gsf/converter.py b/converters/gsf/src/ossie_gsf/converter.py new file mode 100644 index 00000000..70e79c40 --- /dev/null +++ b/converters/gsf/src/ossie_gsf/converter.py @@ -0,0 +1,889 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Offline Apache Ossie ↔ standalone NVIDIA GSF YAML conversion.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections import Counter +from pathlib import Path +from typing import Any, Mapping + +import yaml + +OSSIE_VERSION = "0.2.0.dev0" +GSF_VERSION = "1.0" +GSF_VENDOR = "GSF" + +_SIMPLE_COLUMN = re.compile( + r"^(?:(?P[A-Za-z_][A-Za-z0-9_]*)\.)?" + r"(?P[A-Za-z_][A-Za-z0-9_]*)$" +) +_TERM_REFERENCE = re.compile( + r"\b(?P[A-Za-z_][A-Za-z0-9_]*)\." + r"(?P[A-Za-z_][A-Za-z0-9_]*)\b" +) + + +class GSFConversionError(Exception): + """Raised when a document cannot be converted safely.""" + + +def convert_ossie_to_gsf( + ossie_yaml: str, + *, + database_name: str | None = None, +) -> str: + """Convert Apache Ossie YAML to standalone GSF semantic-model YAML.""" + root, model = _parse_ossie(ossie_yaml) + source_datasets = model.get("datasets") or [] + if not isinstance(source_datasets, list) or not source_datasets: + raise GSFConversionError( + "The Ossie semantic model must contain at least one dataset" + ) + + datasets: dict[str, dict[str, Any]] = {} + terms: list[dict[str, Any]] = [] + resolved_databases: set[str] = set() + for dataset in source_datasets: + if not isinstance(dataset, dict) or not dataset.get("name"): + raise GSFConversionError( + "Every Ossie dataset must be a mapping with a name" + ) + name = str(dataset["name"]) + if name in datasets: + raise GSFConversionError(f"Duplicate dataset name {name!r}") + source = _parse_source(dataset.get("source"), database_name) + if not source["database"] or not source["schema"]: + raise GSFConversionError( + f"Dataset {name!r} source must resolve to database.schema.table" + ) + resolved_databases.add(str(source["database"])) + datasets[name] = {"source": source, "original": dataset} + + term: dict[str, Any] = { + "name": name, + "source": source, + } + _copy_optional(dataset, term, "description") + ai_context = dataset.get("ai_context") + if isinstance(ai_context, dict): + cleaned_ai_context = { + key: value for key, value in ai_context.items() if key != "synonyms" + } + if cleaned_ai_context: + term["ai_context"] = cleaned_ai_context + elif ai_context is not None: + term["ai_context"] = ai_context + synonyms = _synonyms(ai_context) + if synonyms: + term["synonyms"] = synonyms + _copy_optional(dataset, term, "primary_key") + _copy_optional(dataset, term, "unique_keys") + _copy_ossie_metadata(dataset, term) + terms.append(term) + + if len(resolved_databases) != 1: + raise GSFConversionError( + "One GSF model file must target exactly one database; found: " + + ", ".join(sorted(resolved_databases)) + ) + resolved_database = next(iter(resolved_databases)) + + relationships = [ + _relationship_to_gsf(relationship, datasets) + for relationship in model.get("relationships") or [] + ] + term_by_name = {term["name"]: term for term in terms} + sql_names: list[str] = [] + + for dataset_name, context in datasets.items(): + dataset = context["original"] + term = term_by_name[dataset_name] + column_attributes: list[dict[str, Any]] = [] + sql_attributes: list[dict[str, Any]] = [] + used_columns: set[str] = set() + + for field in dataset.get("fields") or []: + if not isinstance(field, dict) or not field.get("name"): + raise GSFConversionError( + f"Every field in dataset {dataset_name!r} needs a name" + ) + name = str(field["name"]) + expressions = _normalize_expressions(field.get("expression"), name) + selected = _pick_expression(expressions, name) + source_column = _simple_source_column( + selected, + dataset_name, + str(context["source"]["table"]), + ) + if source_column is not None: + if source_column in used_columns: + raise GSFConversionError( + f"Multiple fields in dataset {dataset_name!r} map to " + f"physical column {source_column!r}" + ) + used_columns.add(source_column) + attribute: dict[str, Any] = { + "name": name, + "source_column": source_column, + "expressions": expressions, + } + _copy_optional(field, attribute, "description") + _copy_optional(field, attribute, "ai_context") + _copy_optional(field, attribute, "dimension") + _copy_ossie_metadata(field, attribute) + column_attributes.append(attribute) + else: + extension_data = _gsf_extension_data(field) + refs = extension_data.get("table_refs") or [dataset_name] + if not isinstance(refs, list) or any( + ref not in datasets for ref in refs + ): + raise GSFConversionError( + f"Computed field {name!r} has invalid GSF table_refs" + ) + sources = [(str(ref), datasets[str(ref)]["source"]) for ref in refs] + relevant_relationships = [ + relationship + for relationship in relationships + if relationship["from_term"] in refs + or relationship["to_term"] in refs + ] + attribute = { + "name": name, + "kind": "field", + "expressions": expressions, + "sql": str(extension_data.get("sql") or "") + or _wrap_expression( + selected, name, sources, relevant_relationships + ), + "table_refs": list(refs), + } + _copy_optional(field, attribute, "description") + _copy_optional(field, attribute, "ai_context") + _copy_optional(field, attribute, "dimension") + _copy_ossie_metadata(field, attribute) + sql_attributes.append(attribute) + sql_names.append(name) + + if column_attributes: + term["column_attributes"] = column_attributes + if sql_attributes: + term["sql_attributes"] = sql_attributes + + for metric in model.get("metrics") or []: + if not isinstance(metric, dict) or not metric.get("name"): + raise GSFConversionError("Every Ossie metric must be a mapping with a name") + name = str(metric["name"]) + expressions = _normalize_expressions(metric.get("expression"), name) + selected = _pick_expression(expressions, name) + referenced_terms = _referenced_terms(selected, datasets) + owner = _metric_term(metric, referenced_terms, datasets) + refs = referenced_terms or [owner] + sources = [(term_name, datasets[term_name]["source"]) for term_name in refs] + relevant_relationships = [ + relationship + for relationship in relationships + if relationship["from_term"] in refs or relationship["to_term"] in refs + ] + extension_data = _gsf_extension_data(metric) + preserved_refs = extension_data.get("table_refs") + if isinstance(preserved_refs, list) and preserved_refs: + if any(ref not in datasets for ref in preserved_refs): + raise GSFConversionError(f"Metric {name!r} has invalid GSF table_refs") + refs = [str(ref) for ref in preserved_refs] + sources = [(term_name, datasets[term_name]["source"]) for term_name in refs] + relevant_relationships = [ + relationship + for relationship in relationships + if relationship["from_term"] in refs or relationship["to_term"] in refs + ] + attribute = { + "name": name, + "kind": "metric", + "expressions": expressions, + "sql": str(extension_data.get("sql") or "") + or _wrap_expression(selected, name, sources, relevant_relationships), + "table_refs": refs, + } + _copy_optional(metric, attribute, "description") + _copy_optional(metric, attribute, "ai_context") + _copy_ossie_metadata(metric, attribute) + term_by_name[owner].setdefault("sql_attributes", []).append(attribute) + sql_names.append(name) + + duplicate_sql = sorted( + name for name, count in Counter(sql_names).items() if count > 1 + ) + if duplicate_sql: + raise GSFConversionError( + "GSF SqlAttribute names are global; duplicate computed field or " + "metric names: " + ", ".join(duplicate_sql) + ) + + gsf_model: dict[str, Any] = { + "name": str(model["name"]), + "database": resolved_database, + } + _copy_optional(model, gsf_model, "description") + _copy_optional(model, gsf_model, "ai_context") + _copy_ossie_metadata(model, gsf_model) + + output: dict[str, Any] = { + "version": GSF_VERSION, + "model": gsf_model, + "terms": terms, + } + if relationships: + output["semantic_foreign_keys"] = relationships + return _dump_yaml(output) + + +def convert_gsf_to_ossie( + gsf_yaml: str, + *, + model_name: str | None = None, +) -> str: + """Convert standalone GSF semantic-model YAML to Apache Ossie YAML.""" + root = _parse_gsf(gsf_yaml) + source_model = root["model"] + terms = root["terms"] + term_names = { + str(term.get("name")) + for term in terms + if isinstance(term, dict) and term.get("name") + } + if len(term_names) != len(terms): + raise GSFConversionError("Every GSF term needs a unique non-empty name") + + datasets: list[dict[str, Any]] = [] + metrics: list[dict[str, Any]] = [] + for term in terms: + term_name = str(term["name"]) + source = _parse_source( + term.get("source"), + source_model.get("database"), + ) + if not source["database"] or not source["schema"]: + raise GSFConversionError( + f"Term {term_name!r} source must be fully qualified" + ) + dataset: dict[str, Any] = { + "name": term_name, + "source": ".".join( + str(source[key]) for key in ("database", "schema", "table") + ), + } + _copy_optional(term, dataset, "description") + _copy_optional(term, dataset, "primary_key") + _copy_optional(term, dataset, "unique_keys") + ai_context = term.get("ai_context") + synonyms = term.get("synonyms") or [] + merged_ai = _merge_synonyms(ai_context, synonyms) + if merged_ai is not None: + dataset["ai_context"] = merged_ai + dataset_extensions = _native_extensions(term) + if dataset_extensions: + dataset["custom_extensions"] = dataset_extensions + + fields: list[dict[str, Any]] = [] + for attribute in term.get("column_attributes") or []: + _validate_attribute(attribute, term_name, kind="column") + field: dict[str, Any] = { + "name": str(attribute["name"]), + "expression": { + "dialects": attribute.get("expressions") + or [ + { + "dialect": "ANSI_SQL", + "expression": str(attribute["source_column"]), + } + ] + }, + } + _copy_optional(attribute, field, "description") + _copy_optional(attribute, field, "ai_context") + _copy_optional(attribute, field, "dimension") + field_extensions = _native_extensions(attribute) + if field_extensions: + field["custom_extensions"] = field_extensions + fields.append(field) + + for attribute in term.get("sql_attributes") or []: + _validate_attribute(attribute, term_name, kind="sql") + sql_kind = attribute.get("kind") + if sql_kind not in {"field", "metric"}: + raise GSFConversionError( + f"SQL attribute {attribute['name']!r} kind must be " + "'field' or 'metric'" + ) + expression = { + "dialects": _normalize_native_expressions( + attribute.get("expressions"), + str(attribute["name"]), + ) + } + item: dict[str, Any] = { + "name": str(attribute["name"]), + "expression": expression, + } + _copy_optional(attribute, item, "description") + _copy_optional(attribute, item, "ai_context") + if sql_kind == "field": + _copy_optional(attribute, item, "dimension") + item["custom_extensions"] = _native_extensions( + attribute, + gsf_data={ + "kind": "field", + "sql": attribute.get("sql"), + "table_refs": attribute.get("table_refs"), + }, + ) + fields.append(item) + else: + item["custom_extensions"] = _native_extensions( + attribute, + gsf_data={ + "term": term_name, + "sql": attribute.get("sql"), + "table_refs": attribute.get("table_refs"), + }, + ) + metrics.append(item) + + if fields: + dataset["fields"] = fields + datasets.append(dataset) + + relationships: list[dict[str, Any]] = [] + for relationship in root.get("semantic_foreign_keys") or []: + if not isinstance(relationship, dict) or not relationship.get("name"): + raise GSFConversionError("Every semantic foreign key must have a name") + from_term = relationship.get("from_term") + to_term = relationship.get("to_term") + if from_term not in term_names or to_term not in term_names: + raise GSFConversionError( + f"Semantic foreign key {relationship['name']!r} references " + "an unknown term" + ) + from_columns = relationship.get("from_columns") or [] + to_columns = relationship.get("to_columns") or [] + if not from_columns or len(from_columns) != len(to_columns): + raise GSFConversionError( + f"Semantic foreign key {relationship['name']!r} must have " + "equal, non-empty column lists" + ) + converted_relationship: dict[str, Any] = { + "name": str(relationship["name"]), + "from": from_term, + "to": to_term, + "from_columns": list(from_columns), + "to_columns": list(to_columns), + } + _copy_optional( + relationship, + converted_relationship, + "ai_context", + ) + relationship_extensions = _native_extensions(relationship) + if relationship_extensions: + converted_relationship["custom_extensions"] = relationship_extensions + relationships.append(converted_relationship) + + semantic_model: dict[str, Any] = { + "name": model_name or str(source_model["name"]), + "datasets": datasets, + } + _copy_optional(source_model, semantic_model, "description") + _copy_optional(source_model, semantic_model, "ai_context") + model_extensions = _native_extensions(source_model) + if model_extensions: + semantic_model["custom_extensions"] = model_extensions + if relationships: + semantic_model["relationships"] = relationships + if metrics: + semantic_model["metrics"] = metrics + return _dump_yaml( + { + "version": OSSIE_VERSION, + "semantic_model": [semantic_model], + } + ) + + +def _parse_ossie(value: str) -> tuple[dict[str, Any], dict[str, Any]]: + root = _load_yaml(value, "Ossie") + unknown = sorted(set(root) - {"version", "semantic_model"}) + if unknown: + raise GSFConversionError( + "Unsupported Ossie root properties: " + ", ".join(unknown) + ) + if str(root.get("version", "")) != OSSIE_VERSION: + raise GSFConversionError( + f"Unsupported Ossie version {root.get('version')!r}; " + f"supported version is {OSSIE_VERSION!r}" + ) + models = root.get("semantic_model") + if not isinstance(models, list) or len(models) != 1: + raise GSFConversionError("Ossie input must contain exactly one semantic model") + model = models[0] + if not isinstance(model, dict) or not model.get("name"): + raise GSFConversionError("Ossie semantic model requires a name") + return root, model + + +def _parse_gsf(value: str) -> dict[str, Any]: + root = _load_yaml(value, "GSF") + unknown = sorted(set(root) - {"version", "model", "terms", "semantic_foreign_keys"}) + if unknown: + raise GSFConversionError( + "Unsupported GSF root properties: " + ", ".join(unknown) + ) + if str(root.get("version", "")) != GSF_VERSION: + raise GSFConversionError( + f"Unsupported GSF version {root.get('version')!r}; " + f"supported version is {GSF_VERSION!r}" + ) + model = root.get("model") + if not isinstance(model, dict) or not model.get("name"): + raise GSFConversionError("GSF model requires 'model.name'") + terms = root.get("terms") + if not isinstance(terms, list) or not terms: + raise GSFConversionError("'terms' must be a non-empty list") + relationships = root.get("semantic_foreign_keys", []) + if not isinstance(relationships, list): + raise GSFConversionError("'semantic_foreign_keys' must be a list") + return root + + +def _load_yaml(value: str, label: str) -> dict[str, Any]: + try: + root = yaml.safe_load(value) + except yaml.YAMLError as exc: + raise GSFConversionError(f"Invalid {label} YAML: {exc}") from exc + if not isinstance(root, dict): + raise GSFConversionError(f"Invalid {label} YAML: expected a root mapping") + return root + + +def _relationship_to_gsf( + relationship: Any, + datasets: Mapping[str, Any], +) -> dict[str, Any]: + if not isinstance(relationship, dict) or not relationship.get("name"): + raise GSFConversionError("Every Ossie relationship needs a name") + from_term = relationship.get("from") + to_term = relationship.get("to") + if from_term not in datasets or to_term not in datasets: + raise GSFConversionError( + f"Relationship {relationship['name']!r} references an unknown dataset" + ) + from_columns = relationship.get("from_columns") or [] + to_columns = relationship.get("to_columns") or [] + if not from_columns or len(from_columns) != len(to_columns): + raise GSFConversionError( + f"Relationship {relationship['name']!r} must have equal, " + "non-empty column lists" + ) + converted: dict[str, Any] = { + "name": str(relationship["name"]), + "from_term": from_term, + "to_term": to_term, + "from_columns": list(from_columns), + "to_columns": list(to_columns), + } + _copy_optional(relationship, converted, "ai_context") + _copy_ossie_metadata(relationship, converted) + return converted + + +def _normalize_expressions(value: Any, name: str) -> list[dict[str, str]]: + if not isinstance(value, dict): + raise GSFConversionError(f"{name!r} has no valid expression") + return _normalize_native_expressions(value.get("dialects"), name) + + +def _normalize_native_expressions( + value: Any, + name: str, +) -> list[dict[str, str]]: + if not isinstance(value, list) or not value: + raise GSFConversionError(f"{name!r} requires at least one expression dialect") + result: list[dict[str, str]] = [] + for item in value: + if ( + isinstance(item, dict) + and item.get("dialect") + and item.get("expression") is not None + ): + result.append( + { + "dialect": str(item["dialect"]), + "expression": str(item["expression"]), + } + ) + if not result: + raise GSFConversionError(f"{name!r} has no usable expression dialect") + return result + + +def _pick_expression(expressions: list[dict[str, str]], name: str) -> str: + for expression in expressions: + if expression["dialect"].upper() == "ANSI_SQL": + return expression["expression"] + if expressions: + return expressions[0]["expression"] + raise GSFConversionError(f"{name!r} has no usable expression") + + +def _simple_source_column( + expression: str, + dataset_name: str, + table_name: str, +) -> str | None: + match = _SIMPLE_COLUMN.fullmatch(expression.strip()) + if not match: + return None + qualifier = match.group("qualifier") + if qualifier and qualifier not in (dataset_name, table_name): + return None + return match.group("column") + + +def _referenced_terms( + expression: str, + datasets: Mapping[str, Any], +) -> list[str]: + result: list[str] = [] + for match in _TERM_REFERENCE.finditer(expression): + name = match.group("term") + if name in datasets and name not in result: + result.append(name) + return result + + +def _metric_term( + metric: dict[str, Any], + referenced_terms: list[str], + datasets: Mapping[str, Any], +) -> str: + term = _gsf_extension_data(metric).get("term") + if term in datasets: + return str(term) + if referenced_terms: + return referenced_terms[0] + if len(datasets) == 1: + return next(iter(datasets)) + raise GSFConversionError( + f"Metric {metric.get('name')!r} does not identify an owning dataset. " + 'Add a GSF extension with data {"term": "dataset_name"}.' + ) + + +def _gsf_extension_data(item: Mapping[str, Any]) -> dict[str, Any]: + for extension in item.get("custom_extensions") or []: + if extension.get("vendor_name") != GSF_VENDOR: + continue + try: + data = json.loads(str(extension.get("data") or "{}")) + except json.JSONDecodeError: + continue + if isinstance(data, dict): + return data + return {} + + +def _wrap_expression( + expression: str, + name: str, + sources: list[tuple[str, Mapping[str, Any]]], + relationships: list[dict[str, Any]], +) -> str: + stripped = expression.strip() + if stripped.upper().startswith(("SELECT ", "SELECT\n", "WITH ", "WITH\n")): + return stripped + if not sources: + raise GSFConversionError(f"Cannot determine a source table for {name!r}") + source_map = dict(sources) + anchor_name, anchor = sources[0] + from_sql = f"{_qualified_table(anchor)} AS {_quote_identifier(anchor_name)}" + joined = {anchor_name} + remaining = {term_name for term_name, _ in sources[1:]} + joins: list[str] = [] + while remaining: + matched = False + for relationship in relationships: + left = str(relationship["from_term"]) + right = str(relationship["to_term"]) + if left in joined and right in remaining: + new_name = right + elif right in joined and left in remaining: + new_name = left + else: + continue + conditions = [ + f"{_quote_identifier(left)}.{_quote_identifier(left_column)} = " + f"{_quote_identifier(right)}." + f"{_quote_identifier(right_column)}" + for left_column, right_column in zip( + relationship["from_columns"], + relationship["to_columns"], + strict=True, + ) + ] + joins.append( + f"JOIN {_qualified_table(source_map[new_name])} AS " + f"{_quote_identifier(new_name)} ON {' AND '.join(conditions)}" + ) + joined.add(new_name) + remaining.remove(new_name) + matched = True + break + if not matched: + missing = sorted(remaining)[0] + raise GSFConversionError( + f"{name!r} references disconnected dataset {missing!r}; " + "declare a relationship connecting all referenced datasets" + ) + return f"SELECT {stripped} AS {_quote_identifier(name)} FROM {from_sql}" + ( + f" {' '.join(joins)}" if joins else "" + ) + + +def _parse_source( + source: Any, + default_database: str | None, +) -> dict[str, str | None]: + if isinstance(source, dict): + database = source.get("database") or default_database + schema = source.get("schema") + table = source.get("table") + if not table: + raise GSFConversionError("Source mapping requires 'table'") + return { + "database": str(database) if database else None, + "schema": str(schema) if schema else None, + "table": str(table), + } + value = str(source or "").strip() + if not value: + raise GSFConversionError("Every dataset/term needs a source") + upper = value.upper() + if upper.startswith(("SELECT ", "SELECT\n", "WITH ", "WITH\n")): + raise GSFConversionError("GSF term sources must identify physical tables") + parts = _split_identifier(value) + if len(parts) == 3: + database, schema, table = parts + elif len(parts) == 2: + database, (schema, table) = default_database, parts + elif len(parts) == 1: + database, schema, table = default_database, None, parts[0] + else: + raise GSFConversionError( + f"Source {value!r} must be table, schema.table, or database.schema.table" + ) + return {"database": database, "schema": schema, "table": table} + + +def _split_identifier(value: str) -> list[str]: + parts: list[str] = [] + current: list[str] = [] + quote: str | None = None + for char in value: + if char in ('"', "`"): + quote = None if quote == char else char if quote is None else quote + elif char == "." and quote is None: + parts.append("".join(current).strip()) + current = [] + continue + current.append(char) + parts.append("".join(current).strip()) + return [ + part[1:-1] + if len(part) > 1 and part[0] == part[-1] and part[0] in ('"', "`") + else part + for part in parts + ] + + +def _qualified_table(source: Mapping[str, Any]) -> str: + return ".".join( + _quote_identifier(str(source[key])) for key in ("database", "schema", "table") + ) + + +def _quote_identifier(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +def _validate_attribute( + attribute: Any, + term_name: str, + *, + kind: str, +) -> None: + if not isinstance(attribute, dict) or not attribute.get("name"): + raise GSFConversionError( + f"Every {kind} attribute in term {term_name!r} needs a name" + ) + if kind == "column" and not attribute.get("source_column"): + raise GSFConversionError( + f"Column attribute {attribute['name']!r} needs source_column" + ) + + +def _synonyms(ai_context: Any) -> list[str]: + if not isinstance(ai_context, dict): + return [] + return [str(value) for value in ai_context.get("synonyms") or [] if value] + + +def _merge_synonyms(ai_context: Any, synonyms: Any) -> Any: + clean = [str(value) for value in synonyms if value] + if not clean: + return ai_context + if isinstance(ai_context, dict): + result = dict(ai_context) + result["synonyms"] = clean + return result + if isinstance(ai_context, str) and ai_context: + return {"instructions": ai_context, "synonyms": clean} + return {"synonyms": clean} + + +def _copy_optional( + source: Mapping[str, Any], + target: dict[str, Any], + key: str, +) -> None: + if source.get(key) is not None: + target[key] = source[key] + + +def _copy_ossie_metadata( + source: Mapping[str, Any], + target: dict[str, Any], +) -> None: + extension_data = _gsf_extension_data(source) + metadata = extension_data.get("metadata") + if isinstance(metadata, dict): + target["metadata"] = dict(metadata) + preserved_extensions = [ + extension + for extension in source.get("custom_extensions") or [] + if extension.get("vendor_name") != GSF_VENDOR + ] + if preserved_extensions: + native_metadata = dict(target.get("metadata") or {}) + ossie_metadata = dict(native_metadata.get("apache_ossie") or {}) + ossie_metadata["custom_extensions"] = preserved_extensions + native_metadata["apache_ossie"] = ossie_metadata + target["metadata"] = native_metadata + + +def _native_extensions( + source: Mapping[str, Any], + *, + gsf_data: dict[str, Any] | None = None, +) -> list[dict[str, Any]]: + metadata = source.get("metadata") + extensions: list[dict[str, Any]] = [] + if isinstance(metadata, dict): + ossie_metadata = metadata.get("apache_ossie") + if isinstance(ossie_metadata, dict): + extensions.extend(ossie_metadata.get("custom_extensions") or []) + remaining_metadata = { + key: value for key, value in metadata.items() if key != "apache_ossie" + } + else: + remaining_metadata = {} + data = dict(gsf_data or {}) + if remaining_metadata: + data["metadata"] = remaining_metadata + if data: + extensions.append( + { + "vendor_name": GSF_VENDOR, + "data": json.dumps(data), + } + ) + return extensions + + +def _dump_yaml(value: dict[str, Any]) -> str: + return yaml.safe_dump( + value, + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + ) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Convert Apache Ossie and standalone GSF YAML files" + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + export_parser = subparsers.add_parser( + "export", + help="Convert Ossie YAML to standalone GSF YAML", + ) + export_parser.add_argument("-i", "--input", type=Path, required=True) + export_parser.add_argument("-o", "--output", type=Path) + export_parser.add_argument("--database-name") + + import_parser = subparsers.add_parser( + "import", + help="Convert standalone GSF YAML to Ossie YAML", + ) + import_parser.add_argument("-i", "--input", type=Path, required=True) + import_parser.add_argument("-o", "--output", type=Path) + import_parser.add_argument("--name") + return parser + + +def main(argv: list[str] | None = None) -> None: + args = _build_parser().parse_args(argv) + try: + source = args.input.read_text(encoding="utf-8") + if args.command == "export": + output = convert_ossie_to_gsf( + source, + database_name=args.database_name, + ) + else: + output = convert_gsf_to_ossie( + source, + model_name=args.name, + ) + if args.output is None: + print(output, end="") + else: + args.output.write_text(output, encoding="utf-8") + except (GSFConversionError, OSError, UnicodeError) as exc: + print(f"Error: {exc}", file=sys.stderr) + raise SystemExit(1) from exc + + +if __name__ == "__main__": + main() diff --git a/converters/gsf/tests/fixtures/sales.gsf.yaml b/converters/gsf/tests/fixtures/sales.gsf.yaml new file mode 100644 index 00000000..f5f879fc --- /dev/null +++ b/converters/gsf/tests/fixtures/sales.gsf.yaml @@ -0,0 +1,96 @@ +version: '1.0' +model: + name: sales + database: analytics + description: Sales model + ai_context: + instructions: Use approved metrics +terms: +- name: orders + source: + database: analytics + schema: public + table: orders + description: Orders + synonyms: + - purchases + primary_key: + - order_id + column_attributes: + - name: order_id + source_column: order_id + expressions: + - dialect: ANSI_SQL + expression: order_id + dimension: + is_time: false + - name: customer_id + source_column: customer_id + expressions: + - dialect: ANSI_SQL + expression: customer_id + dimension: + is_time: false + - name: order_date + source_column: order_date + expressions: + - dialect: ANSI_SQL + expression: order_date + dimension: + is_time: true + sql_attributes: + - name: net_total + kind: field + expressions: + - dialect: ANSI_SQL + expression: subtotal - discount + sql: SELECT subtotal - discount AS "net_total" FROM "analytics"."public"."orders" + AS "orders" + table_refs: + - orders + dimension: + is_time: false + - name: revenue_per_customer + kind: metric + expressions: + - dialect: ANSI_SQL + expression: SUM(orders.subtotal) / COUNT(DISTINCT customers.customer_id) + - dialect: SNOWFLAKE + expression: SUM(orders.subtotal)::NUMBER / COUNT(DISTINCT customers.customer_id) + sql: SELECT SUM(orders.subtotal) / COUNT(DISTINCT customers.customer_id) AS "revenue_per_customer" + FROM "analytics"."public"."orders" AS "orders" JOIN "analytics"."public"."customers" + AS "customers" ON "orders"."customer_id" = "customers"."customer_id" + table_refs: + - orders + - customers + description: Revenue per customer +- name: customers + source: + database: analytics + schema: public + table: customers + primary_key: + - customer_id + column_attributes: + - name: customer_id + source_column: customer_id + expressions: + - dialect: ANSI_SQL + expression: customer_id + dimension: + is_time: false + - name: customer_name + source_column: name + expressions: + - dialect: ANSI_SQL + expression: name + dimension: + is_time: false +semantic_foreign_keys: +- name: orders_to_customers + from_term: orders + to_term: customers + from_columns: + - customer_id + to_columns: + - customer_id diff --git a/converters/gsf/tests/fixtures/sales.ossie.yaml b/converters/gsf/tests/fixtures/sales.ossie.yaml new file mode 100644 index 00000000..4e0bd6f2 --- /dev/null +++ b/converters/gsf/tests/fixtures/sales.ossie.yaml @@ -0,0 +1,80 @@ +version: 0.2.0.dev0 +semantic_model: +- name: sales + description: Sales model + ai_context: + instructions: Use approved metrics + datasets: + - name: orders + source: analytics.public.orders + primary_key: + - order_id + description: Orders + ai_context: + synonyms: + - purchases + fields: + - name: order_id + expression: + dialects: + - dialect: ANSI_SQL + expression: order_id + dimension: + is_time: false + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + dimension: + is_time: false + - name: order_date + expression: + dialects: + - dialect: ANSI_SQL + expression: order_date + dimension: + is_time: true + - name: net_total + expression: + dialects: + - dialect: ANSI_SQL + expression: subtotal - discount + dimension: + is_time: false + - name: customers + source: analytics.public.customers + primary_key: + - customer_id + fields: + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + dimension: + is_time: false + - name: customer_name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + dimension: + is_time: false + relationships: + - name: orders_to_customers + from: orders + to: customers + from_columns: + - customer_id + to_columns: + - customer_id + metrics: + - name: revenue_per_customer + description: Revenue per customer + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.subtotal) / COUNT(DISTINCT customers.customer_id) + - dialect: SNOWFLAKE + expression: SUM(orders.subtotal)::NUMBER / COUNT(DISTINCT customers.customer_id) diff --git a/converters/gsf/tests/test_converter.py b/converters/gsf/tests/test_converter.py new file mode 100644 index 00000000..220927a3 --- /dev/null +++ b/converters/gsf/tests/test_converter.py @@ -0,0 +1,296 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Tests for the offline Apache Ossie ↔ NVIDIA GSF converter.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from pathlib import Path +from typing import Any + +import pytest +import yaml + +from ossie_gsf.converter import ( + GSFConversionError, + _parse_source, + _simple_source_column, + convert_gsf_to_ossie, + convert_ossie_to_gsf, + main, +) + +OSSIE_VERSION = "0.2.0.dev0" +FIXTURES = Path(__file__).parent / "fixtures" +VALIDATOR = Path(__file__).resolve().parents[3] / "validation" / "validate.py" + + +def _ossie_yaml() -> str: + return (FIXTURES / "sales.ossie.yaml").read_text(encoding="utf-8") + + +def test_checked_in_fixture_pair_matches_conversion() -> None: + expected = yaml.safe_load((FIXTURES / "sales.gsf.yaml").read_text(encoding="utf-8")) + actual = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) + + assert actual == expected + + +def test_generated_ossie_passes_official_validation(tmp_path: Path) -> None: + gsf_yaml = (FIXTURES / "sales.gsf.yaml").read_text(encoding="utf-8") + output_path = tmp_path / "converted.ossie.yaml" + output_path.write_text( + convert_gsf_to_ossie(gsf_yaml), + encoding="utf-8", + ) + + result = subprocess.run( + [sys.executable, str(VALIDATOR), str(output_path)], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "Validation PASSED" in result.stdout + + +def test_ossie_to_gsf_is_offline_and_maps_graph_entities() -> None: + result = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) + + assert result["version"] == "1.0" + assert result["model"] == { + "name": "sales", + "database": "analytics", + "description": "Sales model", + "ai_context": {"instructions": "Use approved metrics"}, + } + terms = {term["name"]: term for term in result["terms"]} + assert len(terms["orders"]["column_attributes"]) == 3 + sql_attributes = { + attribute["name"]: attribute for attribute in terms["orders"]["sql_attributes"] + } + assert sql_attributes["net_total"]["kind"] == "field" + metric = sql_attributes["revenue_per_customer"] + assert metric["kind"] == "metric" + assert metric["table_refs"] == ["orders", "customers"] + assert " JOIN " in metric["sql"] + assert len(metric["expressions"]) == 2 + assert result["semantic_foreign_keys"][0] == { + "name": "orders_to_customers", + "from_term": "orders", + "to_term": "customers", + "from_columns": ["customer_id"], + "to_columns": ["customer_id"], + } + + +def test_gsf_to_ossie_round_trip_preserves_semantics() -> None: + gsf_yaml = convert_ossie_to_gsf(_ossie_yaml()) + result = yaml.safe_load(convert_gsf_to_ossie(gsf_yaml)) + + assert set(result) == {"version", "semantic_model"} + model = result["semantic_model"][0] + assert model["name"] == "sales" + datasets = {dataset["name"]: dataset for dataset in model["datasets"]} + assert datasets["orders"]["primary_key"] == ["order_id"] + assert datasets["orders"]["ai_context"]["synonyms"] == ["purchases"] + fields = {field["name"]: field for field in datasets["orders"]["fields"]} + assert ( + fields["net_total"]["expression"]["dialects"][0]["expression"] + == "subtotal - discount" + ) + metric = model["metrics"][0] + assert len(metric["expression"]["dialects"]) == 2 + extension = next( + item for item in metric["custom_extensions"] if item["vendor_name"] == "GSF" + ) + assert json.loads(extension["data"])["term"] == "orders" + assert model["relationships"][0]["from"] == "orders" + + +def test_gsf_sql_field_preserves_sql_and_multi_term_references() -> None: + gsf = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) + orders = next(term for term in gsf["terms"] if term["name"] == "orders") + field = next( + item for item in orders["sql_attributes"] if item["name"] == "net_total" + ) + field["table_refs"] = ["orders", "customers"] + field["sql"] = "SELECT custom_joined_value FROM orders JOIN customers" + + ossie = convert_gsf_to_ossie(yaml.safe_dump(gsf)) + round_trip = yaml.safe_load(convert_ossie_to_gsf(ossie)) + round_trip_orders = next( + term for term in round_trip["terms"] if term["name"] == "orders" + ) + round_trip_field = next( + item + for item in round_trip_orders["sql_attributes"] + if item["name"] == "net_total" + ) + + assert round_trip_field["table_refs"] == ["orders", "customers"] + assert round_trip_field["sql"] == field["sql"] + + +def test_ossie_extensions_round_trip_through_native_metadata() -> None: + root = yaml.safe_load(_ossie_yaml()) + root["semantic_model"][0]["custom_extensions"] = [ + {"vendor_name": "DBT", "data": '{"project": "analytics"}'} + ] + + native = yaml.safe_load(convert_ossie_to_gsf(yaml.safe_dump(root))) + extensions = native["model"]["metadata"]["apache_ossie"]["custom_extensions"] + assert extensions[0]["vendor_name"] == "DBT" + + ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) + assert ossie["semantic_model"][0]["custom_extensions"][0]["vendor_name"] == "DBT" + + +def test_gsf_to_ossie_allows_model_name_override() -> None: + output = convert_gsf_to_ossie( + convert_ossie_to_gsf(_ossie_yaml()), + model_name="renamed_sales", + ) + assert yaml.safe_load(output)["semantic_model"][0]["name"] == ("renamed_sales") + + +def test_disconnected_metric_fails_instead_of_cross_join() -> None: + root = yaml.safe_load(_ossie_yaml()) + root["semantic_model"][0]["relationships"] = [] + + with pytest.raises(GSFConversionError, match="disconnected dataset"): + convert_ossie_to_gsf(yaml.safe_dump(root)) + + +def test_duplicate_physical_column_mapping_is_rejected() -> None: + root = yaml.safe_load(_ossie_yaml()) + root["semantic_model"][0]["datasets"][0]["fields"].append( + { + "name": "alternate_order_id", + "expression": { + "dialects": [ + { + "dialect": "ANSI_SQL", + "expression": "order_id", + } + ] + }, + } + ) + + with pytest.raises(GSFConversionError, match="Multiple fields"): + convert_ossie_to_gsf(yaml.safe_dump(root)) + + +def test_multiple_catalog_databases_are_rejected() -> None: + root = yaml.safe_load(_ossie_yaml()) + root["semantic_model"][0]["datasets"][1]["source"] = "crm.public.customers" + + with pytest.raises(GSFConversionError, match="exactly one database"): + convert_ossie_to_gsf(yaml.safe_dump(root)) + + +def test_wrong_versions_are_rejected() -> None: + ossie = yaml.safe_load(_ossie_yaml()) + ossie["version"] = "0.1" + with pytest.raises(GSFConversionError, match="Unsupported Ossie"): + convert_ossie_to_gsf(yaml.safe_dump(ossie)) + + gsf = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) + gsf["version"] = "2.0" + with pytest.raises(GSFConversionError, match="Unsupported GSF"): + convert_gsf_to_ossie(yaml.safe_dump(gsf)) + + +@pytest.mark.parametrize( + ("source", "default_database", "expected"), + [ + ( + "analytics.public.orders", + None, + { + "database": "analytics", + "schema": "public", + "table": "orders", + }, + ), + ( + "public.orders", + "analytics", + { + "database": "analytics", + "schema": "public", + "table": "orders", + }, + ), + ( + { + "database": "analytics", + "schema": "public", + "table": "orders", + }, + None, + { + "database": "analytics", + "schema": "public", + "table": "orders", + }, + ), + ], +) +def test_parse_source( + source: Any, + default_database: str | None, + expected: dict[str, str], +) -> None: + assert _parse_source(source, default_database) == expected + + +@pytest.mark.parametrize( + ("expression", "expected"), + [ + ("order_id", "order_id"), + ("orders.order_id", "order_id"), + ("subtotal - discount", None), + ("UPPER(name)", None), + ], +) +def test_simple_source_column( + expression: str, + expected: str | None, +) -> None: + assert _simple_source_column(expression, "orders", "orders") == expected + + +def test_cli_converts_files( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + ossie_path = tmp_path / "model.yaml" + gsf_path = tmp_path / "model.gsf.yaml" + ossie_path.write_text(_ossie_yaml(), encoding="utf-8") + + main(["export", "-i", str(ossie_path), "-o", str(gsf_path)]) + assert yaml.safe_load(gsf_path.read_text())["version"] == "1.0" + + main(["import", "-i", str(gsf_path)]) + output = yaml.safe_load(capsys.readouterr().out) + assert output["version"] == OSSIE_VERSION diff --git a/converters/gsf/uv.lock b/converters/gsf/uv.lock new file mode 100644 index 00000000..ba54dc32 --- /dev/null +++ b/converters/gsf/uv.lock @@ -0,0 +1,335 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "apache-ossie-gsf" +version = "0.1.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "jsonschema" }, + { name = "pytest" }, + { name = "sqlglot" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] + +[package.metadata.requires-dev] +dev = [ + { name = "jsonschema", specifier = ">=4.26.0" }, + { name = "pytest", specifier = ">=8.0" }, + { name = "sqlglot", specifier = ">=30.12.0" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + +[[package]] +name = "sqlglot" +version = "30.13.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/62/6d5bd3169478b7f09e08ab3e50175d486a9e7f3a419b88fc9280ba564ab1/sqlglot-30.13.0.tar.gz", hash = "sha256:f0a6eb79de2fd6efe2689f8cf197caa4f08bfe77c7880315616fa4420b8ba2bf", size = 5932385, upload-time = "2026-07-20T20:16:54.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/2f/2076eca54f6a8ed1c86301bdb4bb2ae4181b0c1c4dbc041062ca997dc1b2/sqlglot-30.13.0-py3-none-any.whl", hash = "sha256:08f87ff7b052246d61b731628c8c2db0bc91f2c9e69f5ba68a1d160a9f5b49b1", size = 719120, upload-time = "2026-07-20T20:16:53.248Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] From a137d01060a9f90e8e64cddc3c99d9c534f209a3 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Wed, 22 Jul 2026 10:51:37 -0500 Subject: [PATCH 2/6] Use NVIDIA_GSF vendor identifier Make the extension namespace reflect the full product name while continuing to accept the legacy GSF identifier on input. --- converters/README.md | 2 +- converters/gsf/README.md | 4 ++-- converters/gsf/src/ossie_gsf/converter.py | 12 +++++++----- converters/gsf/tests/test_converter.py | 4 +++- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/converters/README.md b/converters/README.md index be287179..5c9a4d54 100644 --- a/converters/README.md +++ b/converters/README.md @@ -75,7 +75,7 @@ The Ossie specification currently defines extensions for the following vendors: | `DATABRICKS` | Databricks semantic layer | | `OMNI` | Omni semantic model | | `WISDOM` | WisdomAI domain | -| `GSF` | NVIDIA Generative Semantic Fabric standalone YAML | +| `NVIDIA_GSF` | NVIDIA Generative Semantic Fabric standalone YAML | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/gsf/README.md b/converters/gsf/README.md index e493fb10..b6350fb3 100644 --- a/converters/gsf/README.md +++ b/converters/gsf/README.md @@ -114,11 +114,11 @@ transactionally, validates SQL, and refreshes semantic embeddings. - Native GSF SQL attributes with `kind: attribute` have no unambiguous Ossie field/metric equivalent and fail GSF → Ossie conversion. - Metrics are attached to the first referenced dataset. Unqualified metrics in - multi-dataset models need a `GSF` custom extension containing + multi-dataset models need an `NVIDIA_GSF` custom extension containing `{"term": "dataset_name"}`. - Expression dialect variants are preserved in the standalone GSF file. - Ossie custom extensions and GSF metadata are preserved through metadata and - `GSF` custom-extension payloads. + `NVIDIA_GSF` custom-extension payloads. ## Tests diff --git a/converters/gsf/src/ossie_gsf/converter.py b/converters/gsf/src/ossie_gsf/converter.py index 70e79c40..4ce7b807 100644 --- a/converters/gsf/src/ossie_gsf/converter.py +++ b/converters/gsf/src/ossie_gsf/converter.py @@ -31,7 +31,8 @@ OSSIE_VERSION = "0.2.0.dev0" GSF_VERSION = "1.0" -GSF_VENDOR = "GSF" +NVIDIA_GSF_VENDOR = "NVIDIA_GSF" +GSF_VENDOR_ALIASES = {NVIDIA_GSF_VENDOR, "GSF"} _SIMPLE_COLUMN = re.compile( r"^(?:(?P[A-Za-z_][A-Za-z0-9_]*)\.)?" @@ -595,13 +596,14 @@ def _metric_term( return next(iter(datasets)) raise GSFConversionError( f"Metric {metric.get('name')!r} does not identify an owning dataset. " - 'Add a GSF extension with data {"term": "dataset_name"}.' + "Add an NVIDIA_GSF extension with data " + '{"term": "dataset_name"}.' ) def _gsf_extension_data(item: Mapping[str, Any]) -> dict[str, Any]: for extension in item.get("custom_extensions") or []: - if extension.get("vendor_name") != GSF_VENDOR: + if extension.get("vendor_name") not in GSF_VENDOR_ALIASES: continue try: data = json.loads(str(extension.get("data") or "{}")) @@ -790,7 +792,7 @@ def _copy_ossie_metadata( preserved_extensions = [ extension for extension in source.get("custom_extensions") or [] - if extension.get("vendor_name") != GSF_VENDOR + if extension.get("vendor_name") not in GSF_VENDOR_ALIASES ] if preserved_extensions: native_metadata = dict(target.get("metadata") or {}) @@ -822,7 +824,7 @@ def _native_extensions( if data: extensions.append( { - "vendor_name": GSF_VENDOR, + "vendor_name": NVIDIA_GSF_VENDOR, "data": json.dumps(data), } ) diff --git a/converters/gsf/tests/test_converter.py b/converters/gsf/tests/test_converter.py index 220927a3..f4d502e7 100644 --- a/converters/gsf/tests/test_converter.py +++ b/converters/gsf/tests/test_converter.py @@ -120,7 +120,9 @@ def test_gsf_to_ossie_round_trip_preserves_semantics() -> None: metric = model["metrics"][0] assert len(metric["expression"]["dialects"]) == 2 extension = next( - item for item in metric["custom_extensions"] if item["vendor_name"] == "GSF" + item + for item in metric["custom_extensions"] + if item["vendor_name"] == "NVIDIA_GSF" ) assert json.loads(extension["data"])["term"] == "orders" assert model["relationships"][0]["from"] == "orders" From 032dd57fa726b8fa1883c540f73376c2e7cb9fb1 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Mon, 27 Jul 2026 10:42:10 -0500 Subject: [PATCH 3/6] Emit GSF's native model interchange contract The converter previously targeted a provisional document shape. GSF now defines the interchange format in its model_interchange schema, so emit and consume that contract exactly: a data_layer catalog, a semantic_layer of terms, SQL attributes and custom analyses, and zones, all keyed by id. Ossie-origin entities get deterministic UUIDv5 ids so repeated conversions are stable, and a GSF-origin document keeps its live ids, catalog properties, SQL text and zones so a GSF round trip is lossless. Ossie edits stay authoritative: preserved SQL and relationship records are reused only while they still match the Ossie model. Date-part keywords are excluded from column extraction, since sqlglot parses the unit in DATEDIFF(day, ...) as an unqualified column and it would otherwise be imported as a physical catalog column. A GSF-sourced catalog is also treated as authoritative, so an unrecognized SQL identifier can never widen it. Verified against a live GSF graph: export, convert both directions and re-import reproduces 114 tables, 1717 columns, 17 terms and 33 SQL attributes with no ids lost or invented, and a second import is a no-op. --- converters/gsf/README.md | 130 +- converters/gsf/pyproject.toml | 2 +- converters/gsf/src/ossie_gsf/converter.py | 879 +------- .../gsf/src/ossie_gsf/native_converter.py | 1937 +++++++++++++++++ converters/gsf/tests/fixtures/sales.gsf.yaml | 231 +- converters/gsf/tests/test_converter.py | 650 ++++-- converters/gsf/uv.lock | 8 +- 7 files changed, 2685 insertions(+), 1152 deletions(-) create mode 100644 converters/gsf/src/ossie_gsf/native_converter.py diff --git a/converters/gsf/README.md b/converters/gsf/README.md index b6350fb3..2afc1fa3 100644 --- a/converters/gsf/README.md +++ b/converters/gsf/README.md @@ -19,25 +19,26 @@ # Apache Ossie ↔ NVIDIA GSF Converter -Offline conversion between Apache Ossie YAML and the standalone semantic-model -YAML format supported by [NVIDIA GSF](https://github.com/NVIDIA/GSF). No GSF, -Neo4j, database, or network connection is required. +Offline conversion between Apache Ossie YAML and NVIDIA GSF's native +`GsfModelDocument` YAML contract. Conversion itself does not require GSF, +Neo4j, a database, or network access. ## Mapping -| Apache Ossie | Standalone GSF YAML | +| Apache Ossie | Native GSF model document | |---|---| -| Semantic model | `model` | -| Dataset | `terms[]` | -| Physical field | `column_attributes[]` | -| Computed field | `sql_attributes[]` with `kind: field` | -| Metric | `sql_attributes[]` with `kind: metric` | -| Relationship | `semantic_foreign_keys[]` | -| Expression dialects | `expressions[]` | -| Dataset source | Term `source` mapping | - -Computed fields and metrics include generated full SQL and explicit -`table_refs`, which lets GSF validate and attach them to its ingested catalog. +| Dataset source | `data_layer.databases[].schemas[].tables[]` | +| Dataset field backed by one column | `semantic_layer.terms[].columns_attributes[]` | +| Computed dataset field | `semantic_layer.sql_attributes.manual[]` | +| Model-level metric | `semantic_layer.custom_analyses[]` | +| Relationship | data-layer `joins` and `foreign_keys`, plus `semantic_fks` when possible | +| Dataset | term that `represents` exactly one catalog table | + +The generated root contains exactly `data_layer`, `semantic_layer`, and +`zones`. It does not contain a converter-specific version or model envelope. +Catalog columns are collected from fields, primary and unique keys, +relationships, and SQL column references. Stable UUIDv5 identifiers make +repeated Ossie exports deterministic. ## Setup @@ -55,18 +56,14 @@ uv run ossie-gsf export \ --database-name tpcds ``` -`--database-name` supplies the database for sources written as `schema.table`. -Fully qualified `database.schema.table` sources do not require it. - -Python: +`--database-name` supplies the database for `schema.table` sources. Fully +qualified `database.schema.table` sources do not require it. One document may +contain multiple databases. ```python from ossie_gsf import convert_ossie_to_gsf -gsf_yaml = convert_ossie_to_gsf( - ossie_yaml, - database_name="tpcds", -) +gsf_yaml = convert_ossie_to_gsf(ossie_yaml, database_name="tpcds") ``` ## GSF → Ossie @@ -74,51 +71,67 @@ gsf_yaml = convert_ossie_to_gsf( ```bash uv run ossie-gsf import \ --input tpcds.gsf.yaml \ - --output semantic_model.yaml + --output semantic_model.yaml \ + --name tpcds ``` -Use `--name` to override the exported Ossie semantic-model name. - -Python: +`--name` overrides the Ossie model name. Without it, the converter uses the +single catalog database name when there is one, otherwise `gsf_model`. ```python from ossie_gsf import convert_gsf_to_ossie -ossie_yaml = convert_gsf_to_ossie(gsf_yaml) +ossie_yaml = convert_gsf_to_ossie(gsf_yaml, model_name="tpcds") ``` -## Loading the converted model into GSF +This converter subset currently accepts only GSF terms that represent exactly +one table, because one Ossie dataset cannot represent several physical tables. +Several terms may represent the same table and become distinct Ossie datasets +sharing one source. SQL attributes become Ossie fields regardless of their GSF +source group. Custom analyses remain global by becoming model-level Ossie +metrics. Relationships are recovered from joins, then physical foreign keys, +then semantic foreign keys. + +## Importing the model into GSF -Run the native GSF importer from the GSF repository: +Start GSF, then send the native document to its REST API: ```bash -uv run python -m gsf.semantic import \ - --database-name tpcds \ - --input tpcds.gsf.yaml +curl --fail-with-body \ + -X POST \ + 'http://127.0.0.1:3001/api/model/import?replace=true&embed=true' \ + -H 'Content-Type: application/x-yaml' \ + --data-binary @tpcds.gsf.yaml ``` -GSF resolves the file against its existing catalog, writes the graph -transactionally, validates SQL, and refreshes semantic embeddings. - -## Conversion behavior and limitations - -- Apache Ossie `0.2.0.dev0` and GSF model-file `1.0` are supported. -- One semantic model is converted per document. -- GSF term sources must resolve to physical `database.schema.table` names. -- Simple field expressions become `ColumnAttribute` mappings. Other field - expressions become SQL attributes. -- Multi-dataset metric SQL is joined through declared Ossie relationships. - Disconnected datasets fail conversion instead of producing a cross join. -- GSF SQL attribute names are global, so duplicate computed-field or metric - names fail conversion. -- Native GSF SQL attributes with `kind: attribute` have no unambiguous Ossie - field/metric equivalent and fail GSF → Ossie conversion. -- Metrics are attached to the first referenced dataset. Unqualified metrics in - multi-dataset models need an `NVIDIA_GSF` custom extension containing - `{"term": "dataset_name"}`. -- Expression dialect variants are preserved in the standalone GSF file. -- Ossie custom extensions and GSF metadata are preserved through metadata and - `NVIDIA_GSF` custom-extension payloads. +The endpoint also accepts a multipart upload in a `file` field. + +The target GSF instance must already have a connection configured for each +database named in the document. GSF validates every imported SQL attribute +against that connection's dialect, so importing into an instance with no +matching connection fails. A database's `dialect` is likewise derived from the +live connection rather than stored on import, so it is exported for information +only and does not survive a GSF → GSF cycle. + +## Fidelity and unavoidable losses + +When converting GSF to Ossie, the converter records the native document in an +`NVIDIA_GSF` custom extension. A direct GSF → Ossie → GSF cycle can therefore +reuse live identifiers and preserve catalog properties, SQL source groups, +SQL text, `sql_column_is`, relationships, and zones. Ossie-origin entities use +deterministic IDs when no preserved native ID is available. Current Ossie +expressions and relationships remain authoritative: preserved SQL and native +relationship records are reused only when they still correspond to the Ossie +entities or are outside the represented Ossie catalog scope. + +The GSF contract has no semantic-model envelope, `ai_context`, dimensions, +synonyms, Ossie custom-extension storage, or expression-dialect variants. +Those values cannot be represented in a native GSF document and are +unavoidably lost on Ossie → GSF. GSF joins also have no relationship name, so +GSF → Ossie synthesizes a stable `_to_` name. The converter never +adds fictional fields to the GSF schema. GSF records uniqueness per column, so +Ossie composite unique keys cannot be reconstructed after GSF → Ossie; only +single-column unique keys survive. ## Tests @@ -126,6 +139,7 @@ transactionally, validates SQL, and refreshes semantic embeddings. uv run pytest ``` -The suite includes a checked-in Ossie/GSF fixture pair, bidirectional -round-trip checks, validation failures, CLI coverage, and verification of -generated Ossie YAML with the repository's official validator. +The suite checks the exact native root shape, deterministic and resolvable +IDs, official Ossie validation, semantic round trips, native metadata +preservation, multiple databases, relationships, input validation, and CLI +behavior. diff --git a/converters/gsf/pyproject.toml b/converters/gsf/pyproject.toml index c50c99ad..0ea22c40 100644 --- a/converters/gsf/pyproject.toml +++ b/converters/gsf/pyproject.toml @@ -23,7 +23,6 @@ build-backend = "hatchling.build" dev = [ "jsonschema>=4.26.0", "pytest>=8.0", - "sqlglot>=30.12.0", ] [project] @@ -43,6 +42,7 @@ keywords = [ ] dependencies = [ "PyYAML>=6.0", + "sqlglot>=30.12.0", ] [project.scripts] diff --git a/converters/gsf/src/ossie_gsf/converter.py b/converters/gsf/src/ossie_gsf/converter.py index 4ce7b807..0b0e4811 100644 --- a/converters/gsf/src/ossie_gsf/converter.py +++ b/converters/gsf/src/ossie_gsf/converter.py @@ -15,876 +15,21 @@ # specific language governing permissions and limitations # under the License. -"""Offline Apache Ossie ↔ standalone NVIDIA GSF YAML conversion.""" +"""Public API and CLI for native NVIDIA GSF model conversion.""" -from __future__ import annotations - -import argparse -import json -import re -import sys -from collections import Counter -from pathlib import Path -from typing import Any, Mapping - -import yaml - -OSSIE_VERSION = "0.2.0.dev0" -GSF_VERSION = "1.0" -NVIDIA_GSF_VENDOR = "NVIDIA_GSF" -GSF_VENDOR_ALIASES = {NVIDIA_GSF_VENDOR, "GSF"} - -_SIMPLE_COLUMN = re.compile( - r"^(?:(?P[A-Za-z_][A-Za-z0-9_]*)\.)?" - r"(?P[A-Za-z_][A-Za-z0-9_]*)$" -) -_TERM_REFERENCE = re.compile( - r"\b(?P[A-Za-z_][A-Za-z0-9_]*)\." - r"(?P[A-Za-z_][A-Za-z0-9_]*)\b" +from .native_converter import ( + GSFConversionError, + convert_gsf_to_ossie, + convert_ossie_to_gsf, + main, ) - -class GSFConversionError(Exception): - """Raised when a document cannot be converted safely.""" - - -def convert_ossie_to_gsf( - ossie_yaml: str, - *, - database_name: str | None = None, -) -> str: - """Convert Apache Ossie YAML to standalone GSF semantic-model YAML.""" - root, model = _parse_ossie(ossie_yaml) - source_datasets = model.get("datasets") or [] - if not isinstance(source_datasets, list) or not source_datasets: - raise GSFConversionError( - "The Ossie semantic model must contain at least one dataset" - ) - - datasets: dict[str, dict[str, Any]] = {} - terms: list[dict[str, Any]] = [] - resolved_databases: set[str] = set() - for dataset in source_datasets: - if not isinstance(dataset, dict) or not dataset.get("name"): - raise GSFConversionError( - "Every Ossie dataset must be a mapping with a name" - ) - name = str(dataset["name"]) - if name in datasets: - raise GSFConversionError(f"Duplicate dataset name {name!r}") - source = _parse_source(dataset.get("source"), database_name) - if not source["database"] or not source["schema"]: - raise GSFConversionError( - f"Dataset {name!r} source must resolve to database.schema.table" - ) - resolved_databases.add(str(source["database"])) - datasets[name] = {"source": source, "original": dataset} - - term: dict[str, Any] = { - "name": name, - "source": source, - } - _copy_optional(dataset, term, "description") - ai_context = dataset.get("ai_context") - if isinstance(ai_context, dict): - cleaned_ai_context = { - key: value for key, value in ai_context.items() if key != "synonyms" - } - if cleaned_ai_context: - term["ai_context"] = cleaned_ai_context - elif ai_context is not None: - term["ai_context"] = ai_context - synonyms = _synonyms(ai_context) - if synonyms: - term["synonyms"] = synonyms - _copy_optional(dataset, term, "primary_key") - _copy_optional(dataset, term, "unique_keys") - _copy_ossie_metadata(dataset, term) - terms.append(term) - - if len(resolved_databases) != 1: - raise GSFConversionError( - "One GSF model file must target exactly one database; found: " - + ", ".join(sorted(resolved_databases)) - ) - resolved_database = next(iter(resolved_databases)) - - relationships = [ - _relationship_to_gsf(relationship, datasets) - for relationship in model.get("relationships") or [] - ] - term_by_name = {term["name"]: term for term in terms} - sql_names: list[str] = [] - - for dataset_name, context in datasets.items(): - dataset = context["original"] - term = term_by_name[dataset_name] - column_attributes: list[dict[str, Any]] = [] - sql_attributes: list[dict[str, Any]] = [] - used_columns: set[str] = set() - - for field in dataset.get("fields") or []: - if not isinstance(field, dict) or not field.get("name"): - raise GSFConversionError( - f"Every field in dataset {dataset_name!r} needs a name" - ) - name = str(field["name"]) - expressions = _normalize_expressions(field.get("expression"), name) - selected = _pick_expression(expressions, name) - source_column = _simple_source_column( - selected, - dataset_name, - str(context["source"]["table"]), - ) - if source_column is not None: - if source_column in used_columns: - raise GSFConversionError( - f"Multiple fields in dataset {dataset_name!r} map to " - f"physical column {source_column!r}" - ) - used_columns.add(source_column) - attribute: dict[str, Any] = { - "name": name, - "source_column": source_column, - "expressions": expressions, - } - _copy_optional(field, attribute, "description") - _copy_optional(field, attribute, "ai_context") - _copy_optional(field, attribute, "dimension") - _copy_ossie_metadata(field, attribute) - column_attributes.append(attribute) - else: - extension_data = _gsf_extension_data(field) - refs = extension_data.get("table_refs") or [dataset_name] - if not isinstance(refs, list) or any( - ref not in datasets for ref in refs - ): - raise GSFConversionError( - f"Computed field {name!r} has invalid GSF table_refs" - ) - sources = [(str(ref), datasets[str(ref)]["source"]) for ref in refs] - relevant_relationships = [ - relationship - for relationship in relationships - if relationship["from_term"] in refs - or relationship["to_term"] in refs - ] - attribute = { - "name": name, - "kind": "field", - "expressions": expressions, - "sql": str(extension_data.get("sql") or "") - or _wrap_expression( - selected, name, sources, relevant_relationships - ), - "table_refs": list(refs), - } - _copy_optional(field, attribute, "description") - _copy_optional(field, attribute, "ai_context") - _copy_optional(field, attribute, "dimension") - _copy_ossie_metadata(field, attribute) - sql_attributes.append(attribute) - sql_names.append(name) - - if column_attributes: - term["column_attributes"] = column_attributes - if sql_attributes: - term["sql_attributes"] = sql_attributes - - for metric in model.get("metrics") or []: - if not isinstance(metric, dict) or not metric.get("name"): - raise GSFConversionError("Every Ossie metric must be a mapping with a name") - name = str(metric["name"]) - expressions = _normalize_expressions(metric.get("expression"), name) - selected = _pick_expression(expressions, name) - referenced_terms = _referenced_terms(selected, datasets) - owner = _metric_term(metric, referenced_terms, datasets) - refs = referenced_terms or [owner] - sources = [(term_name, datasets[term_name]["source"]) for term_name in refs] - relevant_relationships = [ - relationship - for relationship in relationships - if relationship["from_term"] in refs or relationship["to_term"] in refs - ] - extension_data = _gsf_extension_data(metric) - preserved_refs = extension_data.get("table_refs") - if isinstance(preserved_refs, list) and preserved_refs: - if any(ref not in datasets for ref in preserved_refs): - raise GSFConversionError(f"Metric {name!r} has invalid GSF table_refs") - refs = [str(ref) for ref in preserved_refs] - sources = [(term_name, datasets[term_name]["source"]) for term_name in refs] - relevant_relationships = [ - relationship - for relationship in relationships - if relationship["from_term"] in refs or relationship["to_term"] in refs - ] - attribute = { - "name": name, - "kind": "metric", - "expressions": expressions, - "sql": str(extension_data.get("sql") or "") - or _wrap_expression(selected, name, sources, relevant_relationships), - "table_refs": refs, - } - _copy_optional(metric, attribute, "description") - _copy_optional(metric, attribute, "ai_context") - _copy_ossie_metadata(metric, attribute) - term_by_name[owner].setdefault("sql_attributes", []).append(attribute) - sql_names.append(name) - - duplicate_sql = sorted( - name for name, count in Counter(sql_names).items() if count > 1 - ) - if duplicate_sql: - raise GSFConversionError( - "GSF SqlAttribute names are global; duplicate computed field or " - "metric names: " + ", ".join(duplicate_sql) - ) - - gsf_model: dict[str, Any] = { - "name": str(model["name"]), - "database": resolved_database, - } - _copy_optional(model, gsf_model, "description") - _copy_optional(model, gsf_model, "ai_context") - _copy_ossie_metadata(model, gsf_model) - - output: dict[str, Any] = { - "version": GSF_VERSION, - "model": gsf_model, - "terms": terms, - } - if relationships: - output["semantic_foreign_keys"] = relationships - return _dump_yaml(output) - - -def convert_gsf_to_ossie( - gsf_yaml: str, - *, - model_name: str | None = None, -) -> str: - """Convert standalone GSF semantic-model YAML to Apache Ossie YAML.""" - root = _parse_gsf(gsf_yaml) - source_model = root["model"] - terms = root["terms"] - term_names = { - str(term.get("name")) - for term in terms - if isinstance(term, dict) and term.get("name") - } - if len(term_names) != len(terms): - raise GSFConversionError("Every GSF term needs a unique non-empty name") - - datasets: list[dict[str, Any]] = [] - metrics: list[dict[str, Any]] = [] - for term in terms: - term_name = str(term["name"]) - source = _parse_source( - term.get("source"), - source_model.get("database"), - ) - if not source["database"] or not source["schema"]: - raise GSFConversionError( - f"Term {term_name!r} source must be fully qualified" - ) - dataset: dict[str, Any] = { - "name": term_name, - "source": ".".join( - str(source[key]) for key in ("database", "schema", "table") - ), - } - _copy_optional(term, dataset, "description") - _copy_optional(term, dataset, "primary_key") - _copy_optional(term, dataset, "unique_keys") - ai_context = term.get("ai_context") - synonyms = term.get("synonyms") or [] - merged_ai = _merge_synonyms(ai_context, synonyms) - if merged_ai is not None: - dataset["ai_context"] = merged_ai - dataset_extensions = _native_extensions(term) - if dataset_extensions: - dataset["custom_extensions"] = dataset_extensions - - fields: list[dict[str, Any]] = [] - for attribute in term.get("column_attributes") or []: - _validate_attribute(attribute, term_name, kind="column") - field: dict[str, Any] = { - "name": str(attribute["name"]), - "expression": { - "dialects": attribute.get("expressions") - or [ - { - "dialect": "ANSI_SQL", - "expression": str(attribute["source_column"]), - } - ] - }, - } - _copy_optional(attribute, field, "description") - _copy_optional(attribute, field, "ai_context") - _copy_optional(attribute, field, "dimension") - field_extensions = _native_extensions(attribute) - if field_extensions: - field["custom_extensions"] = field_extensions - fields.append(field) - - for attribute in term.get("sql_attributes") or []: - _validate_attribute(attribute, term_name, kind="sql") - sql_kind = attribute.get("kind") - if sql_kind not in {"field", "metric"}: - raise GSFConversionError( - f"SQL attribute {attribute['name']!r} kind must be " - "'field' or 'metric'" - ) - expression = { - "dialects": _normalize_native_expressions( - attribute.get("expressions"), - str(attribute["name"]), - ) - } - item: dict[str, Any] = { - "name": str(attribute["name"]), - "expression": expression, - } - _copy_optional(attribute, item, "description") - _copy_optional(attribute, item, "ai_context") - if sql_kind == "field": - _copy_optional(attribute, item, "dimension") - item["custom_extensions"] = _native_extensions( - attribute, - gsf_data={ - "kind": "field", - "sql": attribute.get("sql"), - "table_refs": attribute.get("table_refs"), - }, - ) - fields.append(item) - else: - item["custom_extensions"] = _native_extensions( - attribute, - gsf_data={ - "term": term_name, - "sql": attribute.get("sql"), - "table_refs": attribute.get("table_refs"), - }, - ) - metrics.append(item) - - if fields: - dataset["fields"] = fields - datasets.append(dataset) - - relationships: list[dict[str, Any]] = [] - for relationship in root.get("semantic_foreign_keys") or []: - if not isinstance(relationship, dict) or not relationship.get("name"): - raise GSFConversionError("Every semantic foreign key must have a name") - from_term = relationship.get("from_term") - to_term = relationship.get("to_term") - if from_term not in term_names or to_term not in term_names: - raise GSFConversionError( - f"Semantic foreign key {relationship['name']!r} references " - "an unknown term" - ) - from_columns = relationship.get("from_columns") or [] - to_columns = relationship.get("to_columns") or [] - if not from_columns or len(from_columns) != len(to_columns): - raise GSFConversionError( - f"Semantic foreign key {relationship['name']!r} must have " - "equal, non-empty column lists" - ) - converted_relationship: dict[str, Any] = { - "name": str(relationship["name"]), - "from": from_term, - "to": to_term, - "from_columns": list(from_columns), - "to_columns": list(to_columns), - } - _copy_optional( - relationship, - converted_relationship, - "ai_context", - ) - relationship_extensions = _native_extensions(relationship) - if relationship_extensions: - converted_relationship["custom_extensions"] = relationship_extensions - relationships.append(converted_relationship) - - semantic_model: dict[str, Any] = { - "name": model_name or str(source_model["name"]), - "datasets": datasets, - } - _copy_optional(source_model, semantic_model, "description") - _copy_optional(source_model, semantic_model, "ai_context") - model_extensions = _native_extensions(source_model) - if model_extensions: - semantic_model["custom_extensions"] = model_extensions - if relationships: - semantic_model["relationships"] = relationships - if metrics: - semantic_model["metrics"] = metrics - return _dump_yaml( - { - "version": OSSIE_VERSION, - "semantic_model": [semantic_model], - } - ) - - -def _parse_ossie(value: str) -> tuple[dict[str, Any], dict[str, Any]]: - root = _load_yaml(value, "Ossie") - unknown = sorted(set(root) - {"version", "semantic_model"}) - if unknown: - raise GSFConversionError( - "Unsupported Ossie root properties: " + ", ".join(unknown) - ) - if str(root.get("version", "")) != OSSIE_VERSION: - raise GSFConversionError( - f"Unsupported Ossie version {root.get('version')!r}; " - f"supported version is {OSSIE_VERSION!r}" - ) - models = root.get("semantic_model") - if not isinstance(models, list) or len(models) != 1: - raise GSFConversionError("Ossie input must contain exactly one semantic model") - model = models[0] - if not isinstance(model, dict) or not model.get("name"): - raise GSFConversionError("Ossie semantic model requires a name") - return root, model - - -def _parse_gsf(value: str) -> dict[str, Any]: - root = _load_yaml(value, "GSF") - unknown = sorted(set(root) - {"version", "model", "terms", "semantic_foreign_keys"}) - if unknown: - raise GSFConversionError( - "Unsupported GSF root properties: " + ", ".join(unknown) - ) - if str(root.get("version", "")) != GSF_VERSION: - raise GSFConversionError( - f"Unsupported GSF version {root.get('version')!r}; " - f"supported version is {GSF_VERSION!r}" - ) - model = root.get("model") - if not isinstance(model, dict) or not model.get("name"): - raise GSFConversionError("GSF model requires 'model.name'") - terms = root.get("terms") - if not isinstance(terms, list) or not terms: - raise GSFConversionError("'terms' must be a non-empty list") - relationships = root.get("semantic_foreign_keys", []) - if not isinstance(relationships, list): - raise GSFConversionError("'semantic_foreign_keys' must be a list") - return root - - -def _load_yaml(value: str, label: str) -> dict[str, Any]: - try: - root = yaml.safe_load(value) - except yaml.YAMLError as exc: - raise GSFConversionError(f"Invalid {label} YAML: {exc}") from exc - if not isinstance(root, dict): - raise GSFConversionError(f"Invalid {label} YAML: expected a root mapping") - return root - - -def _relationship_to_gsf( - relationship: Any, - datasets: Mapping[str, Any], -) -> dict[str, Any]: - if not isinstance(relationship, dict) or not relationship.get("name"): - raise GSFConversionError("Every Ossie relationship needs a name") - from_term = relationship.get("from") - to_term = relationship.get("to") - if from_term not in datasets or to_term not in datasets: - raise GSFConversionError( - f"Relationship {relationship['name']!r} references an unknown dataset" - ) - from_columns = relationship.get("from_columns") or [] - to_columns = relationship.get("to_columns") or [] - if not from_columns or len(from_columns) != len(to_columns): - raise GSFConversionError( - f"Relationship {relationship['name']!r} must have equal, " - "non-empty column lists" - ) - converted: dict[str, Any] = { - "name": str(relationship["name"]), - "from_term": from_term, - "to_term": to_term, - "from_columns": list(from_columns), - "to_columns": list(to_columns), - } - _copy_optional(relationship, converted, "ai_context") - _copy_ossie_metadata(relationship, converted) - return converted - - -def _normalize_expressions(value: Any, name: str) -> list[dict[str, str]]: - if not isinstance(value, dict): - raise GSFConversionError(f"{name!r} has no valid expression") - return _normalize_native_expressions(value.get("dialects"), name) - - -def _normalize_native_expressions( - value: Any, - name: str, -) -> list[dict[str, str]]: - if not isinstance(value, list) or not value: - raise GSFConversionError(f"{name!r} requires at least one expression dialect") - result: list[dict[str, str]] = [] - for item in value: - if ( - isinstance(item, dict) - and item.get("dialect") - and item.get("expression") is not None - ): - result.append( - { - "dialect": str(item["dialect"]), - "expression": str(item["expression"]), - } - ) - if not result: - raise GSFConversionError(f"{name!r} has no usable expression dialect") - return result - - -def _pick_expression(expressions: list[dict[str, str]], name: str) -> str: - for expression in expressions: - if expression["dialect"].upper() == "ANSI_SQL": - return expression["expression"] - if expressions: - return expressions[0]["expression"] - raise GSFConversionError(f"{name!r} has no usable expression") - - -def _simple_source_column( - expression: str, - dataset_name: str, - table_name: str, -) -> str | None: - match = _SIMPLE_COLUMN.fullmatch(expression.strip()) - if not match: - return None - qualifier = match.group("qualifier") - if qualifier and qualifier not in (dataset_name, table_name): - return None - return match.group("column") - - -def _referenced_terms( - expression: str, - datasets: Mapping[str, Any], -) -> list[str]: - result: list[str] = [] - for match in _TERM_REFERENCE.finditer(expression): - name = match.group("term") - if name in datasets and name not in result: - result.append(name) - return result - - -def _metric_term( - metric: dict[str, Any], - referenced_terms: list[str], - datasets: Mapping[str, Any], -) -> str: - term = _gsf_extension_data(metric).get("term") - if term in datasets: - return str(term) - if referenced_terms: - return referenced_terms[0] - if len(datasets) == 1: - return next(iter(datasets)) - raise GSFConversionError( - f"Metric {metric.get('name')!r} does not identify an owning dataset. " - "Add an NVIDIA_GSF extension with data " - '{"term": "dataset_name"}.' - ) - - -def _gsf_extension_data(item: Mapping[str, Any]) -> dict[str, Any]: - for extension in item.get("custom_extensions") or []: - if extension.get("vendor_name") not in GSF_VENDOR_ALIASES: - continue - try: - data = json.loads(str(extension.get("data") or "{}")) - except json.JSONDecodeError: - continue - if isinstance(data, dict): - return data - return {} - - -def _wrap_expression( - expression: str, - name: str, - sources: list[tuple[str, Mapping[str, Any]]], - relationships: list[dict[str, Any]], -) -> str: - stripped = expression.strip() - if stripped.upper().startswith(("SELECT ", "SELECT\n", "WITH ", "WITH\n")): - return stripped - if not sources: - raise GSFConversionError(f"Cannot determine a source table for {name!r}") - source_map = dict(sources) - anchor_name, anchor = sources[0] - from_sql = f"{_qualified_table(anchor)} AS {_quote_identifier(anchor_name)}" - joined = {anchor_name} - remaining = {term_name for term_name, _ in sources[1:]} - joins: list[str] = [] - while remaining: - matched = False - for relationship in relationships: - left = str(relationship["from_term"]) - right = str(relationship["to_term"]) - if left in joined and right in remaining: - new_name = right - elif right in joined and left in remaining: - new_name = left - else: - continue - conditions = [ - f"{_quote_identifier(left)}.{_quote_identifier(left_column)} = " - f"{_quote_identifier(right)}." - f"{_quote_identifier(right_column)}" - for left_column, right_column in zip( - relationship["from_columns"], - relationship["to_columns"], - strict=True, - ) - ] - joins.append( - f"JOIN {_qualified_table(source_map[new_name])} AS " - f"{_quote_identifier(new_name)} ON {' AND '.join(conditions)}" - ) - joined.add(new_name) - remaining.remove(new_name) - matched = True - break - if not matched: - missing = sorted(remaining)[0] - raise GSFConversionError( - f"{name!r} references disconnected dataset {missing!r}; " - "declare a relationship connecting all referenced datasets" - ) - return f"SELECT {stripped} AS {_quote_identifier(name)} FROM {from_sql}" + ( - f" {' '.join(joins)}" if joins else "" - ) - - -def _parse_source( - source: Any, - default_database: str | None, -) -> dict[str, str | None]: - if isinstance(source, dict): - database = source.get("database") or default_database - schema = source.get("schema") - table = source.get("table") - if not table: - raise GSFConversionError("Source mapping requires 'table'") - return { - "database": str(database) if database else None, - "schema": str(schema) if schema else None, - "table": str(table), - } - value = str(source or "").strip() - if not value: - raise GSFConversionError("Every dataset/term needs a source") - upper = value.upper() - if upper.startswith(("SELECT ", "SELECT\n", "WITH ", "WITH\n")): - raise GSFConversionError("GSF term sources must identify physical tables") - parts = _split_identifier(value) - if len(parts) == 3: - database, schema, table = parts - elif len(parts) == 2: - database, (schema, table) = default_database, parts - elif len(parts) == 1: - database, schema, table = default_database, None, parts[0] - else: - raise GSFConversionError( - f"Source {value!r} must be table, schema.table, or database.schema.table" - ) - return {"database": database, "schema": schema, "table": table} - - -def _split_identifier(value: str) -> list[str]: - parts: list[str] = [] - current: list[str] = [] - quote: str | None = None - for char in value: - if char in ('"', "`"): - quote = None if quote == char else char if quote is None else quote - elif char == "." and quote is None: - parts.append("".join(current).strip()) - current = [] - continue - current.append(char) - parts.append("".join(current).strip()) - return [ - part[1:-1] - if len(part) > 1 and part[0] == part[-1] and part[0] in ('"', "`") - else part - for part in parts - ] - - -def _qualified_table(source: Mapping[str, Any]) -> str: - return ".".join( - _quote_identifier(str(source[key])) for key in ("database", "schema", "table") - ) - - -def _quote_identifier(value: str) -> str: - return '"' + value.replace('"', '""') + '"' - - -def _validate_attribute( - attribute: Any, - term_name: str, - *, - kind: str, -) -> None: - if not isinstance(attribute, dict) or not attribute.get("name"): - raise GSFConversionError( - f"Every {kind} attribute in term {term_name!r} needs a name" - ) - if kind == "column" and not attribute.get("source_column"): - raise GSFConversionError( - f"Column attribute {attribute['name']!r} needs source_column" - ) - - -def _synonyms(ai_context: Any) -> list[str]: - if not isinstance(ai_context, dict): - return [] - return [str(value) for value in ai_context.get("synonyms") or [] if value] - - -def _merge_synonyms(ai_context: Any, synonyms: Any) -> Any: - clean = [str(value) for value in synonyms if value] - if not clean: - return ai_context - if isinstance(ai_context, dict): - result = dict(ai_context) - result["synonyms"] = clean - return result - if isinstance(ai_context, str) and ai_context: - return {"instructions": ai_context, "synonyms": clean} - return {"synonyms": clean} - - -def _copy_optional( - source: Mapping[str, Any], - target: dict[str, Any], - key: str, -) -> None: - if source.get(key) is not None: - target[key] = source[key] - - -def _copy_ossie_metadata( - source: Mapping[str, Any], - target: dict[str, Any], -) -> None: - extension_data = _gsf_extension_data(source) - metadata = extension_data.get("metadata") - if isinstance(metadata, dict): - target["metadata"] = dict(metadata) - preserved_extensions = [ - extension - for extension in source.get("custom_extensions") or [] - if extension.get("vendor_name") not in GSF_VENDOR_ALIASES - ] - if preserved_extensions: - native_metadata = dict(target.get("metadata") or {}) - ossie_metadata = dict(native_metadata.get("apache_ossie") or {}) - ossie_metadata["custom_extensions"] = preserved_extensions - native_metadata["apache_ossie"] = ossie_metadata - target["metadata"] = native_metadata - - -def _native_extensions( - source: Mapping[str, Any], - *, - gsf_data: dict[str, Any] | None = None, -) -> list[dict[str, Any]]: - metadata = source.get("metadata") - extensions: list[dict[str, Any]] = [] - if isinstance(metadata, dict): - ossie_metadata = metadata.get("apache_ossie") - if isinstance(ossie_metadata, dict): - extensions.extend(ossie_metadata.get("custom_extensions") or []) - remaining_metadata = { - key: value for key, value in metadata.items() if key != "apache_ossie" - } - else: - remaining_metadata = {} - data = dict(gsf_data or {}) - if remaining_metadata: - data["metadata"] = remaining_metadata - if data: - extensions.append( - { - "vendor_name": NVIDIA_GSF_VENDOR, - "data": json.dumps(data), - } - ) - return extensions - - -def _dump_yaml(value: dict[str, Any]) -> str: - return yaml.safe_dump( - value, - default_flow_style=False, - sort_keys=False, - allow_unicode=True, - ) - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description="Convert Apache Ossie and standalone GSF YAML files" - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - export_parser = subparsers.add_parser( - "export", - help="Convert Ossie YAML to standalone GSF YAML", - ) - export_parser.add_argument("-i", "--input", type=Path, required=True) - export_parser.add_argument("-o", "--output", type=Path) - export_parser.add_argument("--database-name") - - import_parser = subparsers.add_parser( - "import", - help="Convert standalone GSF YAML to Ossie YAML", - ) - import_parser.add_argument("-i", "--input", type=Path, required=True) - import_parser.add_argument("-o", "--output", type=Path) - import_parser.add_argument("--name") - return parser - - -def main(argv: list[str] | None = None) -> None: - args = _build_parser().parse_args(argv) - try: - source = args.input.read_text(encoding="utf-8") - if args.command == "export": - output = convert_ossie_to_gsf( - source, - database_name=args.database_name, - ) - else: - output = convert_gsf_to_ossie( - source, - model_name=args.name, - ) - if args.output is None: - print(output, end="") - else: - args.output.write_text(output, encoding="utf-8") - except (GSFConversionError, OSError, UnicodeError) as exc: - print(f"Error: {exc}", file=sys.stderr) - raise SystemExit(1) from exc +__all__ = [ + "GSFConversionError", + "convert_gsf_to_ossie", + "convert_ossie_to_gsf", + "main", +] if __name__ == "__main__": diff --git a/converters/gsf/src/ossie_gsf/native_converter.py b/converters/gsf/src/ossie_gsf/native_converter.py new file mode 100644 index 00000000..b5eb2b4b --- /dev/null +++ b/converters/gsf/src/ossie_gsf/native_converter.py @@ -0,0 +1,1937 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Apache Ossie ↔ native NVIDIA GSF model-document conversion.""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from collections import defaultdict +from collections.abc import Iterable, Mapping +from copy import deepcopy +from pathlib import Path +from typing import Any +from uuid import UUID, uuid5 + +import yaml +from sqlglot import exp, parse_one +from sqlglot.errors import ParseError + +OSSIE_VERSION = "0.2.0.dev0" +NVIDIA_GSF_VENDOR = "NVIDIA_GSF" +GSF_VENDOR_ALIASES = {NVIDIA_GSF_VENDOR, "GSF"} +_ID_NAMESPACE = UUID("03d14261-6432-50fe-b099-77e8061af4f9") +_SQL_GROUPS = ("manual", "table", "sql", "bridge_table") +_SIMPLE_COLUMN = re.compile( + r"^(?:(?P[A-Za-z_][A-Za-z0-9_]*)\.)?" + r"(?P[A-Za-z_][A-Za-z0-9_]*)$" +) +_DATE_PART_UNITS = frozenset( + { + "year", + "years", + "yy", + "yyyy", + "quarter", + "quarters", + "qq", + "q", + "month", + "months", + "mm", + "mon", + "week", + "weeks", + "wk", + "ww", + "isoweek", + "day", + "days", + "dd", + "dayofyear", + "doy", + "dy", + "dayofweek", + "dow", + "weekday", + "hour", + "hours", + "hh", + "minute", + "minutes", + "mi", + "second", + "seconds", + "ss", + "millisecond", + "milliseconds", + "ms", + "microsecond", + "microseconds", + "us", + "nanosecond", + "nanoseconds", + "ns", + "epoch", + } +) +_DATE_FUNCTIONS = frozenset( + { + "datediff", + "date_diff", + "datetime_diff", + "timestampdiff", + "timestamp_diff", + "timediff", + "time_diff", + "dateadd", + "date_add", + "datetime_add", + "timestampadd", + "timestamp_add", + "datesub", + "date_sub", + "timestampsub", + "timestamp_sub", + "date_trunc", + "datetrunc", + "datetime_trunc", + "timestamp_trunc", + "time_trunc", + "date_part", + "datepart", + "datename", + "extract", + "last_day", + "trunc", + } +) + + +class GSFConversionError(Exception): + """Raised when a document cannot be converted safely.""" + + +def convert_ossie_to_gsf( + ossie_yaml: str, + *, + database_name: str | None = None, +) -> str: + """Convert one Apache Ossie model to a native ``GsfModelDocument``.""" + _, model = _parse_ossie(ossie_yaml) + source_datasets = model.get("datasets") or [] + if not isinstance(source_datasets, list) or not source_datasets: + raise GSFConversionError( + "The Ossie semantic model must contain at least one dataset" + ) + + native = _native_snapshot(model) + preserved = _index_native_document(native) + datasets: dict[str, dict[str, Any]] = {} + for item in source_datasets: + if not isinstance(item, dict) or not item.get("name"): + raise GSFConversionError( + "Every Ossie dataset must be a mapping with a name" + ) + name = str(item["name"]) + if name in datasets: + raise GSFConversionError(f"Duplicate dataset name {name!r}") + source = _parse_source(item.get("source"), database_name) + if not source["database"] or not source["schema"]: + raise GSFConversionError( + f"Dataset {name!r} source must resolve to database.schema.table" + ) + datasets[name] = { + "source": source, + "original": item, + "columns": [], + "column_set": set(), + "simple_fields": [], + "computed_fields": [], + } + preserved_table = preserved["tables"].get(_source_key(source), {}) + for column in preserved_table.get("columns") or []: + if isinstance(column, dict) and column.get("name"): + _add_catalog_column(datasets[name], str(column["name"])) + + relationships = [ + _validate_relationship(item, datasets) + for item in model.get("relationships") or [] + ] + for relationship in relationships: + for side, key in (("from", "from_columns"), ("to", "to_columns")): + for column in relationship[key]: + _add_catalog_column(datasets[str(relationship[side])], str(column)) + + for name, context in datasets.items(): + dataset = context["original"] + for column in dataset.get("primary_key") or []: + _add_catalog_column(context, str(column)) + for key in dataset.get("unique_keys") or []: + for column in key: + _add_catalog_column(context, str(column)) + + field_names: set[str] = set() + for field in dataset.get("fields") or []: + if not isinstance(field, dict) or not field.get("name"): + raise GSFConversionError( + f"Every field in dataset {name!r} needs a name" + ) + field_name = str(field["name"]) + if field_name in field_names: + raise GSFConversionError( + f"Duplicate field name {field_name!r} in dataset {name!r}" + ) + field_names.add(field_name) + expressions = _normalize_expressions(field.get("expression"), field_name) + selected = _pick_expression(expressions, field_name) + source_column = _simple_source_column( + selected, name, str(context["source"]["table"]) + ) + if source_column: + _add_catalog_column(context, source_column) + context["simple_fields"].append((field, source_column)) + else: + refs = _field_table_refs(field, selected, name, datasets) + context["computed_fields"].append((field, expressions, selected, refs)) + _collect_expression_columns(selected, name, refs, datasets, preserved) + + metrics: list[tuple[dict[str, Any], list[dict[str, str]], str, list[str]]] = [] + metric_names: set[str] = set() + for metric in model.get("metrics") or []: + if not isinstance(metric, dict) or not metric.get("name"): + raise GSFConversionError("Every Ossie metric must be a mapping with a name") + name = str(metric["name"]) + if name in metric_names: + raise GSFConversionError(f"Duplicate metric name {name!r}") + metric_names.add(name) + expressions = _normalize_expressions(metric.get("expression"), name) + selected = _pick_expression(expressions, name) + refs = _metric_table_refs(metric, selected, datasets) + _collect_expression_columns(selected, None, refs, datasets, preserved) + metrics.append((metric, expressions, selected, refs)) + + databases, table_ids, column_ids = _build_catalog(datasets, preserved) + if native: + databases = _merge_catalog_databases( + native.get("data_layer", {}).get("databases"), databases + ) + for column_id in _catalog_column_ids(databases): + column_ids.setdefault(("__native__", column_id), column_id) + terms: list[dict[str, Any]] = [] + term_ids: dict[str, str] = {} + attribute_ids: dict[tuple[str, str], str] = {} + + for name, context in datasets.items(): + dataset = context["original"] + source_key = _source_key(context["source"]) + preserved_term = preserved["terms"].get((name, source_key), {}) + term_id = str(preserved_term.get("id") or _stable_id("term", name, *source_key)) + term_ids[name] = term_id + term: dict[str, Any] = { + "id": term_id, + "name": name, + "description": str(dataset.get("description") or ""), + "represents": [table_ids[name]], + "columns_attributes": [], + } + for field, source_column in context["simple_fields"]: + field_name = str(field["name"]) + preserved_attr = preserved["column_attributes"].get( + (name, field_name, source_key, source_column), {} + ) + attr_id = str( + preserved_attr.get("id") + or _stable_id( + "column-attribute", name, field_name, *source_key, source_column + ) + ) + attribute_ids[(name, source_column)] = attr_id + term["columns_attributes"].append( + { + "id": attr_id, + "name": field_name, + "description": str(field.get("description") or ""), + "column_id": column_ids[(name, source_column)], + } + ) + terms.append(term) + + joins: list[dict[str, Any]] = [] + foreign_keys: list[dict[str, str]] = [] + semantic_fks: list[dict[str, str]] = [] + for relationship in relationships: + from_name = str(relationship["from"]) + to_name = str(relationship["to"]) + joins.append( + { + "source_table_id": table_ids[from_name], + "target_table_id": table_ids[to_name], + "join_columns": [ + {"source": str(source), "target": str(target)} + for source, target in zip( + relationship["from_columns"], + relationship["to_columns"], + strict=True, + ) + ], + } + ) + for source, target in zip( + relationship["from_columns"], + relationship["to_columns"], + strict=True, + ): + source_name = str(source) + target_name = str(target) + foreign_keys.append( + { + "source_column_id": column_ids[(from_name, source_name)], + "target_column_id": column_ids[(to_name, target_name)], + } + ) + target_attribute_id = attribute_ids.get((to_name, target_name)) + if target_attribute_id: + semantic_fks.append( + { + "column_attribute_id": target_attribute_id, + "column_id": column_ids[(from_name, source_name)], + } + ) + + sql_groups: dict[str, list[dict[str, Any]]] = {key: [] for key in _SQL_GROUPS} + for name, context in datasets.items(): + for field, _, selected, refs in context["computed_fields"]: + field_name = str(field["name"]) + extension = _gsf_extension_data(field) + preserved_attr = preserved["sql_attributes"].get((name, field_name), {}) + expression_unchanged = _expression_matches( + selected, extension.get("ossie_expression") + ) + source_group = str( + extension.get("sql_source") + or preserved_attr.get("source_group") + or "manual" + ) + if source_group not in sql_groups: + source_group = "manual" + full_sql = str( + extension.get("sql") + if expression_unchanged and extension.get("sql") + else _wrap_expression( + selected, + field_name, + refs, + datasets, + relationships, + ) + ) + resolved_column_ids = _sql_column_ids( + selected, name, refs, datasets, column_ids + ) + preserved_column_ids = extension.get( + "sql_column_is", preserved_attr.get("sql_column_is") + ) + if expression_unchanged and _all_resolvable_ids( + preserved_column_ids, column_ids + ): + resolved_column_ids = list(preserved_column_ids) + sql_groups[source_group].append( + { + "id": str( + extension.get("id") + or preserved_attr.get("id") + or _stable_id("sql-attribute", name, field_name) + ), + "name": field_name, + "description": str(field.get("description") or ""), + "sql": full_sql, + "sql_column_is": resolved_column_ids, + "term_id": term_ids[name], + } + ) + + custom_analyses: list[dict[str, Any]] = [] + for metric, _, selected, refs in metrics: + name = str(metric["name"]) + extension = _gsf_extension_data(metric) + preserved_analysis = preserved["custom_analyses"].get(name, {}) + expression_unchanged = _expression_matches( + selected, extension.get("ossie_expression") + ) + full_sql = str( + extension.get("sql") + if expression_unchanged and extension.get("sql") + else _wrap_expression(selected, name, refs, datasets, relationships) + ) + referenced_column_ids = _sql_column_ids( + selected, None, refs, datasets, column_ids + ) + preserved_column_ids = extension.get( + "sql_column_is", preserved_analysis.get("sql_column_is") + ) + if expression_unchanged and _all_resolvable_ids( + preserved_column_ids, column_ids + ): + referenced_column_ids = list(preserved_column_ids) + if not referenced_column_ids: + referenced_column_ids = _first_resolvable_column_ids( + refs, datasets, column_ids + ) + if not referenced_column_ids: + raise GSFConversionError( + f"Metric {name!r} has no resolvable catalog column for " + "custom-analysis SQL validation" + ) + custom_analyses.append( + { + "id": str( + extension.get("id") + or preserved_analysis.get("id") + or _stable_id("custom-analysis", str(model["name"]), name) + ), + "name": name, + "description": str(metric.get("description") or ""), + "sql": full_sql, + "sql_column_is": referenced_column_ids, + } + ) + + if native: + foreign_keys, joins, semantic_fks = _reconcile_native_relationships( + native, + represented_table_ids=set(table_ids.values()), + foreign_keys=foreign_keys, + joins=joins, + semantic_fks=semantic_fks, + ) + + output = { + "data_layer": { + "databases": databases, + "foreign_keys": foreign_keys, + "joins": joins, + }, + "semantic_layer": { + "terms": terms, + "semantic_fks": semantic_fks, + "sql_attributes": sql_groups, + "custom_analyses": custom_analyses, + }, + "zones": deepcopy(native.get("zones") or []) if native else [], + } + return _dump_yaml(output) + + +def convert_gsf_to_ossie( + gsf_yaml: str, + *, + model_name: str | None = None, +) -> str: + """Convert a native ``GsfModelDocument`` to one Apache Ossie model.""" + root = _parse_gsf(gsf_yaml) + catalog = _read_catalog(root) + semantic = root["semantic_layer"] + terms = semantic["terms"] + if not terms: + raise GSFConversionError( + "GSF document has no representable terms; Ossie requires at least " + "one dataset" + ) + + term_by_id: dict[str, dict[str, Any]] = {} + term_id_by_name: dict[str, str] = {} + datasets_by_table: dict[str, list[str]] = defaultdict(list) + datasets: list[dict[str, Any]] = [] + fields_by_term: dict[str, list[dict[str, Any]]] = defaultdict(list) + field_names_by_term: dict[str, set[str]] = defaultdict(set) + term_columns: dict[str, set[str]] = defaultdict(set) + attr_owner: dict[str, tuple[str, dict[str, Any]]] = {} + + for term in terms: + term_id = _required_id(term, "GSF term") + if term_id in term_by_id: + raise GSFConversionError(f"Duplicate GSF term id {term_id!r}") + represents = term.get("represents") or [] + if len(represents) != 1: + raise GSFConversionError( + f"Converter supports only GSF terms that represent exactly one " + f"table; term {term.get('name')!r} represents {len(represents)}" + ) + table_id = str(represents[0]) + table = catalog["tables"].get(table_id) + if table is None: + raise GSFConversionError( + f"GSF term {term.get('name')!r} represents unknown table {table_id!r}" + ) + name = str(term.get("name") or "") + if not name: + raise GSFConversionError("Every GSF term needs a non-empty name") + if name in term_id_by_name: + raise GSFConversionError(f"Duplicate GSF term name {name!r}") + term_id_by_name[name] = term_id + datasets_by_table[table_id].append(name) + term_by_id[term_id] = term + dataset: dict[str, Any] = { + "name": name, + "source": ".".join(table["source"]), + } + if term.get("description"): + dataset["description"] = str(term["description"]) + if table["item"].get("pk"): + dataset["primary_key"] = list(table["item"]["pk"]) + unique_columns = [ + str(column.get("name")) + for column in table["item"].get("columns") or [] + if column.get("is_unique") and column.get("name") + ] + if unique_columns: + dataset["unique_keys"] = [[column] for column in unique_columns] + + for attribute in term.get("columns_attributes") or []: + if not isinstance(attribute, dict) or not attribute.get("id"): + raise GSFConversionError( + f"Term {name!r} contains a column attribute without an id" + ) + column_id = str(attribute.get("column_id") or "") + column = catalog["columns"].get(column_id) + if column is None: + raise GSFConversionError( + f"Column attribute {attribute.get('name')!r} references " + f"unknown column {column_id!r}" + ) + field_name = str(attribute.get("name") or column["name"]) + if field_name in field_names_by_term[term_id]: + raise GSFConversionError( + f"Duplicate field name {field_name!r} in GSF term {name!r}" + ) + field_names_by_term[term_id].add(field_name) + term_columns[name].add(str(column["name"])) + field: dict[str, Any] = { + "name": field_name, + "expression": _ossie_expression(str(column["name"])), + } + if attribute.get("description"): + field["description"] = str(attribute["description"]) + fields_by_term[term_id].append(field) + attr_owner[str(attribute["id"])] = (term_id, attribute) + + datasets.append(dataset) + + for source_group in _SQL_GROUPS: + for attribute in semantic["sql_attributes"][source_group]: + attribute_id = _required_id(attribute, "GSF SQL attribute") + term_id = str(attribute.get("term_id") or "") + if term_id not in term_by_id: + raise GSFConversionError( + f"SQL attribute {attribute.get('name')!r} references " + f"unknown term {term_id!r}" + ) + sql = str(attribute.get("sql") or "") + name = str(attribute.get("name") or "") + if not name or not sql: + raise GSFConversionError( + "Every GSF SQL attribute requires non-empty name and sql" + ) + term_name = str(term_by_id[term_id].get("name") or "") + if name in field_names_by_term[term_id]: + raise GSFConversionError( + f"Duplicate field name {name!r} in GSF term {term_name!r}" + ) + field_names_by_term[term_id].add(name) + represented_table_id = str(term_by_id[term_id]["represents"][0]) + _validate_gsf_sql_databases( + f"SQL attribute {name!r}", + sql, + attribute.get("sql_column_is") or [], + catalog, + attached_table_id=represented_table_id, + ) + ossie_expression = _expression_from_sql(sql) + field = { + "name": name, + "expression": _ossie_expression(ossie_expression), + "custom_extensions": [ + _gsf_extension( + { + "entity": "sql_attribute", + "id": attribute_id, + "sql": sql, + "sql_source": source_group, + "sql_column_is": list(attribute.get("sql_column_is") or []), + "term_id": term_id, + "ossie_expression": ossie_expression, + } + ) + ], + } + if attribute.get("description"): + field["description"] = str(attribute["description"]) + fields_by_term[term_id].append(field) + + for dataset in datasets: + term_id = term_id_by_name[dataset["name"]] + if fields_by_term[term_id]: + dataset["fields"] = fields_by_term[term_id] + + metrics: list[dict[str, Any]] = [] + for analysis in semantic.get("custom_analyses") or []: + analysis_id = _required_id(analysis, "GSF custom analysis") + sql = str(analysis.get("sql") or "") + name = str(analysis.get("name") or "") + if not name or not sql: + raise GSFConversionError( + "Every GSF custom analysis requires non-empty name and sql" + ) + _validate_gsf_sql_databases( + f"Custom analysis {name!r}", + sql, + analysis.get("sql_column_is") or [], + catalog, + ) + ossie_expression = _expression_from_sql(sql) + metric: dict[str, Any] = { + "name": name, + "expression": _ossie_expression(ossie_expression), + "custom_extensions": [ + _gsf_extension( + { + "entity": "custom_analysis", + "id": analysis_id, + "sql": sql, + "sql_column_is": list(analysis.get("sql_column_is") or []), + "ossie_expression": ossie_expression, + } + ) + ], + } + if analysis.get("description"): + metric["description"] = str(analysis["description"]) + metrics.append(metric) + + relationships = _relationships_from_gsf( + root, + catalog, + datasets_by_table, + term_columns, + attr_owner, + term_by_id, + ) + database_names = { + source[0] + for source in (table["source"] for table in catalog["tables"].values()) + } + inferred_name = ( + model_name + or (next(iter(database_names)) if len(database_names) == 1 else None) + or "gsf_model" + ) + semantic_model: dict[str, Any] = { + "name": inferred_name, + "datasets": datasets, + "custom_extensions": [ + _gsf_extension( + { + "model_name": inferred_name, + "native_document": root, + } + ) + ], + } + if relationships: + semantic_model["relationships"] = relationships + if metrics: + semantic_model["metrics"] = metrics + return _dump_yaml({"version": OSSIE_VERSION, "semantic_model": [semantic_model]}) + + +def _build_catalog( + datasets: Mapping[str, dict[str, Any]], + preserved: Mapping[str, Any], +) -> tuple[list[dict[str, Any]], dict[str, str], dict[tuple[str, str], str]]: + db_tree: dict[str, dict[str, dict[str, Any]]] = {} + table_ids: dict[str, str] = {} + column_ids: dict[tuple[str, str], str] = {} + source_groups: dict[tuple[str, str, str], list[tuple[str, dict[str, Any]]]] = ( + defaultdict(list) + ) + for dataset_name, context in datasets.items(): + source_groups[_source_key(context["source"])].append((dataset_name, context)) + + for source_key, contexts in source_groups.items(): + database, schema, table = source_key + preserved_table = preserved["tables"].get(source_key, {}) + preserved_schema = preserved["schemas"].get((database, schema), {}) + preserved_db = preserved["databases"].get(database, {}) + db_entry = db_tree.setdefault( + database, + { + "id": str(preserved_db.get("id") or _stable_id("database", database)), + "dialect": str(preserved_db.get("dialect") or ""), + "schemas": {}, + }, + ) + schema_entry = db_entry["schemas"].setdefault( + schema, + { + "id": str( + preserved_schema.get("id") or _stable_id("schema", database, schema) + ), + "name": schema, + "database_name": database, + "tables": [], + }, + ) + table_id = str( + preserved_table.get("id") or _stable_id("table", database, schema, table) + ) + for dataset_name, _ in contexts: + table_ids[dataset_name] = table_id + pk = list( + dict.fromkeys( + str(value) + for _, context in contexts + for value in context["original"].get("primary_key") or [] + ) + ) + unique_keys = [ + [str(value) for value in key] + for _, context in contexts + for key in context["original"].get("unique_keys") or [] + ] + catalog_columns = list( + dict.fromkeys( + column for _, context in contexts for column in context["columns"] + ) + ) + columns: list[dict[str, Any]] = [] + for column_name in catalog_columns: + preserved_column = preserved["columns"].get((*source_key, column_name), {}) + column_id = str( + preserved_column.get("id") + or _stable_id("column", database, schema, table, column_name) + ) + for dataset_name, _ in contexts: + column_ids[(dataset_name, column_name)] = column_id + single_unique = [column_name] in unique_keys or ( + len(pk) == 1 and pk[0] == column_name + ) + columns.append( + { + "id": column_id, + "name": column_name, + "description": str(preserved_column.get("description") or ""), + "type": str(preserved_column.get("type") or ""), + "sample_values": list(preserved_column.get("sample_values") or []), + "is_nullable": bool( + preserved_column.get("is_nullable", column_name not in pk) + ), + "is_unique": bool(preserved_column.get("is_unique", single_unique)), + } + ) + schema_entry["tables"].append( + { + "id": table_id, + "name": table, + "description": str( + preserved_table.get("description") + if preserved_table + else next( + ( + context["original"].get("description") + for _, context in contexts + if context["original"].get("description") + ), + "", + ) + ), + "pk": pk, + "type": str(preserved_table.get("type") or ""), + "columns": columns, + } + ) + + result: list[dict[str, Any]] = [] + for database in sorted(db_tree): + db_entry = db_tree[database] + schemas = [db_entry["schemas"][name] for name in sorted(db_entry["schemas"])] + for schema in schemas: + schema["tables"].sort(key=lambda item: (item["name"], item["id"])) + result.append( + { + "id": db_entry["id"], + "dialect": db_entry["dialect"], + "schemas": schemas, + } + ) + return result, table_ids, column_ids + + +def _merge_catalog_databases( + preserved: Any, + generated: list[dict[str, Any]], +) -> list[dict[str, Any]]: + """Keep catalog objects that Ossie cannot represent directly.""" + result = deepcopy(generated) + databases_by_id = {str(item["id"]): item for item in result} + for preserved_database in preserved or []: + if not isinstance(preserved_database, dict): + continue + database_id = str(preserved_database.get("id") or "") + database = databases_by_id.get(database_id) + if database is None: + copied = deepcopy(preserved_database) + result.append(copied) + databases_by_id[database_id] = copied + continue + schemas_by_id = { + str(item["id"]): item for item in database.get("schemas") or [] + } + for preserved_schema in preserved_database.get("schemas") or []: + schema_id = str(preserved_schema.get("id") or "") + schema = schemas_by_id.get(schema_id) + if schema is None: + database["schemas"].append(deepcopy(preserved_schema)) + continue + tables_by_id = { + str(item["id"]): item for item in schema.get("tables") or [] + } + for preserved_table in preserved_schema.get("tables") or []: + table_id = str(preserved_table.get("id") or "") + table = tables_by_id.get(table_id) + if table is None: + schema["tables"].append(deepcopy(preserved_table)) + continue + known_column_ids = { + str(item["id"]) for item in table.get("columns") or [] + } + table["columns"].extend( + deepcopy(column) + for column in preserved_table.get("columns") or [] + if str(column.get("id") or "") not in known_column_ids + ) + return result + + +def _catalog_column_ids(databases: Iterable[Mapping[str, Any]]) -> set[str]: + return { + str(column["id"]) + for database in databases + for schema in database.get("schemas") or [] + for table in schema.get("tables") or [] + for column in table.get("columns") or [] + if column.get("id") + } + + +def _index_native_document(root: dict[str, Any] | None) -> dict[str, Any]: + result: dict[str, Any] = { + "databases": {}, + "schemas": {}, + "tables": {}, + "columns": {}, + "terms": {}, + "column_attributes": {}, + "sql_attributes": {}, + "custom_analyses": {}, + } + if not root: + return result + try: + catalog = _read_catalog(root) + except GSFConversionError: + return result + for database in root["data_layer"].get("databases") or []: + database_names = { + str(schema.get("database_name") or "") + for schema in database.get("schemas") or [] + if schema.get("database_name") + } + for name in database_names: + result["databases"][name] = database + for schema in database.get("schemas") or []: + database_name = str(schema.get("database_name") or "") + schema_name = str(schema.get("name") or "") + result["schemas"][(database_name, schema_name)] = schema + for table in schema.get("tables") or []: + source = (database_name, schema_name, str(table.get("name") or "")) + result["tables"][source] = table + for column in table.get("columns") or []: + result["columns"][(*source, str(column.get("name") or ""))] = column + table_source = { + table_id: tuple(table["source"]) + for table_id, table in catalog["tables"].items() + } + for term in root["semantic_layer"].get("terms") or []: + represents = term.get("represents") or [] + if len(represents) != 1 or str(represents[0]) not in table_source: + continue + source = table_source[str(represents[0])] + term_name = str(term.get("name") or "") + result["terms"][(term_name, source)] = term + for attribute in term.get("columns_attributes") or []: + column = catalog["columns"].get(str(attribute.get("column_id") or "")) + if column: + result["column_attributes"][ + ( + term_name, + str(attribute.get("name") or ""), + source, + column["name"], + ) + ] = attribute + term_names = { + str(term.get("id")): str(term.get("name") or "") + for term in root["semantic_layer"].get("terms") or [] + } + for group in _SQL_GROUPS: + for attribute in (root["semantic_layer"].get("sql_attributes") or {}).get( + group, [] + ): + item = dict(attribute) + item["source_group"] = group + result["sql_attributes"][ + ( + term_names.get(str(attribute.get("term_id") or ""), ""), + str(attribute.get("name") or ""), + ) + ] = item + for analysis in root["semantic_layer"].get("custom_analyses") or []: + result["custom_analyses"][str(analysis.get("name") or "")] = analysis + return result + + +def _read_catalog(root: dict[str, Any]) -> dict[str, Any]: + tables: dict[str, dict[str, Any]] = {} + columns: dict[str, dict[str, Any]] = {} + database_ids: set[str] = set() + schema_ids: set[str] = set() + for database in root["data_layer"]["databases"]: + database_id = _required_id(database, "GSF database") + if database_id in database_ids: + raise GSFConversionError(f"Duplicate GSF database id {database_id!r}") + database_ids.add(database_id) + for schema in database.get("schemas") or []: + schema_id = _required_id(schema, "GSF schema") + if schema_id in schema_ids: + raise GSFConversionError(f"Duplicate GSF schema id {schema_id!r}") + schema_ids.add(schema_id) + database_name = str(schema.get("database_name") or "") + schema_name = str(schema.get("name") or "") + if not database_name: + raise GSFConversionError( + f"GSF schema {schema_name!r} requires database_name" + ) + for table in schema.get("tables") or []: + table_id = str(table.get("id") or "") + if not table_id or table_id in tables: + raise GSFConversionError( + f"Every GSF table needs a globally unique id; got {table_id!r}" + ) + table_name = str(table.get("name") or "") + tables[table_id] = { + "item": table, + "source": (database_name, schema_name, table_name), + } + for column in table.get("columns") or []: + column_id = str(column.get("id") or "") + if not column_id or column_id in columns: + raise GSFConversionError( + "Every GSF column needs a globally unique id; " + f"got {column_id!r}" + ) + columns[column_id] = { + "item": column, + "name": str(column.get("name") or ""), + "table_id": table_id, + } + return {"tables": tables, "columns": columns} + + +def _validate_gsf_sql_databases( + context: str, + sql: str, + sql_column_ids: Iterable[Any], + catalog: Mapping[str, Any], + *, + attached_table_id: str | None = None, +) -> None: + databases: set[str] = set() + if attached_table_id: + table = catalog["tables"].get(attached_table_id) + if table: + databases.add(str(table["source"][0])) + for column_id in sql_column_ids: + column = catalog["columns"].get(str(column_id)) + if column: + databases.add(str(catalog["tables"][column["table_id"]]["source"][0])) + + parsed = _parse_sql(sql) + for sql_table in parsed.find_all(exp.Table): + if sql_table.catalog: + databases.add(sql_table.catalog) + matches = [ + table + for table in catalog["tables"].values() + if sql_table.name == table["source"][2] + and (not sql_table.db or sql_table.db == table["source"][1]) + and (not sql_table.catalog or sql_table.catalog == table["source"][0]) + and (sql_table.catalog or not databases or table["source"][0] in databases) + ] + databases.update(str(table["source"][0]) for table in matches) + if len(databases) > 1: + raise GSFConversionError( + f"{context} spans multiple databases ({', '.join(sorted(databases))}); " + "the GSF importer validates each SQL object against one database" + ) + + +def _relationships_from_gsf( + root: dict[str, Any], + catalog: Mapping[str, Any], + datasets_by_table: Mapping[str, list[str]], + term_columns: Mapping[str, set[str]], + attr_owner: Mapping[str, tuple[str, dict[str, Any]]], + term_by_id: Mapping[str, dict[str, Any]], +) -> list[dict[str, Any]]: + pairs: dict[tuple[str, str], list[tuple[str, str]]] = defaultdict(list) + covered_fk_pairs: set[tuple[str, str]] = set() + for join in root["data_layer"].get("joins") or []: + source_table_id = str(join.get("source_table_id") or "") + target_table_id = str(join.get("target_table_id") or "") + if ( + source_table_id not in datasets_by_table + or target_table_id not in datasets_by_table + ): + continue + source_columns: list[tuple[str, str]] = [] + for item in join.get("join_columns") or []: + if not isinstance(item, dict): + continue + source_name = _join_column_name( + item.get("source"), source_table_id, catalog + ) + target_name = _join_column_name( + item.get("target"), target_table_id, catalog + ) + if source_name and target_name: + source_columns.append((source_name, target_name)) + if not source_columns: + source_columns = _fk_columns_for_tables( + root, source_table_id, target_table_id, catalog + ) + if source_columns: + key = ( + _relationship_dataset( + source_table_id, + [source for source, _ in source_columns], + datasets_by_table, + term_columns, + ), + _relationship_dataset( + target_table_id, + [target for _, target in source_columns], + datasets_by_table, + term_columns, + ), + ) + pairs[key].extend(source_columns) + covered_fk_pairs.add((source_table_id, target_table_id)) + + fk_groups: dict[tuple[str, str], list[tuple[str, str]]] = defaultdict(list) + for foreign_key in root["data_layer"].get("foreign_keys") or []: + source = catalog["columns"].get(str(foreign_key.get("source_column_id") or "")) + target = catalog["columns"].get(str(foreign_key.get("target_column_id") or "")) + if not source or not target: + continue + table_pair = (source["table_id"], target["table_id"]) + if table_pair in covered_fk_pairs: + continue + if table_pair[0] in datasets_by_table and table_pair[1] in datasets_by_table: + fk_groups[table_pair].append((source["name"], target["name"])) + for table_pair, columns in fk_groups.items(): + source_name = _relationship_dataset( + table_pair[0], + [source for source, _ in columns], + datasets_by_table, + term_columns, + ) + target_name = _relationship_dataset( + table_pair[1], + [target for _, target in columns], + datasets_by_table, + term_columns, + ) + pairs[(source_name, target_name)].extend(columns) + + for semantic_fk in root["semantic_layer"].get("semantic_fks") or []: + source = catalog["columns"].get(str(semantic_fk.get("column_id") or "")) + owner = attr_owner.get(str(semantic_fk.get("column_attribute_id") or "")) + if not source or not owner: + continue + target_term_id, target_attr = owner + target_column = catalog["columns"].get(str(target_attr.get("column_id") or "")) + target_term = term_by_id.get(target_term_id) + if not target_column or not target_term: + continue + if source["table_id"] not in datasets_by_table: + continue + from_name = _relationship_dataset( + source["table_id"], + [source["name"]], + datasets_by_table, + term_columns, + ) + to_name = str(target_term.get("name") or "") + if not to_name: + continue + pair = (source["name"], target_column["name"]) + if pair not in pairs[(from_name, to_name)]: + pairs[(from_name, to_name)].append(pair) + + relationships: list[dict[str, Any]] = [] + used_names: dict[str, int] = defaultdict(int) + for (from_name, to_name), columns in pairs.items(): + unique_columns = list(dict.fromkeys(columns)) + base_name = f"{from_name}_to_{to_name}" + used_names[base_name] += 1 + suffix = "" if used_names[base_name] == 1 else f"_{used_names[base_name]}" + relationships.append( + { + "name": base_name + suffix, + "from": from_name, + "to": to_name, + "from_columns": [source for source, _ in unique_columns], + "to_columns": [target for _, target in unique_columns], + } + ) + return relationships + + +def _relationship_dataset( + table_id: str, + columns: list[str], + datasets_by_table: Mapping[str, list[str]], + term_columns: Mapping[str, set[str]], +) -> str: + candidates = datasets_by_table.get(table_id) or [] + if len(candidates) == 1: + return candidates[0] + matching = [ + name for name in candidates if set(columns) <= term_columns.get(name, set()) + ] + if len(matching) == 1: + return matching[0] + raise GSFConversionError( + f"Cannot map relationship on table {table_id!r} and columns " + f"{', '.join(columns)} to exactly one represented term; candidates: " + f"{', '.join(candidates) or 'none'}" + ) + + +def _fk_columns_for_tables( + root: Mapping[str, Any], + source_table_id: str, + target_table_id: str, + catalog: Mapping[str, Any], +) -> list[tuple[str, str]]: + result: list[tuple[str, str]] = [] + for foreign_key in root["data_layer"].get("foreign_keys") or []: + source = catalog["columns"].get(str(foreign_key.get("source_column_id") or "")) + target = catalog["columns"].get(str(foreign_key.get("target_column_id") or "")) + if ( + source + and target + and source["table_id"] == source_table_id + and target["table_id"] == target_table_id + ): + result.append((source["name"], target["name"])) + return result + + +def _join_column_name( + value: Any, + table_id: str, + catalog: Mapping[str, Any], +) -> str | None: + text = str(value or "") + column = catalog["columns"].get(text) + if column and column["table_id"] == table_id: + return str(column["name"]) + for item in catalog["columns"].values(): + if item["table_id"] == table_id and item["name"] == text: + return text + return None + + +def _required_id(item: Any, context: str) -> str: + if not isinstance(item, dict) or not item.get("id"): + raise GSFConversionError(f"{context} requires a non-empty id") + return str(item["id"]) + + +def _parse_gsf(value: str) -> dict[str, Any]: + root = _load_yaml(value, "GSF") + expected = {"data_layer", "semantic_layer", "zones"} + unknown = sorted(set(root) - expected) + if unknown: + raise GSFConversionError( + "Unsupported GSF root properties: " + ", ".join(unknown) + ) + root.setdefault("data_layer", {}) + root.setdefault("semantic_layer", {}) + root.setdefault("zones", []) + for key in ("data_layer", "semantic_layer"): + if not isinstance(root[key], dict): + raise GSFConversionError(f"GSF {key!r} must be a mapping") + if not isinstance(root["zones"], list): + raise GSFConversionError("GSF 'zones' must be a list") + data_layer = root["data_layer"] + semantic_layer = root["semantic_layer"] + for key in ("databases", "foreign_keys", "joins"): + data_layer.setdefault(key, []) + if not isinstance(data_layer.get(key), list): + raise GSFConversionError(f"GSF data_layer.{key} must be a list") + for key in ("terms", "semantic_fks", "custom_analyses"): + semantic_layer.setdefault(key, []) + if not isinstance(semantic_layer.get(key), list): + raise GSFConversionError(f"GSF semantic_layer.{key} must be a list") + semantic_layer.setdefault("sql_attributes", {}) + sql_attributes = semantic_layer["sql_attributes"] + if not isinstance(sql_attributes, dict): + raise GSFConversionError("GSF semantic_layer.sql_attributes must be a mapping") + for key in _SQL_GROUPS: + sql_attributes.setdefault(key, []) + if not isinstance(sql_attributes.get(key), list): + raise GSFConversionError( + f"GSF semantic_layer.sql_attributes.{key} must be a list" + ) + return root + + +def _parse_ossie(value: str) -> tuple[dict[str, Any], dict[str, Any]]: + root = _load_yaml(value, "Ossie") + unknown = sorted(set(root) - {"version", "semantic_model"}) + if unknown: + raise GSFConversionError( + "Unsupported Ossie root properties: " + ", ".join(unknown) + ) + if str(root.get("version", "")) != OSSIE_VERSION: + raise GSFConversionError( + f"Unsupported Ossie version {root.get('version')!r}; " + f"supported version is {OSSIE_VERSION!r}" + ) + models = root.get("semantic_model") + if not isinstance(models, list) or len(models) != 1: + raise GSFConversionError("Ossie input must contain exactly one semantic model") + model = models[0] + if not isinstance(model, dict) or not model.get("name"): + raise GSFConversionError("Ossie semantic model requires a name") + return root, model + + +def _load_yaml(value: str, label: str) -> dict[str, Any]: + try: + root = yaml.safe_load(value) + except yaml.YAMLError as exc: + raise GSFConversionError(f"Invalid {label} YAML: {exc}") from exc + if not isinstance(root, dict): + raise GSFConversionError(f"Invalid {label} YAML: expected a root mapping") + return root + + +def _validate_relationship( + relationship: Any, + datasets: Mapping[str, Any], +) -> dict[str, Any]: + if not isinstance(relationship, dict) or not relationship.get("name"): + raise GSFConversionError("Every Ossie relationship needs a name") + from_name = relationship.get("from") + to_name = relationship.get("to") + if from_name not in datasets or to_name not in datasets: + raise GSFConversionError( + f"Relationship {relationship['name']!r} references an unknown dataset" + ) + from_columns = relationship.get("from_columns") or [] + to_columns = relationship.get("to_columns") or [] + if not from_columns or len(from_columns) != len(to_columns): + raise GSFConversionError( + f"Relationship {relationship['name']!r} must have equal, " + "non-empty column lists" + ) + return relationship + + +def _normalize_expressions(value: Any, name: str) -> list[dict[str, str]]: + if not isinstance(value, dict): + raise GSFConversionError(f"{name!r} has no valid expression") + dialects = value.get("dialects") + if not isinstance(dialects, list) or not dialects: + raise GSFConversionError(f"{name!r} requires at least one expression dialect") + result = [ + { + "dialect": str(item["dialect"]), + "expression": str(item["expression"]), + } + for item in dialects + if isinstance(item, dict) + and item.get("dialect") + and item.get("expression") is not None + ] + if not result: + raise GSFConversionError(f"{name!r} has no usable expression dialect") + return result + + +def _pick_expression(expressions: list[dict[str, str]], name: str) -> str: + for expression in expressions: + if expression["dialect"].upper() == "ANSI_SQL": + return expression["expression"] + if expressions: + return expressions[0]["expression"] + raise GSFConversionError(f"{name!r} has no usable expression") + + +def _simple_source_column( + expression: str, + dataset_name: str, + table_name: str, +) -> str | None: + match = _SIMPLE_COLUMN.fullmatch(expression.strip()) + if not match: + return None + qualifier = match.group("qualifier") + if qualifier and qualifier not in (dataset_name, table_name): + return None + return match.group("column") + + +def _field_table_refs( + field: Mapping[str, Any], + expression: str, + owner: str, + datasets: Mapping[str, Any], +) -> list[str]: + extension = _gsf_extension_data(field) + extension_refs = extension.get("table_refs") + expression_unchanged = _expression_matches( + expression, extension.get("ossie_expression") + ) + reference_sql = ( + str(extension["sql"]) + if extension.get("sql") and expression_unchanged + else expression + ) + use_extension_refs = not extension.get("entity") or expression_unchanged + if use_extension_refs and isinstance(extension_refs, list) and extension_refs: + refs = [str(item) for item in extension_refs] + _validate_refs(refs, datasets, str(field.get("name"))) + _validate_single_database_refs( + refs, + datasets, + f"SQL attribute {field.get('name')!r}", + sql=reference_sql, + ) + return refs + refs = _referenced_datasets(reference_sql, datasets) + result = list(dict.fromkeys([owner, *refs])) + _validate_single_database_refs( + result, + datasets, + f"SQL attribute {field.get('name')!r}", + sql=reference_sql, + ) + return result + + +def _metric_table_refs( + metric: Mapping[str, Any], + expression: str, + datasets: Mapping[str, Any], +) -> list[str]: + extension = _gsf_extension_data(metric) + extension_refs = extension.get("table_refs") + expression_unchanged = _expression_matches( + expression, extension.get("ossie_expression") + ) + reference_sql = ( + str(extension["sql"]) + if extension.get("sql") and expression_unchanged + else expression + ) + use_extension_refs = not extension.get("entity") or expression_unchanged + if use_extension_refs and isinstance(extension_refs, list) and extension_refs: + refs = [str(item) for item in extension_refs] + _validate_refs(refs, datasets, str(metric.get("name"))) + _validate_single_database_refs( + refs, + datasets, + f"Custom analysis {metric.get('name')!r}", + sql=reference_sql, + ) + return refs + refs = _referenced_datasets(reference_sql, datasets) + if refs: + _validate_single_database_refs( + refs, + datasets, + f"Custom analysis {metric.get('name')!r}", + sql=reference_sql, + ) + return refs + if len(datasets) == 1: + refs = [next(iter(datasets))] + _validate_single_database_refs( + refs, + datasets, + f"Custom analysis {metric.get('name')!r}", + sql=reference_sql, + ) + return refs + if extension.get("entity") == "custom_analysis" and extension.get("sql"): + refs = [next(iter(datasets))] + _validate_single_database_refs( + refs, + datasets, + f"Custom analysis {metric.get('name')!r}", + sql=reference_sql, + ) + return refs + raise GSFConversionError( + f"Metric {metric.get('name')!r} does not identify a source dataset; " + "qualify a referenced column or add NVIDIA_GSF table_refs" + ) + + +def _validate_refs( + refs: Iterable[str], + datasets: Mapping[str, Any], + name: str, +) -> None: + unknown = [ref for ref in refs if ref not in datasets] + if unknown: + raise GSFConversionError( + f"{name!r} has unknown NVIDIA_GSF table_refs: {', '.join(unknown)}" + ) + + +def _validate_single_database_refs( + refs: Iterable[str], + datasets: Mapping[str, Any], + context: str, + *, + sql: str, +) -> None: + databases = { + str(datasets[ref]["source"]["database"]) for ref in refs if ref in datasets + } + databases.update( + table.catalog for table in _parse_sql(sql).find_all(exp.Table) if table.catalog + ) + if len(databases) > 1: + raise GSFConversionError( + f"{context} spans multiple databases ({', '.join(sorted(databases))}); " + "the GSF importer validates each SQL object against one database" + ) + + +def _referenced_datasets( + sql: str, + datasets: Mapping[str, Any], +) -> list[str]: + references: list[str] = [] + parsed = _parse_sql(sql) + for table in parsed.find_all(exp.Table): + matches = [ + name + for name, context in datasets.items() + if table.name == str(context["source"]["table"]) + and (not table.db or table.db == str(context["source"]["schema"])) + and ( + not table.catalog or table.catalog == str(context["source"]["database"]) + ) + ] + if matches: + match = matches[0] + if match not in references: + references.append(match) + for column in parsed.find_all(exp.Column): + qualifier = column.table + if not qualifier: + continue + matches = [ + name + for name, context in datasets.items() + if qualifier in (name, str(context["source"]["table"])) + ] + if len(matches) == 1 and matches[0] not in references: + references.append(matches[0]) + return references + + +def _collect_expression_columns( + sql: str, + owner: str | None, + refs: list[str], + datasets: Mapping[str, dict[str, Any]], + preserved: Mapping[str, Any], +) -> None: + for column in _sql_columns(sql): + dataset_name = _column_dataset(column, owner, refs, datasets) + if not dataset_name: + continue + context = datasets[dataset_name] + source_key = _source_key(context["source"]) + known_table = source_key in preserved["tables"] + if known_table and (*source_key, column.name) not in preserved["columns"]: + # A GSF-sourced catalog is authoritative: an identifier that is not + # already a column of the table is SQL syntax, not physical data. + continue + _add_catalog_column(context, column.name) + + +def _sql_column_ids( + sql: str, + owner: str | None, + refs: list[str], + datasets: Mapping[str, dict[str, Any]], + column_ids: Mapping[tuple[str, str], str], +) -> list[str]: + result: list[str] = [] + for column in _sql_columns(sql): + dataset_name = _column_dataset(column, owner, refs, datasets) + column_id = ( + column_ids.get((dataset_name, column.name)) if dataset_name else None + ) + if column_id and column_id not in result: + result.append(column_id) + return result + + +def _column_dataset( + column: exp.Column, + owner: str | None, + refs: list[str], + datasets: Mapping[str, Any], +) -> str | None: + qualifier = column.table + if qualifier: + matches = [ + name + for name, context in datasets.items() + if name in refs and qualifier in (name, str(context["source"]["table"])) + ] + return matches[0] if len(matches) == 1 else None + if owner: + return owner + return refs[0] if len(refs) == 1 else None + + +def _sql_columns(sql: str) -> list[exp.Column]: + return [ + column + for column in _parse_sql(sql).find_all(exp.Column) + if not _is_date_part_unit(column) + ] + + +def _is_date_part_unit(column: exp.Column) -> bool: + """Report whether a parsed column is really a date-part keyword. + + ``DATEDIFF(day, a, b)`` and friends put the unit in an argument slot that + sqlglot parses as an unqualified column, which would otherwise be mistaken + for a physical catalog column. + """ + parent = column.parent + if column.table or column.name.lower() not in _DATE_PART_UNITS: + return False + if not isinstance(parent, exp.Func): + return False + if isinstance(parent, exp.Anonymous): + return str(parent.this).lower() in _DATE_FUNCTIONS + try: + names = parent.sql_names() + except (AttributeError, IndexError): + return False + return any(name.lower() in _DATE_FUNCTIONS for name in names) + + +def _parse_sql(sql: str) -> exp.Expression: + try: + return parse_one(sql) + except (ParseError, ValueError) as exc: + raise GSFConversionError( + f"Unable to parse SQL expression {sql!r}: {exc}" + ) from exc + + +def _first_resolvable_column_ids( + refs: list[str], + datasets: Mapping[str, dict[str, Any]], + column_ids: Mapping[tuple[str, str], str], +) -> list[str]: + for ref in refs: + for column in datasets[ref]["columns"]: + column_id = column_ids.get((ref, column)) + if column_id: + return [column_id] + return [] + + +def _wrap_expression( + expression: str, + name: str, + refs: list[str], + datasets: Mapping[str, dict[str, Any]], + relationships: list[dict[str, Any]], +) -> str: + stripped = expression.strip() + if stripped.upper().startswith(("SELECT ", "SELECT\n", "WITH ", "WITH\n")): + return stripped + if not refs: + raise GSFConversionError(f"Cannot determine a source table for {name!r}") + anchor_name = refs[0] + from_sql = ( + f"{_qualified_table(datasets[anchor_name]['source'])} " + f"AS {_quote_identifier(anchor_name)}" + ) + joined = {anchor_name} + remaining = set(refs[1:]) + joins: list[str] = [] + while remaining: + matched = False + for relationship in relationships: + left = str(relationship["from"]) + right = str(relationship["to"]) + if left in joined and right in remaining: + new_name = right + elif right in joined and left in remaining: + new_name = left + else: + continue + conditions = [ + f"{_quote_identifier(left)}.{_quote_identifier(str(left_column))} = " + f"{_quote_identifier(right)}.{_quote_identifier(str(right_column))}" + for left_column, right_column in zip( + relationship["from_columns"], + relationship["to_columns"], + strict=True, + ) + ] + joins.append( + f"JOIN {_qualified_table(datasets[new_name]['source'])} " + f"AS {_quote_identifier(new_name)} ON {' AND '.join(conditions)}" + ) + joined.add(new_name) + remaining.remove(new_name) + matched = True + break + if not matched: + missing = min(remaining) + raise GSFConversionError( + f"{name!r} references disconnected dataset {missing!r}; " + "declare a relationship connecting all referenced datasets" + ) + suffix = f" {' '.join(joins)}" if joins else "" + return f"SELECT {stripped} AS {_quote_identifier(name)} FROM {from_sql}{suffix}" + + +def _expression_from_sql(sql: str) -> str: + try: + parsed = parse_one(sql) + except (ParseError, ValueError): + return sql + select = parsed.find(exp.Select) + if select is None or not select.expressions: + return sql + expression = select.expressions[0] + if isinstance(expression, exp.Alias): + expression = expression.this + return expression.sql() + + +def _expression_matches(current: str, emitted: Any) -> bool: + if not isinstance(emitted, str) or not emitted.strip(): + return False + try: + return parse_one(current).sql() == parse_one(emitted).sql() + except (ParseError, ValueError): + return current.strip() == emitted.strip() + + +def _ossie_expression(expression: str) -> dict[str, Any]: + return { + "dialects": [ + { + "dialect": "ANSI_SQL", + "expression": expression, + } + ] + } + + +def _parse_source( + source: Any, + default_database: str | None, +) -> dict[str, str | None]: + if isinstance(source, dict): + database = source.get("database") or default_database + schema = source.get("schema") + table = source.get("table") + if not table: + raise GSFConversionError("Source mapping requires 'table'") + return { + "database": str(database) if database else None, + "schema": str(schema) if schema else None, + "table": str(table), + } + value = str(source or "").strip() + if not value: + raise GSFConversionError("Every dataset needs a source") + if value.upper().startswith(("SELECT ", "SELECT\n", "WITH ", "WITH\n")): + raise GSFConversionError("GSF terms must identify physical tables") + parts = _split_identifier(value) + if len(parts) == 3: + database, schema, table = parts + elif len(parts) == 2: + database, (schema, table) = default_database, parts + elif len(parts) == 1: + database, schema, table = default_database, None, parts[0] + else: + raise GSFConversionError( + f"Source {value!r} must be table, schema.table, or database.schema.table" + ) + return {"database": database, "schema": schema, "table": table} + + +def _split_identifier(value: str) -> list[str]: + parts: list[str] = [] + current: list[str] = [] + quote: str | None = None + for char in value: + if char in ('"', "`"): + quote = None if quote == char else char if quote is None else quote + elif char == "." and quote is None: + parts.append("".join(current).strip()) + current = [] + continue + current.append(char) + parts.append("".join(current).strip()) + return [ + part[1:-1] + if len(part) > 1 and part[0] == part[-1] and part[0] in ('"', "`") + else part + for part in parts + ] + + +def _qualified_table(source: Mapping[str, Any]) -> str: + return ".".join( + _quote_identifier(str(source[key])) for key in ("database", "schema", "table") + ) + + +def _quote_identifier(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +def _source_key(source: Mapping[str, Any]) -> tuple[str, str, str]: + return tuple(str(source[key]) for key in ("database", "schema", "table")) # type: ignore[return-value] + + +def _add_catalog_column(context: dict[str, Any], name: str) -> None: + if name and name not in context["column_set"]: + context["column_set"].add(name) + context["columns"].append(name) + + +def _all_resolvable_ids( + values: Any, + column_ids: Mapping[tuple[str, str], str], +) -> bool: + return ( + isinstance(values, list) + and bool(values) + and all(str(value) in column_ids.values() for value in values) + ) + + +def _reconcile_native_relationships( + native: dict[str, Any], + *, + represented_table_ids: set[str], + foreign_keys: list[dict[str, str]], + joins: list[dict[str, Any]], + semantic_fks: list[dict[str, str]], +) -> tuple[list[dict[str, str]], list[dict[str, Any]], list[dict[str, str]]]: + catalog = _read_catalog(native) + native_joins = [ + item + for item in native.get("data_layer", {}).get("joins") or [] + if not ( + str(item.get("source_table_id") or "") in represented_table_ids + and str(item.get("target_table_id") or "") in represented_table_ids + ) + ] + native_foreign_keys = [] + for item in native.get("data_layer", {}).get("foreign_keys") or []: + source = catalog["columns"].get(str(item.get("source_column_id") or "")) + target = catalog["columns"].get(str(item.get("target_column_id") or "")) + if ( + source + and target + and source["table_id"] in represented_table_ids + and target["table_id"] in represented_table_ids + ): + continue + native_foreign_keys.append(item) + + attribute_tables: dict[str, str] = {} + for term in native.get("semantic_layer", {}).get("terms") or []: + represents = term.get("represents") or [] + if len(represents) != 1: + continue + for attribute in term.get("columns_attributes") or []: + if attribute.get("id"): + attribute_tables[str(attribute["id"])] = str(represents[0]) + native_semantic_fks = [] + for item in native.get("semantic_layer", {}).get("semantic_fks") or []: + source = catalog["columns"].get(str(item.get("column_id") or "")) + target_table_id = attribute_tables.get( + str(item.get("column_attribute_id") or "") + ) + if ( + source + and source["table_id"] in represented_table_ids + and target_table_id in represented_table_ids + ): + continue + native_semantic_fks.append(item) + + return ( + _merge_records(native_foreign_keys, foreign_keys), + _merge_records(native_joins, joins), + _merge_records(native_semantic_fks, semantic_fks), + ) + + +def _merge_records( + preserved: Any, generated: list[dict[str, Any]] +) -> list[dict[str, Any]]: + result = [deepcopy(item) for item in preserved or [] if isinstance(item, dict)] + serialized = {json.dumps(item, sort_keys=True) for item in result} + for item in generated: + marker = json.dumps(item, sort_keys=True) + if marker not in serialized: + result.append(item) + serialized.add(marker) + return result + + +def _stable_id(kind: str, *parts: str) -> str: + return str(uuid5(_ID_NAMESPACE, "/".join((kind, *map(str, parts))))) + + +def _gsf_extension_data(item: Mapping[str, Any]) -> dict[str, Any]: + for extension in item.get("custom_extensions") or []: + if not isinstance(extension, dict): + continue + if extension.get("vendor_name") not in GSF_VENDOR_ALIASES: + continue + try: + data = json.loads(str(extension.get("data") or "{}")) + except json.JSONDecodeError: + continue + if isinstance(data, dict): + return data + return {} + + +def _native_snapshot(model: Mapping[str, Any]) -> dict[str, Any] | None: + snapshot = _gsf_extension_data(model).get("native_document") + return snapshot if isinstance(snapshot, dict) else None + + +def _gsf_extension(data: Mapping[str, Any]) -> dict[str, str]: + return { + "vendor_name": NVIDIA_GSF_VENDOR, + "data": json.dumps(data, separators=(",", ":"), sort_keys=True), + } + + +def _dump_yaml(value: dict[str, Any]) -> str: + return yaml.safe_dump( + value, + default_flow_style=False, + sort_keys=False, + allow_unicode=True, + ) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Convert Apache Ossie YAML and native NVIDIA GSF model YAML" + ) + subparsers = parser.add_subparsers(dest="command", required=True) + export_parser = subparsers.add_parser( + "export", + help="Convert Ossie YAML to a native GSF model document", + ) + export_parser.add_argument("-i", "--input", type=Path, required=True) + export_parser.add_argument("-o", "--output", type=Path) + export_parser.add_argument( + "--database-name", + help="Default database for Ossie schema.table sources", + ) + import_parser = subparsers.add_parser( + "import", + help="Convert a native GSF model document to Ossie YAML", + ) + import_parser.add_argument("-i", "--input", type=Path, required=True) + import_parser.add_argument("-o", "--output", type=Path) + import_parser.add_argument( + "--name", + help="Override the inferred Ossie semantic-model name", + ) + return parser + + +def main(argv: list[str] | None = None) -> None: + args = _build_parser().parse_args(argv) + try: + source = args.input.read_text(encoding="utf-8") + if args.command == "export": + output = convert_ossie_to_gsf( + source, + database_name=args.database_name, + ) + else: + output = convert_gsf_to_ossie(source, model_name=args.name) + if args.output is None: + print(output, end="") + else: + args.output.write_text(output, encoding="utf-8") + except (GSFConversionError, OSError, UnicodeError) as exc: + print(f"Error: {exc}", file=sys.stderr) + raise SystemExit(1) from exc + + +if __name__ == "__main__": + main() diff --git a/converters/gsf/tests/fixtures/sales.gsf.yaml b/converters/gsf/tests/fixtures/sales.gsf.yaml index f5f879fc..97c1479b 100644 --- a/converters/gsf/tests/fixtures/sales.gsf.yaml +++ b/converters/gsf/tests/fixtures/sales.gsf.yaml @@ -1,96 +1,143 @@ -version: '1.0' -model: - name: sales - database: analytics - description: Sales model - ai_context: - instructions: Use approved metrics -terms: -- name: orders - source: - database: analytics - schema: public - table: orders - description: Orders - synonyms: - - purchases - primary_key: - - order_id - column_attributes: - - name: order_id - source_column: order_id - expressions: - - dialect: ANSI_SQL - expression: order_id - dimension: - is_time: false - - name: customer_id - source_column: customer_id - expressions: - - dialect: ANSI_SQL - expression: customer_id - dimension: - is_time: false - - name: order_date - source_column: order_date - expressions: - - dialect: ANSI_SQL - expression: order_date - dimension: - is_time: true +data_layer: + databases: + - id: 7667e398-1abf-5314-9d5d-f91a6d5d0887 + dialect: '' + schemas: + - id: f6930fd5-1fa4-518a-93e4-2eb9b68ef7a3 + name: public + database_name: analytics + tables: + - id: 23494a50-4085-5c50-b824-3777fc32650a + name: customers + description: '' + pk: + - customer_id + type: '' + columns: + - id: d2dc27f6-a3a9-5d47-a081-2693cf29fc47 + name: customer_id + description: '' + type: '' + sample_values: [] + is_nullable: false + is_unique: true + - id: 8842de13-4e63-5140-9bd1-1f7a75ca9ca4 + name: name + description: '' + type: '' + sample_values: [] + is_nullable: true + is_unique: false + - id: 6863d3fc-fa02-50cd-8ec8-d026ed4bfc61 + name: orders + description: Orders + pk: + - order_id + type: '' + columns: + - id: a65e4cd2-521c-5146-9e48-eb9418e73007 + name: customer_id + description: '' + type: '' + sample_values: [] + is_nullable: true + is_unique: false + - id: 8a9e2c90-d32e-5df3-a9cc-a599aaaf6cfc + name: order_id + description: '' + type: '' + sample_values: [] + is_nullable: false + is_unique: true + - id: 91180d56-8e70-57ea-8973-3f8315741a7e + name: order_date + description: '' + type: '' + sample_values: [] + is_nullable: true + is_unique: false + - id: 6db16cad-de44-50f4-8225-3c943e185f72 + name: subtotal + description: '' + type: '' + sample_values: [] + is_nullable: true + is_unique: false + - id: 94e89a19-b0e4-597d-a8e5-4e4f2c5432f9 + name: discount + description: '' + type: '' + sample_values: [] + is_nullable: true + is_unique: false + foreign_keys: + - source_column_id: a65e4cd2-521c-5146-9e48-eb9418e73007 + target_column_id: d2dc27f6-a3a9-5d47-a081-2693cf29fc47 + joins: + - source_table_id: 6863d3fc-fa02-50cd-8ec8-d026ed4bfc61 + target_table_id: 23494a50-4085-5c50-b824-3777fc32650a + join_columns: + - source: customer_id + target: customer_id +semantic_layer: + terms: + - id: e121e6fb-82bc-5daf-9926-5e0c42a6fca8 + name: orders + description: Orders + represents: + - 6863d3fc-fa02-50cd-8ec8-d026ed4bfc61 + columns_attributes: + - id: 97035227-36d5-564a-ba32-78d8f73a2ddc + name: order_id + description: '' + column_id: 8a9e2c90-d32e-5df3-a9cc-a599aaaf6cfc + - id: ec03aaa3-7abf-5484-9f13-1590e7cb37a5 + name: customer_id + description: '' + column_id: a65e4cd2-521c-5146-9e48-eb9418e73007 + - id: 74dc494d-d54d-55f0-ae8e-fa5826da2c3b + name: order_date + description: '' + column_id: 91180d56-8e70-57ea-8973-3f8315741a7e + - id: 4ca1db6b-c53a-5f97-949a-b466a92f5ce7 + name: customers + description: '' + represents: + - 23494a50-4085-5c50-b824-3777fc32650a + columns_attributes: + - id: 0d6c3859-c00a-57d1-b297-e385ff6e1162 + name: customer_id + description: '' + column_id: d2dc27f6-a3a9-5d47-a081-2693cf29fc47 + - id: 6a1c1af4-f808-5b2e-86a2-1eae3b7c79d3 + name: customer_name + description: '' + column_id: 8842de13-4e63-5140-9bd1-1f7a75ca9ca4 + semantic_fks: + - column_attribute_id: 0d6c3859-c00a-57d1-b297-e385ff6e1162 + column_id: a65e4cd2-521c-5146-9e48-eb9418e73007 sql_attributes: - - name: net_total - kind: field - expressions: - - dialect: ANSI_SQL - expression: subtotal - discount - sql: SELECT subtotal - discount AS "net_total" FROM "analytics"."public"."orders" - AS "orders" - table_refs: - - orders - dimension: - is_time: false - - name: revenue_per_customer - kind: metric - expressions: - - dialect: ANSI_SQL - expression: SUM(orders.subtotal) / COUNT(DISTINCT customers.customer_id) - - dialect: SNOWFLAKE - expression: SUM(orders.subtotal)::NUMBER / COUNT(DISTINCT customers.customer_id) + manual: + - id: 02c68281-7f5c-59dc-8660-d3bef42fe78c + name: net_total + description: '' + sql: SELECT subtotal - discount AS "net_total" FROM "analytics"."public"."orders" + AS "orders" + sql_column_is: + - 6db16cad-de44-50f4-8225-3c943e185f72 + - 94e89a19-b0e4-597d-a8e5-4e4f2c5432f9 + term_id: e121e6fb-82bc-5daf-9926-5e0c42a6fca8 + table: [] + sql: [] + bridge_table: [] + custom_analyses: + - id: fde1d7f0-2234-5f63-8713-176b579be785 + name: revenue_per_customer + description: Revenue per customer sql: SELECT SUM(orders.subtotal) / COUNT(DISTINCT customers.customer_id) AS "revenue_per_customer" FROM "analytics"."public"."orders" AS "orders" JOIN "analytics"."public"."customers" AS "customers" ON "orders"."customer_id" = "customers"."customer_id" - table_refs: - - orders - - customers - description: Revenue per customer -- name: customers - source: - database: analytics - schema: public - table: customers - primary_key: - - customer_id - column_attributes: - - name: customer_id - source_column: customer_id - expressions: - - dialect: ANSI_SQL - expression: customer_id - dimension: - is_time: false - - name: customer_name - source_column: name - expressions: - - dialect: ANSI_SQL - expression: name - dimension: - is_time: false -semantic_foreign_keys: -- name: orders_to_customers - from_term: orders - to_term: customers - from_columns: - - customer_id - to_columns: - - customer_id + sql_column_is: + - 6db16cad-de44-50f4-8225-3c943e185f72 + - d2dc27f6-a3a9-5d47-a081-2693cf29fc47 +zones: [] diff --git a/converters/gsf/tests/test_converter.py b/converters/gsf/tests/test_converter.py index f4d502e7..bc998459 100644 --- a/converters/gsf/tests/test_converter.py +++ b/converters/gsf/tests/test_converter.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Tests for the offline Apache Ossie ↔ NVIDIA GSF converter.""" +"""Tests for Apache Ossie ↔ native NVIDIA GSF conversion.""" from __future__ import annotations @@ -30,12 +30,11 @@ from ossie_gsf.converter import ( GSFConversionError, - _parse_source, - _simple_source_column, convert_gsf_to_ossie, convert_ossie_to_gsf, main, ) +from ossie_gsf.native_converter import _parse_source, _simple_source_column OSSIE_VERSION = "0.2.0.dev0" FIXTURES = Path(__file__).parent / "fixtures" @@ -46,20 +45,73 @@ def _ossie_yaml() -> str: return (FIXTURES / "sales.ossie.yaml").read_text(encoding="utf-8") -def test_checked_in_fixture_pair_matches_conversion() -> None: - expected = yaml.safe_load((FIXTURES / "sales.gsf.yaml").read_text(encoding="utf-8")) +def _gsf_yaml() -> str: + return (FIXTURES / "sales.gsf.yaml").read_text(encoding="utf-8") + + +def _native_extension(item: dict[str, Any]) -> dict[str, Any]: + extension = next( + value + for value in item.get("custom_extensions") or [] + if value["vendor_name"] == "NVIDIA_GSF" + ) + return json.loads(extension["data"]) + + +def _ids(value: Any) -> set[str]: + result: set[str] = set() + if isinstance(value, dict): + if isinstance(value.get("id"), str): + result.add(value["id"]) + for child in value.values(): + result.update(_ids(child)) + elif isinstance(value, list): + for child in value: + result.update(_ids(child)) + return result + + +def test_checked_in_fixture_is_exact_native_contract() -> None: + expected = yaml.safe_load(_gsf_yaml()) actual = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) assert actual == expected + assert set(actual) == {"data_layer", "semantic_layer", "zones"} + assert "version" not in actual + assert "model" not in actual + assert "terms" not in actual + assert set(actual["semantic_layer"]["sql_attributes"]) == { + "manual", + "table", + "sql", + "bridge_table", + } + assert "columns_attributes" in actual["semantic_layer"]["terms"][0] + + +def test_ossie_ids_are_deterministic_and_references_resolve() -> None: + first = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) + second = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) + + assert _ids(first) == _ids(second) + assert first == second + catalog_column_ids = { + column["id"] + for database in first["data_layer"]["databases"] + for schema in database["schemas"] + for table in schema["tables"] + for column in table["columns"] + } + for attribute in first["semantic_layer"]["sql_attributes"]["manual"]: + assert set(attribute["sql_column_is"]) <= catalog_column_ids + for analysis in first["semantic_layer"]["custom_analyses"]: + assert analysis["sql_column_is"] + assert set(analysis["sql_column_is"]) <= catalog_column_ids def test_generated_ossie_passes_official_validation(tmp_path: Path) -> None: - gsf_yaml = (FIXTURES / "sales.gsf.yaml").read_text(encoding="utf-8") output_path = tmp_path / "converted.ossie.yaml" - output_path.write_text( - convert_gsf_to_ossie(gsf_yaml), - encoding="utf-8", - ) + output_path.write_text(convert_gsf_to_ossie(_gsf_yaml()), encoding="utf-8") result = subprocess.run( [sys.executable, str(VALIDATOR), str(output_path)], @@ -72,154 +124,502 @@ def test_generated_ossie_passes_official_validation(tmp_path: Path) -> None: assert "Validation PASSED" in result.stdout -def test_ossie_to_gsf_is_offline_and_maps_graph_entities() -> None: - result = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) +def test_round_trip_preserves_ossie_semantics_and_global_metrics() -> None: + result = yaml.safe_load(convert_gsf_to_ossie(convert_ossie_to_gsf(_ossie_yaml()))) + model = result["semantic_model"][0] + datasets = {dataset["name"]: dataset for dataset in model["datasets"]} + order_fields = {field["name"]: field for field in datasets["orders"]["fields"]} - assert result["version"] == "1.0" - assert result["model"] == { - "name": "sales", - "database": "analytics", - "description": "Sales model", - "ai_context": {"instructions": "Use approved metrics"}, - } - terms = {term["name"]: term for term in result["terms"]} - assert len(terms["orders"]["column_attributes"]) == 3 - sql_attributes = { - attribute["name"]: attribute for attribute in terms["orders"]["sql_attributes"] - } - assert sql_attributes["net_total"]["kind"] == "field" - metric = sql_attributes["revenue_per_customer"] - assert metric["kind"] == "metric" - assert metric["table_refs"] == ["orders", "customers"] - assert " JOIN " in metric["sql"] - assert len(metric["expressions"]) == 2 - assert result["semantic_foreign_keys"][0] == { + assert model["name"] == "analytics" + assert datasets["orders"]["primary_key"] == ["order_id"] + assert ( + order_fields["net_total"]["expression"]["dialects"][0]["expression"] + == "subtotal - discount" + ) + assert [metric["name"] for metric in model["metrics"]] == ["revenue_per_customer"] + assert _native_extension(model)["native_document"]["zones"] == [] + assert model["relationships"][0] == { "name": "orders_to_customers", - "from_term": "orders", - "to_term": "customers", + "from": "orders", + "to": "customers", "from_columns": ["customer_id"], "to_columns": ["customer_id"], } -def test_gsf_to_ossie_round_trip_preserves_semantics() -> None: - gsf_yaml = convert_ossie_to_gsf(_ossie_yaml()) - result = yaml.safe_load(convert_gsf_to_ossie(gsf_yaml)) +def test_edited_ossie_expressions_replace_preserved_native_sql() -> None: + ossie = yaml.safe_load(convert_gsf_to_ossie(_gsf_yaml())) + model = ossie["semantic_model"][0] + orders = next( + dataset for dataset in model["datasets"] if dataset["name"] == "orders" + ) + net_total = next( + field for field in orders["fields"] if field["name"] == "net_total" + ) + net_total["expression"]["dialects"][0]["expression"] = "subtotal + discount" + model["metrics"][0]["expression"]["dialects"][0]["expression"] = ( + "SUM(orders.discount)" + ) - assert set(result) == {"version", "semantic_model"} - model = result["semantic_model"][0] - assert model["name"] == "sales" - datasets = {dataset["name"]: dataset for dataset in model["datasets"]} - assert datasets["orders"]["primary_key"] == ["order_id"] - assert datasets["orders"]["ai_context"]["synonyms"] == ["purchases"] - fields = {field["name"]: field for field in datasets["orders"]["fields"]} - assert ( - fields["net_total"]["expression"]["dialects"][0]["expression"] - == "subtotal - discount" + regenerated = yaml.safe_load( + convert_ossie_to_gsf(yaml.safe_dump(ossie, sort_keys=False)) ) - metric = model["metrics"][0] - assert len(metric["expression"]["dialects"]) == 2 - extension = next( - item - for item in metric["custom_extensions"] - if item["vendor_name"] == "NVIDIA_GSF" + sql_attribute = regenerated["semantic_layer"]["sql_attributes"]["manual"][0] + analysis = regenerated["semantic_layer"]["custom_analyses"][0] + + assert "subtotal + discount" in sql_attribute["sql"] + assert "subtotal - discount" not in sql_attribute["sql"] + assert "SUM(orders.discount)" in analysis["sql"] + assert "COUNT(DISTINCT customers.customer_id)" not in analysis["sql"] + + +def test_native_round_trip_preserves_ids_catalog_sql_source_and_zones() -> None: + native = yaml.safe_load(_gsf_yaml()) + database = native["data_layer"]["databases"][0] + database["dialect"] = "snowflake" + first_column = database["schemas"][0]["tables"][0]["columns"][0] + first_column["type"] = "NUMBER" + first_column["sample_values"] = ["1", "2"] + database["schemas"][0]["tables"].append( + { + "id": "native-audit-table", + "name": "audit_log", + "description": "Catalog-only table", + "pk": [], + "type": "table", + "columns": [ + { + "id": "native-audit-column", + "name": "message", + "description": "", + "type": "TEXT", + "sample_values": [], + "is_nullable": True, + "is_unique": False, + } + ], + } ) - assert json.loads(extension["data"])["term"] == "orders" - assert model["relationships"][0]["from"] == "orders" + native["zones"] = [{"id": "zone-1", "name": "finance"}] + manual = native["semantic_layer"]["sql_attributes"]["manual"] + native["semantic_layer"]["sql_attributes"]["table"] = manual + native["semantic_layer"]["sql_attributes"]["manual"] = [] + ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) + assert _native_extension(ossie["semantic_model"][0])["native_document"] == native -def test_gsf_sql_field_preserves_sql_and_multi_term_references() -> None: - gsf = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) - orders = next(term for term in gsf["terms"] if term["name"] == "orders") - field = next( - item for item in orders["sql_attributes"] if item["name"] == "net_total" + restored = yaml.safe_load( + convert_ossie_to_gsf(yaml.safe_dump(ossie, sort_keys=False)) ) - field["table_refs"] = ["orders", "customers"] - field["sql"] = "SELECT custom_joined_value FROM orders JOIN customers" - - ossie = convert_gsf_to_ossie(yaml.safe_dump(gsf)) - round_trip = yaml.safe_load(convert_ossie_to_gsf(ossie)) - round_trip_orders = next( - term for term in round_trip["terms"] if term["name"] == "orders" + restored_database = restored["data_layer"]["databases"][0] + restored_first_column = restored_database["schemas"][0]["tables"][0]["columns"][0] + + assert _ids(restored) == _ids(native) + assert restored_database["dialect"] == "snowflake" + assert restored_first_column["type"] == "NUMBER" + assert restored_first_column["sample_values"] == ["1", "2"] + assert any( + table["id"] == "native-audit-table" + for table in restored_database["schemas"][0]["tables"] ) - round_trip_field = next( - item - for item in round_trip_orders["sql_attributes"] - if item["name"] == "net_total" + assert restored["semantic_layer"]["sql_attributes"]["manual"] == [] + assert ( + restored["semantic_layer"]["sql_attributes"]["table"][0]["id"] + == manual[0]["id"] ) + assert restored["zones"] == [{"id": "zone-1", "name": "finance"}] - assert round_trip_field["table_refs"] == ["orders", "customers"] - assert round_trip_field["sql"] == field["sql"] +def test_relationship_edits_replace_preserved_native_records() -> None: + ossie = yaml.safe_load(convert_gsf_to_ossie(_gsf_yaml())) + relationship = ossie["semantic_model"][0]["relationships"][0] + relationship["from_columns"] = ["order_id"] -def test_ossie_extensions_round_trip_through_native_metadata() -> None: - root = yaml.safe_load(_ossie_yaml()) - root["semantic_model"][0]["custom_extensions"] = [ - {"vendor_name": "DBT", "data": '{"project": "analytics"}'} + regenerated = yaml.safe_load( + convert_ossie_to_gsf(yaml.safe_dump(ossie, sort_keys=False)) + ) + + assert regenerated["data_layer"]["joins"][0]["join_columns"] == [ + {"source": "order_id", "target": "customer_id"} ] + orders_table = next( + table + for table in regenerated["data_layer"]["databases"][0]["schemas"][0]["tables"] + if table["name"] == "orders" + ) + order_id = next( + column["id"] + for column in orders_table["columns"] + if column["name"] == "order_id" + ) + assert regenerated["data_layer"]["foreign_keys"][0]["source_column_id"] == order_id + - native = yaml.safe_load(convert_ossie_to_gsf(yaml.safe_dump(root))) - extensions = native["model"]["metadata"]["apache_ossie"]["custom_extensions"] - assert extensions[0]["vendor_name"] == "DBT" +def test_relationship_deletion_removes_preserved_native_records() -> None: + ossie = yaml.safe_load(convert_gsf_to_ossie(_gsf_yaml())) + ossie["semantic_model"][0].pop("relationships") + + regenerated = yaml.safe_load( + convert_ossie_to_gsf(yaml.safe_dump(ossie, sort_keys=False)) + ) + + assert regenerated["data_layer"]["joins"] == [] + assert regenerated["data_layer"]["foreign_keys"] == [] + assert regenerated["semantic_layer"]["semantic_fks"] == [] + + +def test_relationship_reconciliation_preserves_catalog_only_records() -> None: + native = yaml.safe_load(_gsf_yaml()) + schema = native["data_layer"]["databases"][0]["schemas"][0] + orders = next(table for table in schema["tables"] if table["name"] == "orders") + order_id = next( + column["id"] for column in orders["columns"] if column["name"] == "order_id" + ) + schema["tables"].append( + { + "id": "audit-table", + "name": "audit_log", + "description": "", + "pk": [], + "type": "table", + "columns": [ + { + "id": "audit-column", + "name": "order_id", + "description": "", + "type": "", + "sample_values": [], + "is_nullable": True, + "is_unique": False, + } + ], + } + ) + audit_join = { + "source_table_id": orders["id"], + "target_table_id": "audit-table", + "join_columns": [{"source": "order_id", "target": "order_id"}], + } + audit_fk = { + "source_column_id": order_id, + "target_column_id": "audit-column", + } + native["data_layer"]["joins"].append(audit_join) + native["data_layer"]["foreign_keys"].append(audit_fk) ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) - assert ossie["semantic_model"][0]["custom_extensions"][0]["vendor_name"] == "DBT" + ossie["semantic_model"][0].pop("relationships") + regenerated = yaml.safe_load( + convert_ossie_to_gsf(yaml.safe_dump(ossie, sort_keys=False)) + ) + assert regenerated["data_layer"]["joins"] == [audit_join] + assert regenerated["data_layer"]["foreign_keys"] == [audit_fk] -def test_gsf_to_ossie_allows_model_name_override() -> None: - output = convert_gsf_to_ossie( - convert_ossie_to_gsf(_ossie_yaml()), - model_name="renamed_sales", + +def test_multiple_databases_are_supported_and_name_falls_back() -> None: + ossie = yaml.safe_load(_ossie_yaml()) + model = ossie["semantic_model"][0] + model["datasets"][1]["source"] = "crm.public.customers" + model["relationships"] = [] + model["metrics"] = [] + + native_yaml = convert_ossie_to_gsf(yaml.safe_dump(ossie)) + native = yaml.safe_load(native_yaml) + database_names = { + schema["database_name"] + for database in native["data_layer"]["databases"] + for schema in database["schemas"] + } + restored = yaml.safe_load(convert_gsf_to_ossie(native_yaml)) + + assert database_names == {"analytics", "crm"} + assert len(native["data_layer"]["databases"]) == 2 + assert restored["semantic_model"][0]["name"] == "gsf_model" + + +def test_shared_physical_source_uses_one_catalog_table_and_valid_ossie( + tmp_path: Path, +) -> None: + ossie = yaml.safe_load(_ossie_yaml()) + model = ossie["semantic_model"][0] + model["datasets"].append( + { + "name": "order_amounts", + "source": "analytics.public.orders", + "fields": [ + { + "name": "subtotal", + "expression": { + "dialects": [{"dialect": "ANSI_SQL", "expression": "subtotal"}] + }, + } + ], + } + ) + + native_yaml = convert_ossie_to_gsf(yaml.safe_dump(ossie, sort_keys=False)) + native = yaml.safe_load(native_yaml) + order_tables = [ + table + for database in native["data_layer"]["databases"] + for schema in database["schemas"] + for table in schema["tables"] + if table["name"] == "orders" + ] + represented_ids = { + term["name"]: term["represents"][0] + for term in native["semantic_layer"]["terms"] + } + + assert len(order_tables) == 1 + assert {column["name"] for column in order_tables[0]["columns"]} >= { + "order_id", + "subtotal", + "discount", + } + assert represented_ids["orders"] == represented_ids["order_amounts"] + + restored = yaml.safe_load(convert_gsf_to_ossie(native_yaml)) + assert { + dataset["name"] for dataset in restored["semantic_model"][0]["datasets"] + } >= {"orders", "order_amounts"} + output_path = tmp_path / "shared-source.ossie.yaml" + output_path.write_text(yaml.safe_dump(restored), encoding="utf-8") + result = subprocess.run( + [sys.executable, str(VALIDATOR), str(output_path)], + check=False, + capture_output=True, + text=True, ) - assert yaml.safe_load(output)["semantic_model"][0]["name"] == ("renamed_sales") + assert result.returncode == 0, result.stdout + result.stderr -def test_disconnected_metric_fails_instead_of_cross_join() -> None: - root = yaml.safe_load(_ossie_yaml()) - root["semantic_model"][0]["relationships"] = [] +def test_cross_database_ossie_metric_is_rejected() -> None: + ossie = yaml.safe_load(_ossie_yaml()) + ossie["semantic_model"][0]["datasets"][1]["source"] = "crm.public.customers" - with pytest.raises(GSFConversionError, match="disconnected dataset"): - convert_ossie_to_gsf(yaml.safe_dump(root)) + with pytest.raises(GSFConversionError, match="spans multiple databases"): + convert_ossie_to_gsf(yaml.safe_dump(ossie)) -def test_duplicate_physical_column_mapping_is_rejected() -> None: - root = yaml.safe_load(_ossie_yaml()) - root["semantic_model"][0]["datasets"][0]["fields"].append( +def test_cross_database_full_query_field_is_rejected() -> None: + ossie = yaml.safe_load(_ossie_yaml()) + model = ossie["semantic_model"][0] + model["datasets"][1]["source"] = "crm.public.customers" + model["metrics"] = [] + model["relationships"] = [] + model["datasets"][0]["fields"].append( { - "name": "alternate_order_id", + "name": "remote_customer", "expression": { "dialects": [ { "dialect": "ANSI_SQL", - "expression": "order_id", + "expression": ( + "SELECT customers.customer_id " + "FROM crm.public.customers AS customers" + ), } ] }, } ) - with pytest.raises(GSFConversionError, match="Multiple fields"): - convert_ossie_to_gsf(yaml.safe_dump(root)) + with pytest.raises(GSFConversionError, match="SQL attribute.*multiple databases"): + convert_ossie_to_gsf(yaml.safe_dump(ossie)) + + +@pytest.mark.parametrize("kind", ["sql_attribute", "custom_analysis"]) +def test_cross_database_gsf_sql_objects_are_rejected(kind: str) -> None: + ossie = yaml.safe_load(_ossie_yaml()) + model = ossie["semantic_model"][0] + model["datasets"][1]["source"] = "crm.public.customers" + model["metrics"] = [] + model["relationships"] = [] + native = yaml.safe_load(convert_ossie_to_gsf(yaml.safe_dump(ossie))) + terms = {term["name"]: term for term in native["semantic_layer"]["terms"]} + columns = { + (schema["database_name"], table["name"], column["name"]): column["id"] + for database in native["data_layer"]["databases"] + for schema in database["schemas"] + for table in schema["tables"] + for column in table["columns"] + } + sql = ( + "SELECT orders.order_id, customers.customer_id " + "FROM analytics.public.orders AS orders " + "JOIN crm.public.customers AS customers " + "ON orders.customer_id = customers.customer_id" + ) + sql_column_is = [ + columns[("analytics", "orders", "order_id")], + columns[("crm", "customers", "customer_id")], + ] + if kind == "sql_attribute": + native["semantic_layer"]["sql_attributes"]["manual"].append( + { + "id": "cross-db-attribute", + "name": "cross_db", + "description": "", + "sql": sql, + "sql_column_is": sql_column_is, + "term_id": terms["orders"]["id"], + } + ) + else: + native["semantic_layer"]["custom_analyses"].append( + { + "id": "cross-db-analysis", + "name": "cross_db", + "description": "", + "sql": sql, + "sql_column_is": sql_column_is, + } + ) + + with pytest.raises(GSFConversionError, match="spans multiple databases"): + convert_gsf_to_ossie(yaml.safe_dump(native)) + + +def test_relationships_emit_join_physical_fk_and_semantic_fk() -> None: + native = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) + + assert len(native["data_layer"]["joins"]) == 1 + assert native["data_layer"]["joins"][0]["join_columns"] == [ + {"source": "customer_id", "target": "customer_id"} + ] + assert len(native["data_layer"]["foreign_keys"]) == 1 + assert len(native["semantic_layer"]["semantic_fks"]) == 1 + + native["data_layer"]["joins"] = [] + restored = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) + assert restored["semantic_model"][0]["relationships"][0]["from"] == "orders" + assert restored["semantic_model"][0]["relationships"][0]["to"] == "customers" + + +def test_gsf_requires_one_represented_table_per_term() -> None: + native = yaml.safe_load(_gsf_yaml()) + term = native["semantic_layer"]["terms"][0] + term["represents"].append(native["semantic_layer"]["terms"][1]["represents"][0]) + + with pytest.raises(GSFConversionError, match="exactly one table"): + convert_gsf_to_ossie(yaml.safe_dump(native)) + + +def test_duplicate_gsf_term_names_are_rejected() -> None: + native = yaml.safe_load(_gsf_yaml()) + native["semantic_layer"]["terms"][1]["name"] = "orders" + + with pytest.raises(GSFConversionError, match="Duplicate GSF term name"): + convert_gsf_to_ossie(yaml.safe_dump(native)) + + +def test_duplicate_gsf_field_names_across_attribute_kinds_are_rejected() -> None: + native = yaml.safe_load(_gsf_yaml()) + native["semantic_layer"]["sql_attributes"]["manual"][0]["name"] = "order_id" + with pytest.raises(GSFConversionError, match="Duplicate field name"): + convert_gsf_to_ossie(yaml.safe_dump(native)) -def test_multiple_catalog_databases_are_rejected() -> None: - root = yaml.safe_load(_ossie_yaml()) - root["semantic_model"][0]["datasets"][1]["source"] = "crm.public.customers" - with pytest.raises(GSFConversionError, match="exactly one database"): - convert_ossie_to_gsf(yaml.safe_dump(root)) +def test_catalog_only_gsf_has_no_representable_terms() -> None: + native = yaml.safe_load(_gsf_yaml()) + native["semantic_layer"]["terms"] = [] + native["semantic_layer"]["sql_attributes"]["manual"] = [] + native["semantic_layer"]["custom_analyses"] = [] + with pytest.raises(GSFConversionError, match="no representable terms"): + convert_gsf_to_ossie(yaml.safe_dump(native)) -def test_wrong_versions_are_rejected() -> None: + +@pytest.mark.parametrize( + ("expression", "unit"), + [ + ("DATEDIFF(day, order_date, CURRENT_TIMESTAMP())", "day"), + ("DATEDIFF(hour, order_date, CURRENT_TIMESTAMP())", "hour"), + ("TIMESTAMPDIFF(second, order_date, CURRENT_TIMESTAMP())", "second"), + ("DATEADD(month, 1, order_date)", "month"), + ], +) +def test_date_part_keywords_do_not_become_catalog_columns( + expression: str, + unit: str, +) -> None: ossie = yaml.safe_load(_ossie_yaml()) - ossie["version"] = "0.1" - with pytest.raises(GSFConversionError, match="Unsupported Ossie"): - convert_ossie_to_gsf(yaml.safe_dump(ossie)) + ossie["semantic_model"][0]["datasets"][0]["fields"].append( + { + "name": "order_age", + "expression": { + "dialects": [{"dialect": "ANSI_SQL", "expression": expression}] + }, + } + ) + + native = yaml.safe_load(convert_ossie_to_gsf(yaml.safe_dump(ossie))) + orders = next( + table + for database in native["data_layer"]["databases"] + for schema in database["schemas"] + for table in schema["tables"] + if table["name"] == "orders" + ) + column_names = {column["name"] for column in orders["columns"]} + attribute = next( + item + for item in native["semantic_layer"]["sql_attributes"]["manual"] + if item["name"] == "order_age" + ) + referenced = {column["id"]: column["name"] for column in orders["columns"]} + + assert unit not in column_names + assert "order_date" in column_names + assert unit not in {referenced.get(item) for item in attribute["sql_column_is"]} + + +def test_gsf_sourced_catalog_is_never_widened_by_sql_identifiers() -> None: + native = yaml.safe_load(_gsf_yaml()) + manual = native["semantic_layer"]["sql_attributes"]["manual"][0] + manual["sql"] = ( + "SELECT DATEDIFF(day, order_date, CURRENT_TIMESTAMP()) + not_a_real_column " + 'AS "net_total" FROM "analytics"."public"."orders" AS "orders"' + ) + before = { + column["id"] + for database in native["data_layer"]["databases"] + for schema in database["schemas"] + for table in schema["tables"] + for column in table["columns"] + } + + ossie = convert_gsf_to_ossie(yaml.safe_dump(native)) + restored = yaml.safe_load(convert_ossie_to_gsf(ossie)) + after_columns = [ + column + for database in restored["data_layer"]["databases"] + for schema in database["schemas"] + for table in schema["tables"] + for column in table["columns"] + ] + + assert {column["id"] for column in after_columns} == before + assert "not_a_real_column" not in {column["name"] for column in after_columns} + + +def test_old_fictional_gsf_root_is_rejected() -> None: + old_shape = { + "version": "1.0", + "model": {"name": "sales"}, + "terms": [], + } - gsf = yaml.safe_load(convert_ossie_to_gsf(_ossie_yaml())) - gsf["version"] = "2.0" - with pytest.raises(GSFConversionError, match="Unsupported GSF"): - convert_gsf_to_ossie(yaml.safe_dump(gsf)) + with pytest.raises(GSFConversionError, match="Unsupported GSF root"): + convert_gsf_to_ossie(yaml.safe_dump(old_shape)) + + +def test_model_name_override() -> None: + result = yaml.safe_load(convert_gsf_to_ossie(_gsf_yaml(), model_name="sales")) + assert result["semantic_model"][0]["name"] == "sales" @pytest.mark.parametrize( @@ -243,19 +643,6 @@ def test_wrong_versions_are_rejected() -> None: "table": "orders", }, ), - ( - { - "database": "analytics", - "schema": "public", - "table": "orders", - }, - None, - { - "database": "analytics", - "schema": "public", - "table": "orders", - }, - ), ], ) def test_parse_source( @@ -272,17 +659,13 @@ def test_parse_source( ("order_id", "order_id"), ("orders.order_id", "order_id"), ("subtotal - discount", None), - ("UPPER(name)", None), ], ) -def test_simple_source_column( - expression: str, - expected: str | None, -) -> None: +def test_simple_source_column(expression: str, expected: str | None) -> None: assert _simple_source_column(expression, "orders", "orders") == expected -def test_cli_converts_files( +def test_cli_converts_native_files( tmp_path: Path, capsys: pytest.CaptureFixture[str], ) -> None: @@ -291,8 +674,13 @@ def test_cli_converts_files( ossie_path.write_text(_ossie_yaml(), encoding="utf-8") main(["export", "-i", str(ossie_path), "-o", str(gsf_path)]) - assert yaml.safe_load(gsf_path.read_text())["version"] == "1.0" + assert set(yaml.safe_load(gsf_path.read_text(encoding="utf-8"))) == { + "data_layer", + "semantic_layer", + "zones", + } - main(["import", "-i", str(gsf_path)]) + main(["import", "-i", str(gsf_path), "--name", "sales"]) output = yaml.safe_load(capsys.readouterr().out) assert output["version"] == OSSIE_VERSION + assert output["semantic_model"][0]["name"] == "sales" diff --git a/converters/gsf/uv.lock b/converters/gsf/uv.lock index ba54dc32..2be98a07 100644 --- a/converters/gsf/uv.lock +++ b/converters/gsf/uv.lock @@ -8,23 +8,25 @@ version = "0.1.0.dev0" source = { editable = "." } dependencies = [ { name = "pyyaml" }, + { name = "sqlglot" }, ] [package.dev-dependencies] dev = [ { name = "jsonschema" }, { name = "pytest" }, - { name = "sqlglot" }, ] [package.metadata] -requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] +requires-dist = [ + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=30.12.0" }, +] [package.metadata.requires-dev] dev = [ { name = "jsonschema", specifier = ">=4.26.0" }, { name = "pytest", specifier = ">=8.0" }, - { name = "sqlglot", specifier = ">=30.12.0" }, ] [[package]] From 972e48636f22762d2e3e8628a97209ca95fe4fd5 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Tue, 28 Jul 2026 09:51:42 -0500 Subject: [PATCH 4/6] Loosen version and SQL-dialect assumptions in GSF converter Accept any 0.2.x Ossie version rather than one exact string, so a patch or dev bump of the spec no longer rejects every model until a constant is hand-edited. Parse SQL across candidate dialects instead of the default parser alone, and treat SQL that no dialect can parse as opaque rather than aborting. Preserved GSF SQL carries whatever dialect its connection reported, so a model that imported cleanly could previously fail to export again. Treat a date-part keyword as a unit only when it occupies the unit argument itself, so a column genuinely named day or month is kept everywhere else in the same call. Drop LAST_DAY, TRUNC and EXTRACT from the function list, whose first argument is data. Label expressions with the source connection's dialect where Ossie names one, and read a preserved native snapshot through a single helper so a hand-edited extension fails alike in both paths. --- converters/gsf/README.md | 30 ++- .../gsf/src/ossie_gsf/native_converter.py | 211 +++++++++++++----- converters/gsf/tests/test_converter.py | 155 ++++++++++++- 3 files changed, 343 insertions(+), 53 deletions(-) diff --git a/converters/gsf/README.md b/converters/gsf/README.md index 2afc1fa3..a1fe19ef 100644 --- a/converters/gsf/README.md +++ b/converters/gsf/README.md @@ -38,7 +38,9 @@ The generated root contains exactly `data_layer`, `semantic_layer`, and `zones`. It does not contain a converter-specific version or model envelope. Catalog columns are collected from fields, primary and unique keys, relationships, and SQL column references. Stable UUIDv5 identifiers make -repeated Ossie exports deterministic. +repeated Ossie exports deterministic. Any Ossie `0.2.x` version is accepted on +input, including `.dev` releases; output is written as the spec version the +converter targets. ## Setup @@ -124,6 +126,32 @@ expressions and relationships remain authoritative: preserved SQL and native relationship records are reused only when they still correspond to the Ossie entities or are outside the represented Ossie catalog scope. +That extension holds the whole native document, so an Ossie file produced from +GSF carries a full copy of the GSF catalog alongside the model derived from it. +This is a deliberate trade of size for round-trip fidelity: it is what lets a +GSF → Ossie → GSF cycle keep live identifiers, and it means the Ossie output of +a large catalog is bulky and not meant to be reviewed by hand. Converting +Ossie → GSF from a hand-written Ossie file, which has no such extension, is +unaffected. + +SQL is parsed with sqlglot across a list of candidate dialects rather than a +single one, because preserved GSF SQL carries whatever dialect its connection +reported. SQL that no candidate can parse is treated as opaque: it is still +carried through verbatim, and only the parse-derived enrichment (discovering +which tables and columns an expression touches) is skipped, so a model that +imported cleanly can always be exported again. On GSF → Ossie, expressions are +labelled with the source connection's dialect when Ossie names it +(`SNOWFLAKE`, `DATABRICKS`, `BIGQUERY`) and `ANSI_SQL` otherwise. + +For an Ossie-origin model there is no GSF catalog to check against, so physical +columns are synthesized from the identifiers in each expression. Date-part +keywords are excluded, but only in the unit argument of a recognized date +function, so a column genuinely named `day` or `month` is kept everywhere else. +A unit passed to a date function the converter does not recognize is still +synthesized as a column; this is inherent to deriving a catalog from SQL text +and does not apply once a GSF catalog is present, since a GSF-sourced catalog +is authoritative and never widened. + The GSF contract has no semantic-model envelope, `ai_context`, dimensions, synonyms, Ossie custom-extension storage, or expression-dialect variants. Those values cannot be represented in a native GSF document and are diff --git a/converters/gsf/src/ossie_gsf/native_converter.py b/converters/gsf/src/ossie_gsf/native_converter.py index b5eb2b4b..6f7a5da8 100644 --- a/converters/gsf/src/ossie_gsf/native_converter.py +++ b/converters/gsf/src/ossie_gsf/native_converter.py @@ -32,9 +32,13 @@ import yaml from sqlglot import exp, parse_one -from sqlglot.errors import ParseError +from sqlglot.errors import ParseError, TokenError OSSIE_VERSION = "0.2.0.dev0" +# Any release in this major.minor series is accepted on input. The spec is +# still on a .dev line, so pinning the exact string would reject every real +# model as soon as the patch or dev suffix moves. +OSSIE_SERIES = tuple(int(part) for part in OSSIE_VERSION.split(".")[:2]) NVIDIA_GSF_VENDOR = "NVIDIA_GSF" GSF_VENDOR_ALIASES = {NVIDIA_GSF_VENDOR, "GSF"} _ID_NAMESPACE = UUID("03d14261-6432-50fe-b099-77e8061af4f9") @@ -43,6 +47,21 @@ r"^(?:(?P[A-Za-z_][A-Za-z0-9_]*)\.)?" r"(?P[A-Za-z_][A-Za-z0-9_]*)$" ) +# sqlglot's default parser first, since it is closest to ANSI, then the +# dialects GSF connections commonly report. +_SQL_DIALECTS = ( + "", + "snowflake", + "databricks", + "bigquery", + "tsql", + "postgres", + "mysql", + "duckdb", + "spark", + "oracle", + "sqlite", +) _DATE_PART_UNITS = frozenset( { "year", @@ -92,7 +111,10 @@ "epoch", } ) -_DATE_FUNCTIONS = frozenset( +# Functions whose *first* argument is a date-part unit rather than data. +# LAST_DAY, TRUNC and EXTRACT are deliberately absent: the first two take data +# there, and EXTRACT's unit uses syntax sqlglot does not parse as a column. +_UNIT_FIRST_FUNCTIONS = frozenset( { "datediff", "date_diff", @@ -118,11 +140,24 @@ "date_part", "datepart", "datename", - "extract", - "last_day", - "trunc", } ) +# Typed sqlglot nodes that hold the unit in their ``this`` slot. DATE_ADD and +# friends are excluded: they keep the unit in ``unit`` and put data in ``this``. +_UNIT_IN_THIS_FUNCTIONS = tuple( + node + for node in ( + getattr(exp, name, None) + for name in ("DateDiff", "TimestampDiff", "DatetimeDiff", "TimeDiff") + ) + if isinstance(node, type) +) +# Ossie names only a few dialects; everything else has no equivalent. +_GSF_TO_OSSIE_DIALECT = { + "snowflake": "SNOWFLAKE", + "databricks": "DATABRICKS", + "bigquery": "BIGQUERY", +} class GSFConversionError(Exception): @@ -448,6 +483,7 @@ def convert_gsf_to_ossie( """Convert a native ``GsfModelDocument`` to one Apache Ossie model.""" root = _parse_gsf(gsf_yaml) catalog = _read_catalog(root) + dialects = _dialects_by_database(root) semantic = root["semantic_layer"] terms = semantic["terms"] if not terms: @@ -557,7 +593,7 @@ def convert_gsf_to_ossie( ) field_names_by_term[term_id].add(name) represented_table_id = str(term_by_id[term_id]["represents"][0]) - _validate_gsf_sql_databases( + sql_databases = _validate_gsf_sql_databases( f"SQL attribute {name!r}", sql, attribute.get("sql_column_is") or [], @@ -567,7 +603,10 @@ def convert_gsf_to_ossie( ossie_expression = _expression_from_sql(sql) field = { "name": name, - "expression": _ossie_expression(ossie_expression), + "expression": _ossie_expression( + ossie_expression, + _ossie_dialect(dialects, sql_databases), + ), "custom_extensions": [ _gsf_extension( { @@ -600,7 +639,7 @@ def convert_gsf_to_ossie( raise GSFConversionError( "Every GSF custom analysis requires non-empty name and sql" ) - _validate_gsf_sql_databases( + sql_databases = _validate_gsf_sql_databases( f"Custom analysis {name!r}", sql, analysis.get("sql_column_is") or [], @@ -609,7 +648,10 @@ def convert_gsf_to_ossie( ossie_expression = _expression_from_sql(sql) metric: dict[str, Any] = { "name": name, - "expression": _ossie_expression(ossie_expression), + "expression": _ossie_expression( + ossie_expression, + _ossie_dialect(dialects, sql_databases), + ), "custom_extensions": [ _gsf_extension( { @@ -854,10 +896,7 @@ def _index_native_document(root: dict[str, Any] | None) -> dict[str, Any]: } if not root: return result - try: - catalog = _read_catalog(root) - except GSFConversionError: - return result + catalog = _read_native_catalog(root) for database in root["data_layer"].get("databases") or []: database_names = { str(schema.get("database_name") or "") @@ -918,6 +957,21 @@ def _index_native_document(root: dict[str, Any] | None) -> dict[str, Any]: return result +def _read_native_catalog(root: dict[str, Any]) -> dict[str, Any]: + """Read the catalog out of a preserved native snapshot. + + Both the indexing and the relationship-reconciliation paths go through + here so a hand-edited extension fails the same way in either, rather than + silently dropping the preserved identifiers in one of them. + """ + try: + return _read_catalog(root) + except GSFConversionError as exc: + raise GSFConversionError( + f"Malformed {NVIDIA_GSF_VENDOR} 'native_document' extension: {exc}" + ) from exc + + def _read_catalog(root: dict[str, Any]) -> dict[str, Any]: tables: dict[str, dict[str, Any]] = {} columns: dict[str, dict[str, Any]] = {} @@ -972,7 +1026,8 @@ def _validate_gsf_sql_databases( catalog: Mapping[str, Any], *, attached_table_id: str | None = None, -) -> None: +) -> set[str]: + """Validate the SQL resolves to one database, and return the databases.""" databases: set[str] = set() if attached_table_id: table = catalog["tables"].get(attached_table_id) @@ -984,7 +1039,7 @@ def _validate_gsf_sql_databases( databases.add(str(catalog["tables"][column["table_id"]]["source"][0])) parsed = _parse_sql(sql) - for sql_table in parsed.find_all(exp.Table): + for sql_table in parsed.find_all(exp.Table) if parsed is not None else []: if sql_table.catalog: databases.add(sql_table.catalog) matches = [ @@ -1001,6 +1056,7 @@ def _validate_gsf_sql_databases( f"{context} spans multiple databases ({', '.join(sorted(databases))}); " "the GSF importer validates each SQL object against one database" ) + return databases def _relationships_from_gsf( @@ -1233,11 +1289,7 @@ def _parse_ossie(value: str) -> tuple[dict[str, Any], dict[str, Any]]: raise GSFConversionError( "Unsupported Ossie root properties: " + ", ".join(unknown) ) - if str(root.get("version", "")) != OSSIE_VERSION: - raise GSFConversionError( - f"Unsupported Ossie version {root.get('version')!r}; " - f"supported version is {OSSIE_VERSION!r}" - ) + _check_ossie_version(root.get("version")) models = root.get("semantic_model") if not isinstance(models, list) or len(models) != 1: raise GSFConversionError("Ossie input must contain exactly one semantic model") @@ -1247,6 +1299,16 @@ def _parse_ossie(value: str) -> tuple[dict[str, Any], dict[str, Any]]: return root, model +def _check_ossie_version(value: Any) -> None: + """Accept any Ossie version in the supported major.minor series.""" + series = re.match(r"^\s*(\d+)\.(\d+)", str(value or "")) + if not series or (int(series.group(1)), int(series.group(2))) != OSSIE_SERIES: + expected = ".".join(str(part) for part in OSSIE_SERIES) + raise GSFConversionError( + f"Unsupported Ossie version {value!r}; expected {expected}.x" + ) + + def _load_yaml(value: str, label: str) -> dict[str, Any]: try: root = yaml.safe_load(value) @@ -1442,9 +1504,7 @@ def _validate_single_database_refs( databases = { str(datasets[ref]["source"]["database"]) for ref in refs if ref in datasets } - databases.update( - table.catalog for table in _parse_sql(sql).find_all(exp.Table) if table.catalog - ) + databases.update(table.catalog for table in _sql_tables(sql) if table.catalog) if len(databases) > 1: raise GSFConversionError( f"{context} spans multiple databases ({', '.join(sorted(databases))}); " @@ -1458,6 +1518,8 @@ def _referenced_datasets( ) -> list[str]: references: list[str] = [] parsed = _parse_sql(sql) + if parsed is None: + return references for table in parsed.find_all(exp.Table): matches = [ name @@ -1545,41 +1607,62 @@ def _column_dataset( def _sql_columns(sql: str) -> list[exp.Column]: + parsed = _parse_sql(sql) + if parsed is None: + return [] return [ column - for column in _parse_sql(sql).find_all(exp.Column) + for column in parsed.find_all(exp.Column) if not _is_date_part_unit(column) ] +def _sql_tables(sql: str) -> list[exp.Table]: + parsed = _parse_sql(sql) + return list(parsed.find_all(exp.Table)) if parsed is not None else [] + + def _is_date_part_unit(column: exp.Column) -> bool: """Report whether a parsed column is really a date-part keyword. - ``DATEDIFF(day, a, b)`` and friends put the unit in an argument slot that - sqlglot parses as an unqualified column, which would otherwise be mistaken - for a physical catalog column. + ``DATEDIFF(day, a, b)`` puts the unit where sqlglot parses an unqualified + column, which would otherwise be mistaken for a physical catalog column. + Only the unit slot itself counts, so a column genuinely named ``day`` + elsewhere in the same call still resolves as data. """ parent = column.parent if column.table or column.name.lower() not in _DATE_PART_UNITS: return False - if not isinstance(parent, exp.Func): - return False if isinstance(parent, exp.Anonymous): - return str(parent.this).lower() in _DATE_FUNCTIONS - try: - names = parent.sql_names() - except (AttributeError, IndexError): - return False - return any(name.lower() in _DATE_FUNCTIONS for name in names) + if str(parent.this).lower() not in _UNIT_FIRST_FUNCTIONS: + return False + arguments = parent.args.get("expressions") or [] + return bool(arguments) and arguments[0] is column + if isinstance(parent, _UNIT_IN_THIS_FUNCTIONS): + # MySQL's two-argument DATEDIFF(ended, started) has no unit, so its + # first argument is data rather than a keyword. + return column.arg_key == "this" and _argument_count(parent) >= 3 + return False -def _parse_sql(sql: str) -> exp.Expression: - try: - return parse_one(sql) - except (ParseError, ValueError) as exc: - raise GSFConversionError( - f"Unable to parse SQL expression {sql!r}: {exc}" - ) from exc +def _argument_count(func: exp.Expression) -> int: + return sum(1 for value in func.args.values() if value is not None) + + +def _parse_sql(sql: str) -> exp.Expression | None: + """Parse *sql*, trying each candidate dialect, or return ``None``. + + Preserved GSF SQL carries whatever dialect its connection reported, so a + single parser is not enough. SQL that no candidate can parse is treated as + opaque: it is still carried through verbatim, and only the parse-derived + enrichment (column and table discovery) is skipped. + """ + for dialect in _SQL_DIALECTS: + try: + return parse_one(sql, dialect=dialect or None) + except (ParseError, TokenError, ValueError): + continue + return None def _first_resolvable_column_ids( @@ -1654,9 +1737,8 @@ def _wrap_expression( def _expression_from_sql(sql: str) -> str: - try: - parsed = parse_one(sql) - except (ParseError, ValueError): + parsed = _parse_sql(sql) + if parsed is None: return sql select = parsed.find(exp.Select) if select is None or not select.expressions: @@ -1670,23 +1752,50 @@ def _expression_from_sql(sql: str) -> str: def _expression_matches(current: str, emitted: Any) -> bool: if not isinstance(emitted, str) or not emitted.strip(): return False - try: - return parse_one(current).sql() == parse_one(emitted).sql() - except (ParseError, ValueError): + parsed_current = _parse_sql(current) + parsed_emitted = _parse_sql(emitted) + if parsed_current is None or parsed_emitted is None: return current.strip() == emitted.strip() + return parsed_current.sql() == parsed_emitted.sql() -def _ossie_expression(expression: str) -> dict[str, Any]: +def _ossie_expression(expression: str, dialect: str = "ANSI_SQL") -> dict[str, Any]: return { "dialects": [ { - "dialect": "ANSI_SQL", + "dialect": dialect, "expression": expression, } ] } +def _dialects_by_database(root: Mapping[str, Any]) -> dict[str, str]: + result: dict[str, str] = {} + for database in root["data_layer"].get("databases") or []: + dialect = str(database.get("dialect") or "") + if not dialect: + continue + for schema in database.get("schemas") or []: + name = str(schema.get("database_name") or "") + if name: + result[name] = dialect + return result + + +def _ossie_dialect(dialects: Mapping[str, str], databases: Iterable[str]) -> str: + """Map a GSF connection dialect onto the Ossie dialect enum. + + Ossie names only a few dialects, so anything else stays ANSI_SQL rather + than being labelled inaccurately. + """ + names = {str(dialects.get(database, "")).lower() for database in databases} + labels = { + _GSF_TO_OSSIE_DIALECT[name] for name in names if name in _GSF_TO_OSSIE_DIALECT + } + return labels.pop() if len(labels) == 1 else "ANSI_SQL" + + def _parse_source( source: Any, default_database: str | None, @@ -1781,7 +1890,7 @@ def _reconcile_native_relationships( joins: list[dict[str, Any]], semantic_fks: list[dict[str, str]], ) -> tuple[list[dict[str, str]], list[dict[str, Any]], list[dict[str, str]]]: - catalog = _read_catalog(native) + catalog = _read_native_catalog(native) native_joins = [ item for item in native.get("data_layer", {}).get("joins") or [] diff --git a/converters/gsf/tests/test_converter.py b/converters/gsf/tests/test_converter.py index bc998459..18c825bf 100644 --- a/converters/gsf/tests/test_converter.py +++ b/converters/gsf/tests/test_converter.py @@ -22,6 +22,7 @@ import json import subprocess import sys +from copy import deepcopy from pathlib import Path from typing import Any @@ -34,7 +35,12 @@ convert_ossie_to_gsf, main, ) -from ossie_gsf.native_converter import _parse_source, _simple_source_column +from ossie_gsf.native_converter import ( + _index_native_document, + _parse_source, + _reconcile_native_relationships, + _simple_source_column, +) OSSIE_VERSION = "0.2.0.dev0" FIXTURES = Path(__file__).parent / "fixtures" @@ -58,6 +64,15 @@ def _native_extension(item: dict[str, Any]) -> dict[str, Any]: return json.loads(extension["data"]) +def _manual_sql(native: dict[str, Any], name: str) -> str: + attribute = next( + item + for item in native["semantic_layer"]["sql_attributes"]["manual"] + if item["name"] == name + ) + return str(attribute["sql"]) + + def _ids(value: Any) -> set[str]: result: set[str] = set() if isinstance(value, dict): @@ -606,6 +621,144 @@ def test_gsf_sourced_catalog_is_never_widened_by_sql_identifiers() -> None: assert "not_a_real_column" not in {column["name"] for column in after_columns} +@pytest.mark.parametrize("version", ["0.2.0.dev0", "0.2.0", "0.2.1", "0.2.7.dev3"]) +def test_any_release_in_the_supported_series_is_accepted(version: str) -> None: + ossie = yaml.safe_load(_ossie_yaml()) + ossie["version"] = version + + native = yaml.safe_load(convert_ossie_to_gsf(yaml.safe_dump(ossie))) + + assert native["semantic_layer"]["terms"] + + +@pytest.mark.parametrize("version", ["0.1.9", "0.3.0", "1.0.0", "", "dev"]) +def test_versions_outside_the_supported_series_are_rejected(version: str) -> None: + ossie = yaml.safe_load(_ossie_yaml()) + ossie["version"] = version + + with pytest.raises(GSFConversionError, match="Unsupported Ossie version"): + convert_ossie_to_gsf(yaml.safe_dump(ossie)) + + +def test_dialect_specific_native_sql_survives_a_round_trip() -> None: + """``TOP n`` is valid Snowflake but the default parser rejects it.""" + native = yaml.safe_load(_gsf_yaml()) + manual = native["semantic_layer"]["sql_attributes"]["manual"][0] + manual["sql"] = ( + 'SELECT TOP 1 "orders"."subtotal" AS "net_total" ' + 'FROM "analytics"."public"."orders" AS "orders"' + ) + + ossie = convert_gsf_to_ossie(yaml.safe_dump(native)) + restored = yaml.safe_load(convert_ossie_to_gsf(ossie)) + + assert _manual_sql(restored, "net_total") == manual["sql"] + + +def test_native_sql_no_dialect_can_parse_is_carried_through_verbatim() -> None: + native = yaml.safe_load(_gsf_yaml()) + manual = native["semantic_layer"]["sql_attributes"]["manual"][0] + manual["sql"] = "SELECT not ((parseable by any dialect" + + ossie = convert_gsf_to_ossie(yaml.safe_dump(native)) + restored = yaml.safe_load(convert_ossie_to_gsf(ossie)) + + assert _manual_sql(restored, "net_total") == manual["sql"] + + +@pytest.mark.parametrize( + "expression", + [ + "DATEDIFF(month, day, CURRENT_TIMESTAMP())", + "DATEDIFF(day, order_date)", + "LAST_DAY(day)", + "TRUNC(day)", + "SUM(day)", + ], +) +def test_columns_named_like_units_survive_outside_the_unit_slot( + expression: str, +) -> None: + """Only the unit argument itself is treated as a keyword.""" + ossie = yaml.safe_load(_ossie_yaml()) + ossie["semantic_model"][0]["datasets"][0]["fields"].append( + { + "name": "order_age", + "expression": { + "dialects": [{"dialect": "ANSI_SQL", "expression": expression}] + }, + } + ) + + native = yaml.safe_load(convert_ossie_to_gsf(yaml.safe_dump(ossie))) + orders = next( + table + for database in native["data_layer"]["databases"] + for schema in database["schemas"] + for table in schema["tables"] + if table["name"] == "orders" + ) + + assert "day" in {column["name"] for column in orders["columns"]} + + +def test_malformed_native_snapshot_fails_alike_in_both_paths() -> None: + """Indexing and relationship reconciliation read the same snapshot.""" + native = yaml.safe_load(_gsf_yaml()) + native["data_layer"]["databases"].append( + deepcopy(native["data_layer"]["databases"][0]) + ) + + with pytest.raises(GSFConversionError, match="Malformed NVIDIA_GSF"): + _index_native_document(native) + + with pytest.raises(GSFConversionError, match="Malformed NVIDIA_GSF"): + _reconcile_native_relationships( + native, + represented_table_ids=set(), + foreign_keys=[], + joins=[], + semantic_fks=[], + ) + + +def test_expression_dialect_follows_the_gsf_connection() -> None: + native = yaml.safe_load(_gsf_yaml()) + native["data_layer"]["databases"][0]["dialect"] = "snowflake" + + ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) + orders = next( + dataset + for dataset in ossie["semantic_model"][0]["datasets"] + if dataset["name"] == "orders" + ) + dialects = { + field["name"]: field["expression"]["dialects"][0]["dialect"] + for field in orders["fields"] + } + + assert dialects["net_total"] == "SNOWFLAKE" + # A bare column reference is dialect-neutral. + assert dialects["order_id"] == "ANSI_SQL" + + +def test_dialects_ossie_cannot_name_stay_ansi() -> None: + native = yaml.safe_load(_gsf_yaml()) + native["data_layer"]["databases"][0]["dialect"] = "mysql" + + ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) + orders = next( + dataset + for dataset in ossie["semantic_model"][0]["datasets"] + if dataset["name"] == "orders" + ) + net_total = next( + field for field in orders["fields"] if field["name"] == "net_total" + ) + + assert net_total["expression"]["dialects"][0]["dialect"] == "ANSI_SQL" + + def test_old_fictional_gsf_root_is_rejected() -> None: old_shape = { "version": "1.0", From 5a26fd77dec6de8b1626c21acc58c12bbdc4edc1 Mon Sep 17 00:00:00 2001 From: Lio Fleishman Date: Tue, 28 Jul 2026 10:20:32 -0500 Subject: [PATCH 5/6] Map Ossie datatype to and from GSF column types The spec gained an optional logical datatype on fields and metrics. Reduce a GSF column's physical type to that vocabulary on import, so NUMBER(38,0) becomes Integer and a type Ossie cannot name, such as VARIANT, becomes Opaque. On export, write a canonical physical type for a column the Ossie model introduces, and never override one preserved from a real GSF catalog, since GSF reports what the connection actually holds. The two mappings are inverses, so a declared datatype survives a full cycle. A computed field and a metric have no single column behind them and GSF stores no type for either, so theirs is still not carried. --- converters/gsf/README.md | 16 +++ .../gsf/src/ossie_gsf/native_converter.py | 124 ++++++++++++++++- converters/gsf/tests/test_converter.py | 130 ++++++++++++++++++ 3 files changed, 269 insertions(+), 1 deletion(-) diff --git a/converters/gsf/README.md b/converters/gsf/README.md index a1fe19ef..35219cc1 100644 --- a/converters/gsf/README.md +++ b/converters/gsf/README.md @@ -31,6 +31,7 @@ Neo4j, a database, or network access. | Dataset field backed by one column | `semantic_layer.terms[].columns_attributes[]` | | Computed dataset field | `semantic_layer.sql_attributes.manual[]` | | Model-level metric | `semantic_layer.custom_analyses[]` | +| Field `datatype` | physical `type` on the catalog column behind the field | | Relationship | data-layer `joins` and `foreign_keys`, plus `semantic_fks` when possible | | Dataset | term that `represents` exactly one catalog table | @@ -161,6 +162,21 @@ adds fictional fields to the GSF schema. GSF records uniqueness per column, so Ossie composite unique keys cannot be reconstructed after GSF → Ossie; only single-column unique keys survive. +Ossie's `datatype` maps to and from the physical type on a GSF catalog column, +for fields backed by a single column. GSF → Ossie reduces the physical type to +Ossie's logical vocabulary, so `NUMBER(38,0)` becomes `Integer`, `NUMBER(12,2)` +becomes `Decimal`, and a type Ossie cannot name, such as Snowflake's `VARIANT`, +becomes `Opaque` as the spec prescribes. Ossie → GSF writes a canonical physical +type for a column the Ossie model introduces, and never overrides a type +preserved from a real GSF catalog, since GSF reports what the connection +actually holds. The two mappings are inverses, so a declared `datatype` survives +a full cycle. + +A computed field or a metric has no single column behind it and GSF stores no +type for either, so their `datatype` is not carried. `Opaque` is not written +back, because it names a type outside the vocabulary and there is no physical +type worth inventing from it. + ## Tests ```bash diff --git a/converters/gsf/src/ossie_gsf/native_converter.py b/converters/gsf/src/ossie_gsf/native_converter.py index 6f7a5da8..f2c4fd7e 100644 --- a/converters/gsf/src/ossie_gsf/native_converter.py +++ b/converters/gsf/src/ossie_gsf/native_converter.py @@ -158,6 +158,75 @@ "databricks": "DATABRICKS", "bigquery": "BIGQUERY", } +# A GSF column carries the physical type its connection reports. Ossie names ten +# logical types, so the mapping is deliberately coarse in that direction and +# canonical in the other, which is what lets a datatype survive a full cycle. +_OSSIE_DATATYPE_BY_SQL_TYPE = { + "VARCHAR": "String", + "VARCHAR2": "String", + "NVARCHAR": "String", + "NVARCHAR2": "String", + "CHAR": "String", + "NCHAR": "String", + "CHARACTER": "String", + "CHARACTER VARYING": "String", + "TEXT": "String", + "STRING": "String", + "CLOB": "String", + "NCLOB": "String", + "INT": "Integer", + "INTEGER": "Integer", + "BIGINT": "Integer", + "SMALLINT": "Integer", + "TINYINT": "Integer", + "BYTEINT": "Integer", + "INT2": "Integer", + "INT4": "Integer", + "INT8": "Integer", + "DEC": "Decimal", + "DECIMAL": "Decimal", + "NUMERIC": "Decimal", + "NUMBER": "Decimal", + "MONEY": "Decimal", + "FLOAT": "Float", + "FLOAT4": "Float", + "FLOAT8": "Float", + "REAL": "Float", + "DOUBLE": "Float", + "DOUBLE PRECISION": "Float", + "BINARY_FLOAT": "Float", + "BINARY_DOUBLE": "Float", + "BOOL": "Boolean", + "BOOLEAN": "Boolean", + "DATE": "Date", + "TIME": "Time", + "TIME WITHOUT TIME ZONE": "Time", + "DATETIME": "DateTime", + "DATETIME2": "DateTime", + "SMALLDATETIME": "DateTime", + "TIMESTAMP": "DateTime", + "TIMESTAMP_NTZ": "DateTime", + "TIMESTAMP WITHOUT TIME ZONE": "DateTime", + "DATETIMEOFFSET": "DateTimeTz", + "TIMESTAMPTZ": "DateTimeTz", + "TIMESTAMP_LTZ": "DateTimeTz", + "TIMESTAMP_TZ": "DateTimeTz", + "TIMESTAMP WITH LOCAL TIME ZONE": "DateTimeTz", + "TIMESTAMP WITH TIME ZONE": "DateTimeTz", +} +# Opaque is absent on purpose: it names a type outside Ossie's vocabulary, so +# there is nothing to write back and no physical type worth inventing. +_SQL_TYPE_BY_OSSIE_DATATYPE = { + "String": "TEXT", + "Integer": "BIGINT", + "Decimal": "DECIMAL", + "Float": "DOUBLE", + "Boolean": "BOOLEAN", + "Date": "DATE", + "Time": "TIME", + "DateTime": "TIMESTAMP", + "DateTimeTz": "TIMESTAMP WITH TIME ZONE", +} class GSFConversionError(Exception): @@ -564,6 +633,9 @@ def convert_gsf_to_ossie( "name": field_name, "expression": _ossie_expression(str(column["name"])), } + datatype = _ossie_datatype(column["item"].get("type")) + if datatype: + field["datatype"] = datatype if attribute.get("description"): field["description"] = str(attribute["description"]) fields_by_term[term_id].append(field) @@ -763,6 +835,15 @@ def _build_catalog( column for _, context in contexts for column in context["columns"] ) ) + # A field states the logical type of the column behind it, which is the + # only type information an Ossie-origin catalog has to offer. + declared_types: dict[str, str] = {} + for _, context in contexts: + for field, column_name in context["simple_fields"]: + sql_type = _gsf_column_type(field.get("datatype")) + if sql_type: + declared_types.setdefault(column_name, sql_type) + columns: list[dict[str, Any]] = [] for column_name in catalog_columns: preserved_column = preserved["columns"].get((*source_key, column_name), {}) @@ -780,7 +861,10 @@ def _build_catalog( "id": column_id, "name": column_name, "description": str(preserved_column.get("description") or ""), - "type": str(preserved_column.get("type") or ""), + "type": str( + preserved_column.get("type") + or declared_types.get(column_name, "") + ), "sample_values": list(preserved_column.get("sample_values") or []), "is_nullable": bool( preserved_column.get("is_nullable", column_name not in pk) @@ -1770,6 +1854,44 @@ def _ossie_expression(expression: str, dialect: str = "ANSI_SQL") -> dict[str, A } +def _ossie_datatype(sql_type: Any) -> str | None: + """Map a GSF column's physical type onto Ossie's logical vocabulary. + + A type Ossie cannot name becomes ``Opaque``, as the spec prescribes for a + known type outside the portable vocabulary. An absent type stays unset + rather than being guessed. + """ + base, scale = _split_sql_type(sql_type) + if not base: + return None + datatype = _OSSIE_DATATYPE_BY_SQL_TYPE.get(base) + if datatype is None: + return "Opaque" + if datatype == "Decimal" and scale == 0: + # NUMBER(38,0) and friends are exact integers. + return "Integer" + return datatype + + +def _split_sql_type(sql_type: Any) -> tuple[str, int | None]: + """Split a physical type into its base name and declared scale.""" + text = " ".join(str(sql_type or "").upper().split()) + if not text: + return "", None + scale: int | None = None + parameters = re.search(r"\(([^)]*)\)", text) + if parameters: + parts = [part.strip() for part in parameters.group(1).split(",")] + if len(parts) > 1 and parts[1].isdigit(): + scale = int(parts[1]) + return " ".join(re.sub(r"\([^)]*\)", " ", text).split()), scale + + +def _gsf_column_type(datatype: Any) -> str: + """Map an Ossie logical datatype onto a physical type for a new column.""" + return _SQL_TYPE_BY_OSSIE_DATATYPE.get(str(datatype or ""), "") + + def _dialects_by_database(root: Mapping[str, Any]) -> dict[str, str]: result: dict[str, str] = {} for database in root["data_layer"].get("databases") or []: diff --git a/converters/gsf/tests/test_converter.py b/converters/gsf/tests/test_converter.py index 18c825bf..29ac624d 100644 --- a/converters/gsf/tests/test_converter.py +++ b/converters/gsf/tests/test_converter.py @@ -36,7 +36,9 @@ main, ) from ossie_gsf.native_converter import ( + _SQL_TYPE_BY_OSSIE_DATATYPE, _index_native_document, + _ossie_datatype, _parse_source, _reconcile_native_relationships, _simple_source_column, @@ -45,6 +47,7 @@ OSSIE_VERSION = "0.2.0.dev0" FIXTURES = Path(__file__).parent / "fixtures" VALIDATOR = Path(__file__).resolve().parents[3] / "validation" / "validate.py" +SCHEMA = Path(__file__).resolve().parents[3] / "core-spec" / "osi-schema.json" def _ossie_yaml() -> str: @@ -759,6 +762,133 @@ def test_dialects_ossie_cannot_name_stay_ansi() -> None: assert net_total["expression"]["dialects"][0]["dialect"] == "ANSI_SQL" +@pytest.mark.parametrize("datatype", sorted(_SQL_TYPE_BY_OSSIE_DATATYPE)) +def test_every_mappable_datatype_survives_a_round_trip(datatype: str) -> None: + ossie = yaml.safe_load(_ossie_yaml()) + orders = next( + dataset + for dataset in ossie["semantic_model"][0]["datasets"] + if dataset["name"] == "orders" + ) + next(field for field in orders["fields"] if field["name"] == "order_id")[ + "datatype" + ] = datatype + + native = convert_ossie_to_gsf(yaml.safe_dump(ossie)) + restored = yaml.safe_load(convert_gsf_to_ossie(native)) + field = next( + item + for dataset in restored["semantic_model"][0]["datasets"] + if dataset["name"] == "orders" + for item in dataset["fields"] + if item["name"] == "order_id" + ) + + assert field["datatype"] == datatype + + +def test_the_physical_type_chosen_for_each_datatype_maps_back_to_it() -> None: + """The two directions have to be inverses or a cycle would drift.""" + for datatype, sql_type in _SQL_TYPE_BY_OSSIE_DATATYPE.items(): + assert _ossie_datatype(sql_type) == datatype + + +def test_mapping_covers_the_specs_datatype_vocabulary() -> None: + """Fail loudly if the spec grows a logical type the mapping ignores.""" + schema = json.loads(SCHEMA.read_text(encoding="utf-8")) + + assert set(schema["$defs"]["DataType"]["enum"]) == { + *_SQL_TYPE_BY_OSSIE_DATATYPE, + "Opaque", + } + + +@pytest.mark.parametrize( + ("sql_type", "expected"), + [ + ("TEXT", "String"), + ("VARCHAR(255)", "String"), + ("NUMBER(38,0)", "Integer"), + ("NUMBER(12,2)", "Decimal"), + ("DECIMAL", "Decimal"), + ("double precision", "Float"), + ("TIMESTAMP_NTZ(9)", "DateTime"), + ("TIMESTAMP(6) WITH TIME ZONE", "DateTimeTz"), + ("VARIANT", "Opaque"), + ("GEOGRAPHY", "Opaque"), + ("", None), + (None, None), + ], +) +def test_physical_types_map_onto_the_ossie_vocabulary( + sql_type: str | None, + expected: str | None, +) -> None: + assert _ossie_datatype(sql_type) == expected + + +def test_gsf_column_types_reach_the_ossie_field() -> None: + native = yaml.safe_load(_gsf_yaml()) + orders = next( + table + for database in native["data_layer"]["databases"] + for schema in database["schemas"] + for table in schema["tables"] + if table["name"] == "orders" + ) + for column in orders["columns"]: + column["type"] = "NUMBER(38,0)" if column["name"] == "order_id" else "TEXT" + + ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) + fields = { + field["name"]: field.get("datatype") + for dataset in ossie["semantic_model"][0]["datasets"] + if dataset["name"] == "orders" + for field in dataset["fields"] + } + + assert fields["order_id"] == "Integer" + assert fields["customer_id"] == "String" + # A computed attribute has no column, so GSF holds no type for it. + assert fields["net_total"] is None + + +def test_a_live_gsf_column_type_outranks_an_ossie_datatype() -> None: + """GSF reports the physical type; Ossie only names a logical one.""" + native = yaml.safe_load(_gsf_yaml()) + orders = next( + table + for database in native["data_layer"]["databases"] + for schema in database["schemas"] + for table in schema["tables"] + if table["name"] == "orders" + ) + next(column for column in orders["columns"] if column["name"] == "order_id")[ + "type" + ] = "NUMBER(38,0)" + + ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) + next( + field + for dataset in ossie["semantic_model"][0]["datasets"] + if dataset["name"] == "orders" + for field in dataset["fields"] + if field["name"] == "order_id" + )["datatype"] = "String" + + restored = yaml.safe_load(convert_ossie_to_gsf(yaml.safe_dump(ossie))) + column = next( + column + for database in restored["data_layer"]["databases"] + for schema in database["schemas"] + for table in schema["tables"] + for column in table["columns"] + if column["name"] == "order_id" + ) + + assert column["type"] == "NUMBER(38,0)" + + def test_old_fictional_gsf_root_is_rejected() -> None: old_shape = { "version": "1.0", From 61f943d14ff60b023551b1ee829c0643d12ffcc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?JB=20Onofr=C3=A9?= Date: Thu, 30 Jul 2026 07:52:38 +0200 Subject: [PATCH 6/6] Add CI workflow for the GSF converter Runs the GSF converter test suite (uv sync + pytest) on pushes and pull requests that touch converters/gsf, matching the pattern used by the other converter CI workflows. The matrix targets Python 3.11-3.14 to match the package's requires-python floor. --- .github/workflows/converter-gsf-ci.yml | 63 ++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 .github/workflows/converter-gsf-ci.yml diff --git a/.github/workflows/converter-gsf-ci.yml b/.github/workflows/converter-gsf-ci.yml new file mode 100644 index 00000000..e2f3412c --- /dev/null +++ b/.github/workflows/converter-gsf-ci.yml @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters GSF CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/gsf/**' + - '.github/workflows/converter-gsf-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/gsf/**' + - '.github/workflows/converter-gsf-ci.yml' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + working-directory: converters/gsf + run: | + uv sync + + - name: Unit Tests + working-directory: converters/gsf + run: | + uv run pytest