From 59c6e629a59d9da1838597d92cb91debdb3995ad Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 3 Jun 2026 18:20:51 +0200 Subject: [PATCH 01/29] Adding ontology Python project to the converters folder --- converters/ontology/.gitignore | 42 + converters/ontology/README.md | 93 ++ converters/ontology/pyproject.toml | 29 + converters/ontology/requirements.lock | 21 + .../ontology/scripts/palantir_to_osi.py | 55 + converters/ontology/src/osi/__init__.py | 72 ++ .../ontology/src/osi/common/__init__.py | 0 .../ontology/src/osi/common/file_utils.py | 42 + converters/ontology/src/osi/common/graph.py | 180 +++ converters/ontology/src/osi/common/utils.py | 43 + .../ontology/src/osi/converter/__init__.py | 0 .../src/osi/converter/osi_to_spec/__init__.py | 0 .../osi/converter/osi_to_spec/converter.py | 271 ++++ .../osi/converter/palantir_to_osi/__init__.py | 0 .../converter/palantir_to_osi/converter.py | 778 ++++++++++++ .../src/osi/converter/spec_to_osi/__init__.py | 0 .../osi/converter/spec_to_osi/converter.py | 499 ++++++++ .../ontology/src/osi/external/__init__.py | 0 .../src/osi/external/palantir/__init__.py | 0 .../src/osi/external/palantir/model.py | 632 +++++++++ .../osi/external/palantir/parser/__init__.py | 680 ++++++++++ converters/ontology/src/osi/model.py | 1128 +++++++++++++++++ .../ontology/src/osi/parser/__init__.py | 49 + converters/ontology/src/osi/spec.py | 248 ++++ 24 files changed, 4862 insertions(+) create mode 100644 converters/ontology/.gitignore create mode 100644 converters/ontology/README.md create mode 100644 converters/ontology/pyproject.toml create mode 100644 converters/ontology/requirements.lock create mode 100644 converters/ontology/scripts/palantir_to_osi.py create mode 100644 converters/ontology/src/osi/__init__.py create mode 100644 converters/ontology/src/osi/common/__init__.py create mode 100644 converters/ontology/src/osi/common/file_utils.py create mode 100644 converters/ontology/src/osi/common/graph.py create mode 100644 converters/ontology/src/osi/common/utils.py create mode 100644 converters/ontology/src/osi/converter/__init__.py create mode 100644 converters/ontology/src/osi/converter/osi_to_spec/__init__.py create mode 100644 converters/ontology/src/osi/converter/osi_to_spec/converter.py create mode 100644 converters/ontology/src/osi/converter/palantir_to_osi/__init__.py create mode 100644 converters/ontology/src/osi/converter/palantir_to_osi/converter.py create mode 100644 converters/ontology/src/osi/converter/spec_to_osi/__init__.py create mode 100644 converters/ontology/src/osi/converter/spec_to_osi/converter.py create mode 100644 converters/ontology/src/osi/external/__init__.py create mode 100644 converters/ontology/src/osi/external/palantir/__init__.py create mode 100644 converters/ontology/src/osi/external/palantir/model.py create mode 100644 converters/ontology/src/osi/external/palantir/parser/__init__.py create mode 100644 converters/ontology/src/osi/model.py create mode 100644 converters/ontology/src/osi/parser/__init__.py create mode 100644 converters/ontology/src/osi/spec.py diff --git a/converters/ontology/.gitignore b/converters/ontology/.gitignore new file mode 100644 index 00000000..211fd5c5 --- /dev/null +++ b/converters/ontology/.gitignore @@ -0,0 +1,42 @@ +# Python +__pycache__/ +*.py[cod] +*.pyo +*.pyd + +# Virtual environments +venv/ +.venv/ +env/ + +# pyenv +.python-version + +# Build / packaging +dist/ +build/ +*.egg-info/ +*.egg +.eggs/ + +# Pytest +.pytest_cache/ +.coverage +htmlcov/ + +# Mypy +.mypy_cache/ + +# Ruff +.ruff_cache/ + +# VS Code +.vscode/ +*.code-workspace +.history/ + +# JetBrains (PyCharm, IntelliJ, etc.) +.idea/ +*.iml +*.iws +*.ipr \ No newline at end of file diff --git a/converters/ontology/README.md b/converters/ontology/README.md new file mode 100644 index 00000000..0e09873c --- /dev/null +++ b/converters/ontology/README.md @@ -0,0 +1,93 @@ +# OSI Ontology Converters + +Converters between OSI, Palantir, and Spec ontology formats. + +| Converter | Direction | +|-----------|-----------| +| `palantir_to_osi` | Palantir ontology → OSI model | +| `osi_to_spec` | OSI model → Spec YAML | +| `spec_to_osi` | Spec YAML → OSI model | + +## Prerequisites + +- [pyenv](https://github.com/pyenv/pyenv) — manages the Python version + +Install pyenv if you don't have it: + +```bash +brew install pyenv +``` + +Add to your shell profile (`~/.zshrc` or `~/.bashrc`) and restart the shell: + +```bash +export PYENV_ROOT="$HOME/.pyenv" +export PATH="$PYENV_ROOT/bin:$PATH" +eval "$(pyenv init -)" +``` + +## Setup + +```bash +pyenv install 3.11 +pyenv local 3.11 +pip install --upgrade pip +pip install virtualenv +python -m virtualenv venv +source ./venv/bin/activate +pip install -r requirements.lock +pip install -e ".[dev]" +``` + +## Generating / updating the lock file + +`requirements.lock` is produced by [pip-tools](https://github.com/jazzband/pip-tools) from `pyproject.toml`. +Run this whenever you add or change a dependency: + +```bash +pip-compile --output-file requirements.lock pyproject.toml +``` + +## Usage + +The package is importable as `osi` after installation: + +```python +from osi.converter.palantir_to_osi.converter import PalantirToOsiConverter +from osi.converter.osi_to_spec.converter import OsiToSpecConverter +from osi.converter.spec_to_osi.converter import SpecToOsiConverter +``` + +## Scripts + +### `scripts/palantir_to_osi.py` + +Converts a Palantir ontology export (`.zip` file containing a Palantir ontology JSON and one or more dataset spec JSON files) into an OSI-compliant YAML representation, printed to stdout. + +**Usage:** + +```bash +python scripts/palantir_to_osi.py path/to/palantir_export.zip +``` + +Warnings are written to stderr; the OSI YAML is written to stdout. + +**Environment variables (optional):** + +| Variable | Default | Description | +|---------------------------|------------|----------------------------------------------------------| +| `SNOWFLAKE_DATABASE_NAME` | `PALANTIR` | Snowflake database name used to qualify table references | +| `SNOWFLAKE_SCHEMA_NAME` | `PALANTIR` | Snowflake schema name used to qualify table references | + +If already set in your environment they will be picked up automatically. To override them for a single run: + +```bash +SNOWFLAKE_DATABASE_NAME=MY_DB SNOWFLAKE_SCHEMA_NAME=MY_SCHEMA \ + python scripts/palantir_to_osi.py path/to/palantir_export.zip +``` + +## Deactivating the environment + +```bash +deactivate +``` \ No newline at end of file diff --git a/converters/ontology/pyproject.toml b/converters/ontology/pyproject.toml new file mode 100644 index 00000000..6a881478 --- /dev/null +++ b/converters/ontology/pyproject.toml @@ -0,0 +1,29 @@ +[project] +name = "ontology" +version = "0.1.0" +description = "OSI ontology converters — Palantir → OSI, OSI → Spec, Spec → OSI" +readme = "README.md" +authors = [ + { name = "RelationalAI", email = "support@relational.ai" }, +] +requires-python = ">= 3.11" +dependencies = [ + "pydantic", + "pyyaml", +] + +[project.optional-dependencies] +dev = [ + "pytest==9.0.3", + "pytest-snapshot", + "parameterized", + "pip-tools", +] + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] + +pythonVersion = "3.11" \ No newline at end of file diff --git a/converters/ontology/requirements.lock b/converters/ontology/requirements.lock new file mode 100644 index 00000000..39f9e31d --- /dev/null +++ b/converters/ontology/requirements.lock @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --output-file=requirements.lock pyproject.toml +# +annotated-types==0.7.0 + # via pydantic +pydantic==2.13.4 + # via osi-ontology-converters (pyproject.toml) +pydantic-core==2.46.4 + # via pydantic +pyyaml==6.0.3 + # via osi-ontology-converters (pyproject.toml) +typing-extensions==4.15.0 + # via + # pydantic + # pydantic-core + # typing-inspection +typing-inspection==0.4.2 + # via pydantic diff --git a/converters/ontology/scripts/palantir_to_osi.py b/converters/ontology/scripts/palantir_to_osi.py new file mode 100644 index 00000000..ecdab07a --- /dev/null +++ b/converters/ontology/scripts/palantir_to_osi.py @@ -0,0 +1,55 @@ +# Description: +# +# This script converts a zip file that contains: +# 1. A Palantir ontology (JSON file) and +# 2. A folder containing one or more Palantir dataset specs (JSON files) +# into an OSI compliant YAML representation of that ontology, using environment +# variables to configure the Snowflake database and schema names. +# +# Usage: +# +# $ python palantir_to_osi.py +# +# Environment variables used: +# +# - SNOWFLAKE_DATABASE_NAME +# - SNOWFLAKE_SCHEMA_NAME +# +# The tables that populate the ontology are named +# "{SNOWFLAKE_DATABASE_NAME}.{SNOWFLAKE_SCHEMA_NAME}.{TABLE_NAME}" +# where TABLE_NAME is the name of a data set that is referenced in +# the Palantir ontology. +# +# Outputs: +# +# - stderr: Warnings +# +import os +import sys +from pathlib import Path + +from osi.converter.palantir_to_osi.converter import PalantirToOsiConverter +from osi.converter.osi_to_spec.converter import OsiToSpecConverter + +from osi.external.palantir.parser import PalantirParser + +if __name__ == "__main__": + db_name = os.environ.get("SNOWFLAKE_DATABASE_NAME", "PALANTIR") + schema_name = os.environ.get("SNOWFLAKE_SCHEMA_NAME", "PALANTIR") + + if len(sys.argv) != 2: + raise Exception(f"++ Usage: {sys.argv[0]} path to Palantir sources") + + path = Path(sys.argv[1]) + + parser = PalantirParser() + + mode = "rb" if path.suffix.lower() == ".zip" else "r" + with open(path, mode) as file: + parser.parse(file) + + ontology_model = PalantirToOsiConverter.convert(parser.model(), db_name, schema_name) + + osi_spec = OsiToSpecConverter.convert(ontology_model) + print(osi_spec.dump_yaml()) + diff --git a/converters/ontology/src/osi/__init__.py b/converters/ontology/src/osi/__init__.py new file mode 100644 index 00000000..5aa05c23 --- /dev/null +++ b/converters/ontology/src/osi/__init__.py @@ -0,0 +1,72 @@ +""" +Public API surface for osi. + +Consumers should import from here rather than from deep sub-paths. +""" + +from osi.model import ( + Concept, + ConceptMapping, + ConceptType, + CustomExtension, + Dataset, + DatasetField, + DialectExpression, + DialectExpressionSet, + Formula, + JoinPath, + LinkMapping, + Metric, + ObjectMapping, + OntologyComponent, + OntologyMapping, + OsiOntology, + ReferentMapping, + Relationship, + RelationshipMultiplicity, + Role, + SemanticModel, +) +from osi.spec import OsiSpec +from osi.parser import OsiParser +from osi.external.palantir.parser import PalantirParser +from osi.converter.spec_to_osi.converter import SpecToOsiConverter +from osi.converter.osi_to_spec.converter import OsiToSpecConverter +from osi.converter.palantir_to_osi.converter import PalantirToOsiConverter + +__all__ = [ + # Model — ontology layer + "Concept", + "ConceptType", + "Relationship", + "RelationshipMultiplicity", + "Role", + "Formula", + # Model — semantic layer + "Dataset", + "DatasetField", + "DialectExpression", + "DialectExpressionSet", + "JoinPath", + "Metric", + "SemanticModel", + # Model — mapping layer + "ObjectMapping", + "ReferentMapping", + "LinkMapping", + "ConceptMapping", + "OntologyMapping", + "OntologyComponent", + "OsiOntology", + # Supporting types + "CustomExtension", + # Spec DTO + "OsiSpec", + # Parsers + "OsiParser", + "PalantirParser", + # Converters + "SpecToOsiConverter", + "OsiToSpecConverter", + "PalantirToOsiConverter", +] \ No newline at end of file diff --git a/converters/ontology/src/osi/common/__init__.py b/converters/ontology/src/osi/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/ontology/src/osi/common/file_utils.py b/converters/ontology/src/osi/common/file_utils.py new file mode 100644 index 00000000..99590524 --- /dev/null +++ b/converters/ontology/src/osi/common/file_utils.py @@ -0,0 +1,42 @@ +import io +import zipfile +from typing import Iterable + + +def iter_json_files_from_dir_in_zip(zf: zipfile.ZipFile, dir_prefix: str) -> Iterable[tuple[str, io.IOBase]]: + names = zf.namelist() + base_prefix = dir_prefix.rstrip("/") + "/" + roots = {n.split("/", 1)[0] for n in names if "/" in n} + candidate_prefixes = [base_prefix] + if len(roots) == 1: + root = next(iter(roots)) + candidate_prefixes.append(f"{root}/{base_prefix}") + + seen = set() + for name in names: + if name.endswith("/") or not name.lower().endswith(".json"): + continue + if any(name.startswith(p) for p in candidate_prefixes): + if name in seen: + continue + seen.add(name) + with zf.open(name, "r") as fp: + yield name, io.BytesIO(fp.read()) + +def open_top_level_file_from_zip(zf: zipfile.ZipFile, filename: str) -> io.IOBase: + names = set(zf.namelist()) + + def _open_to_io_base(n: str) -> io.IOBase: + with zf.open(n, "r") as fp: + return io.BytesIO(fp.read()) + + if filename in names: + return _open_to_io_base(filename) + + roots = {n.split("/", 1)[0] for n in names if "/" in n} + if len(roots) == 1: + candidate = f"{next(iter(roots))}/{filename}" + if candidate in names: + return _open_to_io_base(candidate) + + raise FileNotFoundError(f"Missing required top-level file: {filename}") \ No newline at end of file diff --git a/converters/ontology/src/osi/common/graph.py b/converters/ontology/src/osi/common/graph.py new file mode 100644 index 00000000..aa09f61a --- /dev/null +++ b/converters/ontology/src/osi/common/graph.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import warnings +from collections import defaultdict +from typing import TypeVar + +T = TypeVar("T") + + +def topological_sort(nodes: list[T], edges: list[tuple[T, T]]) -> list[T]: + order = _topological_sort(nodes, edges) + if order is None: + raise ValueError("The graph contains a cycle") + return order + + +def topological_sort_break_cycles(nodes: list[T], edges: list[tuple[T, T]]) -> tuple[list[T], list[tuple[T, T]]]: + order, removed_edges = _topological_sort_break_cycles(nodes, edges) + # `order` should always exist; defensive check: + if order is None: + raise ValueError("Could not break cycles to obtain a topological order") + + return order, removed_edges + + +def is_acyclic_graph(nodes: list[T], edges: list[tuple[T, T]]) -> bool: + return _topological_sort(nodes, edges) is not None + + +def _find_cycle_closing_edge_index( + nodes: list[T], + edge_list: defaultdict[T, list[tuple[T, int]]], + active: list[bool], + remaining_set: set[T], +) -> int | None: + """ + Find a cycle in the active subgraph induced by remaining_set and return the + index of a "cycle-closing" edge (a back-edge u->v where v is on the recursion stack). + """ + visited: set[T] = set() + on_stack: set[T] = set() + + def dfs(u: T) -> int | None: + visited.add(u) + on_stack.add(u) + + for v, eidx in edge_list.get(u, []): + if not active[eidx]: + continue + if v not in remaining_set: + continue + + if v not in visited: + found = dfs(v) + if found is not None: + return found + elif v in on_stack: + # Back-edge found: u -> v closes a directed cycle + return eidx + + on_stack.remove(u) + return None + + for start in nodes: + if start in remaining_set and start not in visited: + found = dfs(start) + if found is not None: + return found + + return None + + +def _topological_sort_break_cycles(nodes: list[T], edges: list[tuple[T, T]]) -> tuple[list[T] | None, list[tuple[T, T]]]: + """ + Returns (topological_order, removed_edges). + + Strategy: + - Run a Kahn-like process. + - When it gets stuck, detect a real cycle in the remaining subgraph via DFS + and remove the cycle-closing edge (back-edge) from that cycle. + - Continue until all nodes can be processed. + - Then run a clean topological sort once on the pruned edge list. + """ + node_set = set(nodes) + + edge_list: defaultdict[T, list[tuple[T, int]]] = defaultdict(list) + active = [True] * len(edges) + + in_degree: dict[T, int] = {n: 0 for n in nodes} + for idx, (src, tgt) in enumerate(edges): + if src not in node_set or tgt not in node_set: + active[idx] = False + continue + edge_list[src].append((tgt, idx)) + in_degree[tgt] += 1 + + processed: set[T] = set() + removed_edges: list[tuple[T, T]] = [] + + work: list[T] = [n for n in nodes if in_degree.get(n, 0) == 0] + + while len(processed) < len(nodes): + if work: + n = work.pop() + if n in processed: + continue + processed.add(n) + + for neighbour, eidx in edge_list.get(n, []): + if not active[eidx]: + continue + in_degree[neighbour] -= 1 + if in_degree[neighbour] == 0: + work.append(neighbour) + continue + + remaining_set = {n for n in nodes if n not in processed} + + edge_idx = _find_cycle_closing_edge_index( + nodes=nodes, + edge_list=edge_list, + active=active, + remaining_set=remaining_set, + ) + if edge_idx is None: + raise ValueError("Cycle suspected but could not identify a cycle edge to remove") + + src, tgt = edges[edge_idx] + active[edge_idx] = False + removed_edges.append((src, tgt)) + warnings.warn(f"Cycle detected: removing cycle-closing edge {src!r} -> {tgt!r}") + + # Update in_degree to reflect edge removal + in_degree[tgt] -= 1 + if in_degree[tgt] == 0: + work.append(tgt) + + cleaned_edges = [e for i, e in enumerate(edges) if active[i]] + order = _topological_sort(nodes, cleaned_edges) + if order is None: + raise ValueError("Graph is still cyclic after cycle-breaking edge removals") + + return order, removed_edges + + +def _topological_sort(nodes: list[T], edges: list[tuple[T, T]]) -> list[T] | None: + order = [] + + # simple implementation of Kahn's Algorithm + + # index edges + edge_list = defaultdict(list) + for src, tgt in edges: + edge_list[src].append(tgt) + + # compute in_degree of nodes + in_degree = dict() + for _, tgt in edges: + if tgt in in_degree: + in_degree[tgt] = in_degree[tgt] + 1 + else: + in_degree[tgt] = 1 + + # start the working list with nodes that don't have incoming edges + work = list(filter(lambda n: n not in in_degree, nodes)) + while work: + n = work.pop() + order.append(n) + for neighbour in edge_list[n]: + new_in_degree = in_degree[neighbour] - 1 + in_degree[neighbour] = new_in_degree + if new_in_degree == 0: + work.append(neighbour) + + # all nodes sorted, return the order + if len(order) == len(nodes): + return order + + # some nodes were not sorted, so the graph is cyclic, return None + return None diff --git a/converters/ontology/src/osi/common/utils.py b/converters/ontology/src/osi/common/utils.py new file mode 100644 index 00000000..e2fa8bdd --- /dev/null +++ b/converters/ontology/src/osi/common/utils.py @@ -0,0 +1,43 @@ +import logging +import re +from keyword import iskeyword + + +def camel_to_snake(name: str) -> str: + return re.sub(r'(? str: + words = re.split(r'[\s_\-\(\)<>:]+', text) + return ''.join(capitalize_first(word) for word in words)\ + .replace('[', '')\ + .replace(']', '_')\ + .replace('&', 'And') + +def capitalize_first(s): + return s[0].upper() + s[1:] if s else s + +digit_names = {'0': 'Zero', '1': 'One', '2': 'Two', '3': 'Three', '4': 'Four', + '5': 'Five', '6': 'Six', '7': 'Seven', '8': 'Eight', '9': 'Nine'} + +def to_verbalization_string(verb_string: str) -> str: + canonical_name = verb_string.lower().strip() + # replace ' ' and '-' with '_' + canonical_name = re.sub(r'[-\s]', '_', canonical_name) + # drop subsequent '_' + canonical_name = re.sub(r'_+', '_', canonical_name) + # replace unsupported symbols with '_' + new_name = re.sub(r'[^a-zA-Z0-9_-]', '_', canonical_name) + + if not new_name: + raise ValueError(f"Verbalization string {verb_string!r} reduces to an empty identifier after normalisation") + + # replace leading digits with alpha + if new_name[0].isdigit(): + new_name = digit_names[new_name[0]] + new_name[1:] + + if new_name != canonical_name: + logging.warning(f"Verbalization string {verb_string} has unsupported symbols. Replacing them with '_'") + if iskeyword(new_name): + new_name = f"{new_name}_k" + logging.warning(f"Verbalization string {verb_string} is a reserved keyword. Appending '_k' suffix.") + return new_name \ No newline at end of file diff --git a/converters/ontology/src/osi/converter/__init__.py b/converters/ontology/src/osi/converter/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/ontology/src/osi/converter/osi_to_spec/__init__.py b/converters/ontology/src/osi/converter/osi_to_spec/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/ontology/src/osi/converter/osi_to_spec/converter.py b/converters/ontology/src/osi/converter/osi_to_spec/converter.py new file mode 100644 index 00000000..6bc66562 --- /dev/null +++ b/converters/ontology/src/osi/converter/osi_to_spec/converter.py @@ -0,0 +1,271 @@ +"""Reverse converter: OsiOntology (runtime) -> OsiSpec (Pydantic DTO). + +Pairs with spec_to_osi.SpecToOsiConverter so a full round-trip +yaml -> spec -> model -> spec -> yaml is structurally stable.""" + +from __future__ import annotations + +from osi.model import ( + Concept, + ConceptMapping, + ConceptType, + CustomExtension, + Dataset, + DatasetField, + DialectExpressionSet, + Dimension, + JoinPath, + LinkMapping, + SemanticModel, + Metric, + ObjectMapping, + OntologyComponent, + OntologyMapping, + ReferentMapping, + Relationship, + OsiOntology, +) +from osi.spec import ( + Concept as SpecConcept, + ConceptComponent, + ConceptMapping as SpecConceptMapping, + CustomExtension as SpecCustomExtension, + Dataset as SpecDataset, + DatasetField as SpecDatasetField, + DialectExpression as SpecDialectExpression, + Dimension as SpecDimension, + Expression as SpecExpression, + JoinPath as SpecJoinPath, + LinkMapping as SpecLinkMapping, + SemanticModel as SpecSemanticModel, + Metric as SpecMetric, + ObjectMapping as SpecObjectMapping, + OntologyMapping as SpecOntologyMapping, + OsiSpec, + ReferentMapping as SpecReferentMapping, + Relationship as SpecRelationship, + Role as SpecRole, +) + + +class OsiToSpecConverter: + """Top-level reverse converter.""" + + @staticmethod + def convert(model: OsiOntology) -> OsiSpec: + ont = model.ontology + ontology_mappings = [_convert_ontology_mapping(ontology_mapping) for ontology_mapping in model.ontology_mappings] + return OsiSpec( + version=model.version, + name=model.name, + description=model.description, + ai_context=model.ai_context, + ontology=_convert_ontology_concepts(ont), + ontology_mappings=ontology_mappings, + ) + + +# --------------------------------------------------------------------------- +# Ontology +# --------------------------------------------------------------------------- + +def _convert_ontology_concepts(ont: OntologyComponent) -> list[ConceptComponent]: + components: list[ConceptComponent] = [] + for concept in ont.concepts(): + rels = [rel for rel in ont.relationships if rel.container is concept] + if rels: + components.append( + ConceptComponent( + concept=_convert_concept(concept), + relationships=[_convert_relationship(rel) for rel in rels], + ) + ) + return components + + +def _convert_concept(concept: Concept) -> SpecConcept: + type_value: str | None = None + if isinstance(concept.type, ConceptType): + type_value = concept.type.value # type: ignore[union-attr] + extends = [p.name for p in concept.extends] if concept.extends else None + + identify_by: list[str] = [rel.name for rel in concept.identify_by.values()] + derived_by = [f.raw_expr for f in concept.derived_by] + requires = [f.raw_expr for f in concept.requires] + + return SpecConcept( + name=concept.name, + type=type_value, # type: ignore[arg-type] + description=concept.description, + extends=extends, + identify_by=identify_by, + derived_by=derived_by, + requires=requires, + ) + + +def _convert_relationship(rel: Relationship) -> SpecRelationship: + extra_roles = list(rel.roles)[1:] + roles = [SpecRole(concept=role.player.name, name=role.explicit_name) for role in extra_roles] + + multiplicity = rel.multiplicity.value if rel.multiplicity is not None else None + verbalizes = rel.verbalizes_raw if rel.verbalizes_raw is not None else [] + + return SpecRelationship( + name=rel.name, + description=rel.description, + roles=roles, + verbalizes=verbalizes, + multiplicity=multiplicity, # type: ignore[arg-type] + derived_by=[f.raw_expr for f in rel.derived_by], + requires=[f.raw_expr for f in rel.requires], + ) + + +# --------------------------------------------------------------------------- +# Semantic model +# --------------------------------------------------------------------------- + +def _convert_semantic_model(semantic_model: SemanticModel) -> SpecSemanticModel: + return SpecSemanticModel( + name=semantic_model.name, + description=semantic_model.description, + ai_context=semantic_model.ai_context, + datasets=[_convert_dataset(ds) for ds in semantic_model.datasets], + relationships=[_convert_join_path(jp) for jp in semantic_model.join_paths], + metrics=[_convert_metric(metric) for metric in semantic_model.metrics], + custom_extensions=[_convert_custom_extension(ce) for ce in semantic_model.custom_extensions], + ) + + +def _convert_dataset(ds: Dataset) -> SpecDataset: + return SpecDataset( + name=ds.name, + source=ds.source, + primary_key=ds.primary_key, + unique_keys=ds.unique_keys, + description=ds.description, + ai_context=ds.ai_context, + fields=[_convert_dataset_field(fl) for fl in ds.fields], + custom_extensions=[_convert_custom_extension(ce) for ce in ds.custom_extensions], + ) + + +def _convert_dataset_field(fl: DatasetField) -> SpecDatasetField: + return SpecDatasetField( + name=fl.name, + expression=_convert_expression(fl.expression), + dimension=_convert_dimension(fl.dimension), + label=fl.label, + description=fl.description, + ai_context=fl.ai_context, + custom_extensions=[_convert_custom_extension(ce) for ce in fl.custom_extensions], + ) + + +def _convert_expression(es: DialectExpressionSet) -> SpecExpression: + return SpecExpression( + dialects=[SpecDialectExpression(dialect=d.dialect, expression=d.expression) for d in es.dialects] + ) + + +def _convert_dimension(dim: Dimension | None) -> SpecDimension | None: + if dim is None: + return None + return SpecDimension(is_time=dim.is_time) + + +def _convert_join_path(jp: JoinPath) -> SpecJoinPath: + return SpecJoinPath( + name=jp.name, + **{"from": jp.from_dataset.name}, # `from` is a reserved word in Python + to=jp.to_dataset.name, + from_columns=[from_col.name for from_col in jp.from_columns], + to_columns=[to_col.name for to_col in jp.to_columns], + ai_context=jp.ai_context, + custom_extensions=[_convert_custom_extension(ce) for ce in jp.custom_extensions], + ) + + +def _convert_metric(metric: Metric) -> SpecMetric: + return SpecMetric( + name=metric.name, + expression=_convert_expression(metric.expression), + description=metric.description, + ai_context=metric.ai_context, + custom_extensions=[_convert_custom_extension(ce) for ce in metric.custom_extensions], + ) + + +def _convert_custom_extension(ce: CustomExtension) -> SpecCustomExtension: + return SpecCustomExtension(vendor_name=ce.vendor_name, data=ce.data) + + +# --------------------------------------------------------------------------- +# Ontology mapping (tree) +# --------------------------------------------------------------------------- + +def _convert_ontology_mapping(ontology_mapping: OntologyMapping) -> SpecOntologyMapping: + return SpecOntologyMapping( + name=ontology_mapping.name, + description=ontology_mapping.description, + semantic_model=_convert_semantic_model(ontology_mapping.semantic_model), + concept_mappings=[_convert_concept_mapping(concept_mapping) for concept_mapping in ontology_mapping.concept_mappings], + ) + + +def _convert_concept_mapping(concept_mapping: ConceptMapping) -> SpecConceptMapping: + return SpecConceptMapping( + concept=concept_mapping.concept.name, + object_mappings=[_convert_object_mapping(object_mapping) for object_mapping in concept_mapping.object_mappings], + link_mappings=[_convert_link_mapping(link_mapping) for link_mapping in concept_mapping.link_mappings], + ) + + +def _convert_object_mapping(object_mapping: ObjectMapping) -> SpecObjectMapping: + referent_mappings = None + if object_mapping.referent_mappings is not None: + referent_mappings = [_convert_referent_mapping(rm) for rm in object_mapping.referent_mappings] + return SpecObjectMapping( + concept=object_mapping.concept.name if object_mapping.concept is not None else None, + expression=_render_mapping_expression(object_mapping.expression), + referent_mappings=referent_mappings, + ) + + +def _convert_referent_mapping(referent_mapping: ReferentMapping) -> SpecReferentMapping: + nested = None + if referent_mapping.referent_mappings is not None: + nested = [_convert_referent_mapping(child) for child in referent_mapping.referent_mappings] + return SpecReferentMapping( + relationship=referent_mapping.relationship.name, + expression=_render_mapping_expression(referent_mapping.expression), + referent_mappings=nested, + ) + + +def _render_mapping_expression(expr) -> str | None: + """Reconstruct the source string for a parsed mapping expression. The + runtime model carries either a `DatasetField` (single field reference) + or a `Formula` (richer expression); both round-trip back to the same + string the forward converter saw in the spec.""" + if expr is None: + return None + from osi.model import DatasetField as _DF, Formula as _F + if isinstance(expr, _DF): + ds = expr.dataset + return f"{ds.name}.{expr.name}" if ds is not None else expr.name + if isinstance(expr, _F): + return expr.raw_expr + return str(expr) + + +def _convert_link_mapping(link_mapping: LinkMapping) -> SpecLinkMapping: + children = None + if link_mapping.children is not None: + children = [_convert_link_mapping(child) for child in link_mapping.children] + return SpecLinkMapping( + object_mapping=_convert_object_mapping(link_mapping.object_mapping), + relationship=link_mapping.relationship.name if link_mapping.relationship is not None else None, + children=children, + ) \ No newline at end of file diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/__init__.py b/converters/ontology/src/osi/converter/palantir_to_osi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py new file mode 100644 index 00000000..a3578828 --- /dev/null +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -0,0 +1,778 @@ +"""Palantir `Ontology` -> `OsiOntology`.""" + +from __future__ import annotations + +import warnings + +from osi.common.graph import topological_sort_break_cycles +from osi.common.utils import to_pascal_case, to_verbalization_string +from osi.external.palantir.model import ( + ArrayDataType, + DataSet as PalantirDataSet, + DataSetColumn, + DataType, + IntermediaryRelation, + ManyToManyRelation, + ManyToOneRelation, + ObjectType, + Ontology as PalantirOntology, + Property as PalantirProperty, + Relation, +) +from osi.model import ( + Concept, + ConceptMapping, + ConceptType, + Dataset, + DatasetField, + DialectExpression, + DialectExpressionSet, + Formula, + LinkMapping, + SemanticModel, + ObjectMapping, + OntologyComponent, + OntologyMapping, + ReferentMapping, + Relationship, + RelationshipMultiplicity, + OsiOntology +) + + +_DEFAULT_DIALECT = "ANSI_SQL" + + +class PalantirToOsiConverter: + """Top-level converter. Use `convert(ontology_model)` to obtain an `OsiOntology`.""" + + depths_role_names = {1: "fst", 2: "snd", 3: "thd", 4: "frt"} + + # ------------------------------------------------------------------ + # Entry point + # ------------------------------------------------------------------ + + @staticmethod + def convert( + palantir_ontology: PalantirOntology, + db_name: str = "palantir", + schema_name: str = "palantir", + ) -> OsiOntology: + ontology = OntologyComponent() + model = OsiOntology(name="Palantir model", ontology=ontology, version="0.1.0") + + semantic_model = SemanticModel(name="Palantir semantic model") + + ontology_mapping = OntologyMapping(name="palantir_map", ontology=ontology, semantic_model=semantic_model) + model.add_ontology_mapping(ontology_mapping) + + # Per-(concept, dataset) ConceptMappings accumulate here as datasets + # get created; emitted into the OntologyMapping at the end so they appear in a stable order. + concept_mappings: list[ConceptMapping] = [] + + PalantirToOsiConverter._convert_concepts( + ontology, semantic_model, palantir_ontology, concept_mappings, db_name, schema_name + ) + PalantirToOsiConverter._convert_relationships( + ontology, palantir_ontology, concept_mappings, semantic_model + ) + + for cm in concept_mappings: + ontology_mapping.add_concept_mapping(cm) + + return model + + # ------------------------------------------------------------------ + # Concepts + # ------------------------------------------------------------------ + + @staticmethod + def _convert_concepts( + ontology: OntologyComponent, + semantic_model: SemanticModel, + palantir_ontology: PalantirOntology, + concept_mappings: list[ConceptMapping], + db_name: str, + schema_name: str, + ) -> None: + subtype_relations = palantir_ontology.subtypes_relations() + + nodes = [ot.guid() for ot in palantir_ontology.object_types().values()] + edges: list[tuple[str, str]] = [] + edge_to_relation_guid: dict[tuple[str, str], str] = {} + for child, rel in subtype_relations.items(): + parent = rel.many_object_type() + if child == parent: + continue + edge = (parent.guid(), child.guid()) + edges.append(edge) + edge_to_relation_guid[edge] = rel.guid() + + order, removed_edges = topological_sort_break_cycles(nodes, edges) + # Subtype edges that would form cycles get dropped by the topo sort — + # treat them as ignored inheritance below. + ignore_subtype_relation_ids = {edge_to_relation_guid[e] for e in removed_edges} + + for ot_guid in order: + ot = palantir_ontology.object_types()[ot_guid] + if ot.active() or ot.endorsed() or ot.intermediary(): + PalantirToOsiConverter._convert_object_type( + ontology, + semantic_model, + ot, + subtype_relations, + ignore_subtype_relation_ids, + concept_mappings, + db_name, + schema_name, + ) + + @staticmethod + def _convert_object_type( + ontology: OntologyComponent, + semantic_model: SemanticModel, + ot: ObjectType, + subtype_relations: dict[ObjectType, ManyToOneRelation], + ignore_subtype_relation_ids: set[str], + concept_mappings: list[ConceptMapping], + db_name: str, + schema_name: str, + ) -> None: + concept_name = PalantirToOsiConverter._concept_name(ot) + relevant_props = [ + p for p in ot.properties().values() if p.active() or p.experimental() or p.intermediary() + ] + concept: Concept | None = None + + if ontology.lookup_concept(concept_name) is None: + is_subtype = ot in subtype_relations + subtype_relation = subtype_relations.get(ot) + ignore_subtype = bool( + subtype_relation and subtype_relation.guid() in ignore_subtype_relation_ids + ) + + if is_subtype and not ignore_subtype: + parent_ot = subtype_relation.many_object_type() # type: ignore[union-attr] + parent_name = PalantirToOsiConverter._concept_name(parent_ot) + parent = ontology.lookup_concept(parent_name) + assert parent is not None, f"Parent concept '{parent_name}' not found (expected from topological order)" + concept = Concept(name=concept_name, type=ConceptType.ENTITY_TYPE, extends=[parent]) + else: + concept = Concept(name=concept_name, type=ConceptType.ENTITY_TYPE) + ontology.add_concept(concept) + + for prop in relevant_props: + PalantirToOsiConverter._convert_property(ontology, concept, prop) + + if not is_subtype or ignore_subtype: + identifiers: dict[str, Relationship] = {} + for prop in ot.primary_keys(): + prop_name = PalantirToOsiConverter._attribute_name(prop) + rel = ontology.lookup_concept_relationship(concept, prop_name) + if rel is None: + raise ValueError( + f"Identifier relationship '{concept_name}.{prop_name}' not found " + f"while wiring primary keys for ObjectType '{ot.name()}'." + ) + identifiers[rel.full_name] = rel + concept.set_identify_by(identifiers) + # Set multiplicities now that we know which relationship is the sole identifier. + # A non-composite identifier is OneToOne; all others stay ManyToOne. + sole = next(iter(identifiers.values())) if len(identifiers) == 1 else None + for prop in relevant_props: + prop_name = PalantirToOsiConverter._attribute_name(prop) + prop_rel = ontology.lookup_concept_relationship(concept, prop_name) + if prop_rel is not None: + mult = RelationshipMultiplicity.ONE_TO_ONE if prop_rel is sole else RelationshipMultiplicity.MANY_TO_ONE + prop_rel.set_multiplicity(mult) + else: + concept = ontology.lookup_concept(concept_name) + assert concept is not None + # Re-encountered concept (multiple datasets feeding the same OT). + # Verify every relevant property already has its relationship — + # otherwise the second dataset is contributing fields the first + # didn't declare, which produces an asymmetric model. + for prop in relevant_props: + prop_name = PalantirToOsiConverter._attribute_name(prop) + if ontology.lookup_concept_relationship(concept, prop_name) is None: + raise ValueError( + f"Concept '{concept_name}' refers to multiple datasets but not all " + f"contain the '{prop_name}' property." + ) + + PalantirToOsiConverter._convert_mappings( + ontology, semantic_model, ot, subtype_relations, concept, concept_mappings, db_name, schema_name + ) + + @staticmethod + def _convert_property(ontology: OntologyComponent, concept: Concept, prop: PalantirProperty) -> None: + def madlib_decl(c: Concept, p: PalantirProperty) -> str: + return ( + f"{{{c}}} {p.readable_id()} " + f"{PalantirToOsiConverter._type_to_madlib_suffix(p.type())}" + ) + + prop_name = PalantirToOsiConverter._attribute_name(prop) + if ontology.lookup_concept_relationship(concept, prop_name) is not None: + return + + relates: list[tuple[Concept, str | None]] = [] + relates = PalantirToOsiConverter._convert_property_type_roles(ontology, relates, prop.type()) + + ontology.add_relationship(Relationship( + name=prop_name, + container=concept, + relates=relates, + verbalizes=[madlib_decl(concept, prop)], + )) + + # ------------------------------------------------------------------ + # Mappings: ConceptMapping per (concept, dataset) + # ------------------------------------------------------------------ + + @staticmethod + def _convert_mappings( + ontology: OntologyComponent, + semantic_model: SemanticModel, + ot: ObjectType, + subtype_relations: dict[ObjectType, ManyToOneRelation], + concept: Concept, + concept_mappings: list[ConceptMapping], + db_name: str, + schema_name: str, + ) -> None: + if not ot._syncs_from: + return + + parent_concept: Concept | None = None + subtype_relation = subtype_relations.get(ot) + + if subtype_relation is not None: + parent_ot = subtype_relation.many_object_type() + parent_concept = ontology.lookup_concept( + PalantirToOsiConverter._concept_name(parent_ot) + ) + property_map = subtype_relation.property_map() + identifier_props = list(parent_ot.primary_keys()) + + def resolve(p: PalantirProperty) -> PalantirProperty: + return property_map[p] + else: + identifier_props = list(ot.primary_keys()) + + def resolve(p: PalantirProperty) -> PalantirProperty: + return p + + for palantir_ds in ot.syncs_from(): + dataset = PalantirToOsiConverter._convert_dataset( + semantic_model, ontology, ot, palantir_ds, db_name, schema_name + ) + + # Build referent_mappings that locate `concept` instances by + # walking the (effective) identifying relationships against this + # dataset's columns. + id_referents: list[ReferentMapping] = [] + for prop in identifier_props: + prop_name = PalantirToOsiConverter._attribute_name(prop) + # For subtypes, identifying relationships live on the parent + # concept; the child reaches them via `lookup_concept_relationship`. + rel = ontology.lookup_concept_relationship(concept, prop_name) + if rel is None: + continue + field = PalantirToOsiConverter._get_dataset_field_by_palantir_property( + resolve(prop), palantir_ds, dataset + ) + if field is None: + continue + id_referents.append(ReferentMapping(relationship=rel, expression=field)) + + cm = ConceptMapping(concept=concept) + + # object_mappings: how to construct/identify this concept's + # instances from this dataset. Always uses referent_mappings to + # walk the identifying relationships (whether own or inherited). + cm.object_mappings.append( + ObjectMapping( + concept=parent_concept, + referent_mappings=list(id_referents) if id_referents else None, + ) + ) + + # link_mappings: the root identifies the source object (same as + # object_mapping), children populate each property relationship. + children: list[LinkMapping] = [] + primary_keys = set(ot.primary_keys()) + for prop in ot.properties().values(): + if not (prop.active() or prop.intermediary()): + continue + if prop in primary_keys: + continue + if not prop.pk_mapping() and prop.datasource_resource_id() != palantir_ds.guid(): + continue + if isinstance(prop.type(), ArrayDataType): + warnings.warn( + f"Skipping property '{prop.readable_id()}'. Array datatype is not supported" + ) + continue + + prop_name = PalantirToOsiConverter._attribute_name(prop) + relationship = ontology.lookup_concept_relationship(concept, prop_name) + if relationship is None: + continue + field = PalantirToOsiConverter._get_dataset_field_by_palantir_property( + prop, palantir_ds, dataset + ) + if field is None: + continue + value_concept = relationship.last_role.player + children.append( + LinkMapping( + object_mapping=ObjectMapping(concept=value_concept,expression=field), + relationship=relationship, + ) + ) + + if id_referents or children: + cm.link_mappings.append( + LinkMapping( + object_mapping=ObjectMapping( + concept=parent_concept, + referent_mappings=list(id_referents) if id_referents else None, + ), + children=children if children else None, + ) + ) + + concept_mappings.append(cm) + + # ------------------------------------------------------------------ + # Relations (M:1, M:M, intermediary) + # ------------------------------------------------------------------ + + @staticmethod + def _convert_relationships( + ontology: OntologyComponent, + palantir_ontology: PalantirOntology, + concept_mappings: list[ConceptMapping], + semantic_model: SemanticModel, + ) -> None: + for rel in palantir_ontology.relations().values(): + if rel.active() or rel.intermediary(): + PalantirToOsiConverter._convert_relation(ontology, rel, concept_mappings, semantic_model) + elif ( + isinstance(rel, ManyToOneRelation) + and rel.experimental() + and rel.one_object_type().active() + and rel.many_object_type().active() + ): + PalantirToOsiConverter._convert_relation(ontology, rel, concept_mappings, semantic_model) + + for ir in palantir_ontology.intermediary_relations().values(): + if ir.active() or ir.intermediary(): + PalantirToOsiConverter._convert_intermediary_relation(ontology, palantir_ontology, ir) + elif ( + ir.experimental() + and ir.role_a_player().active() + and ir.role_b_player().active() + and ir.intermediary_player().active() + ): + PalantirToOsiConverter._convert_intermediary_relation(ontology, palantir_ontology, ir) + + @staticmethod + def _convert_relation( + ontology: OntologyComponent, + relation: Relation, + concept_mappings: list[ConceptMapping], + semantic_model: SemanticModel, + ) -> None: + if isinstance(relation, ManyToOneRelation): + PalantirToOsiConverter._convert_many_to_one( + ontology, relation, concept_mappings, semantic_model + ) + elif isinstance(relation, ManyToManyRelation): + PalantirToOsiConverter._convert_many_to_many(ontology, relation) + + @staticmethod + def _convert_many_to_one( + ontology: OntologyComponent, + rel: ManyToOneRelation, + concept_mappings: list[ConceptMapping], + semantic_model: SemanticModel, + ) -> None: + mot = rel.many_object_type() + mot_name = PalantirToOsiConverter._concept_name(mot) + mot_concept = ontology.lookup_concept(mot_name) + oot = rel.one_object_type() + oot_name = PalantirToOsiConverter._concept_name(oot) + oot_concept = ontology.lookup_concept(oot_name) + if mot_concept is None or oot_concept is None: + return + prop_name = PalantirToOsiConverter._attribute_name(rel) + + if mot_concept is oot_concept: + verbalize = f"{{{mot_concept}}} {prop_name} {{{oot_concept}:snd}}" + relates: list[tuple[Concept, str | None]] = [(oot_concept, "snd")] + else: + verbalize = f"{{{mot_concept}}} {prop_name} {{{oot_concept}}}" + relates = [(oot_concept, None)] + + relationship = Relationship( + name=prop_name, + container=mot_concept, + relates=relates, + verbalizes=[verbalize], + multiplicity=RelationshipMultiplicity.MANY_TO_ONE, + ) + ontology.add_relationship(relationship) + + if mot._syncs_from: + PalantirToOsiConverter._attach_link_to_concept_mappings( + ontology, rel, relationship, mot, mot_concept, oot_concept, concept_mappings, semantic_model + ) + else: + # No many-side datasets: fall back to a derived_by formula that + # equates FK columns. + frags = [ + f"{relationship.first_role.name}.{PalantirToOsiConverter._attribute_name(mprop)}" + f" == {relationship.last_role.name}.{PalantirToOsiConverter._attribute_name(oprop)}" + for mprop, oprop in rel.property_map().items() + ] + if frags: + formula = Formula(raw_expr=" AND ".join(frags), parent=relationship) + relationship.add_derived_by(formula) + ontology.add_rule(formula) + + @staticmethod + def _attach_link_to_concept_mappings( + ontology: OntologyComponent, + rel: ManyToOneRelation, + relationship: Relationship, + mot: ObjectType, + mot_concept: Concept, + oot_concept: Concept, + concept_mappings: list[ConceptMapping], + semantic_model: SemanticModel, + ) -> None: + """For each (mot_concept, dataset) ConceptMapping, append a link_mapping + child that walks the target concept's identifying relationships through + the source's FK columns.""" + property_map = rel.property_map() + if not property_map: + return + + # Resolve target (oot) identifying relationships once. + target_id_rels: list[tuple[Relationship, PalantirProperty]] = [] + for mprop, oprop in property_map.items(): + oot_attr = PalantirToOsiConverter._attribute_name(oprop) + id_rel = ontology.lookup_concept_relationship(oot_concept, oot_attr) + if id_rel is None: + return + target_id_rels.append((id_rel, mprop)) + + for palantir_ds in mot.syncs_from(): + ds_name = ( + f"{PalantirToOsiConverter._concept_name(mot)}_{palantir_ds.readable_id()}" + ) + dataset = semantic_model.lookup_dataset(ds_name) + if dataset is None: + continue + + cm = PalantirToOsiConverter._find_concept_mapping(concept_mappings, mot_concept, dataset) + if cm is None: + warnings.warn( + f"No ConceptMapping for entity '{mot_concept.name}' and dataset " + f"'{ds_name}'; cannot attach link '{relationship.full_name}'" + ) + continue + + # Build referent_mappings that look up the target via FK columns. + referents: list[ReferentMapping] = [] + resolved = True + for id_rel, mprop in target_id_rels: + fk_field = PalantirToOsiConverter._get_dataset_field_by_palantir_property( + mprop, palantir_ds, dataset + ) + if fk_field is None: + resolved = False + break + referents.append(ReferentMapping(relationship=id_rel, expression=fk_field)) + if not resolved: + continue + + child = LinkMapping( + object_mapping=ObjectMapping(concept=oot_concept, referent_mappings=referents), + relationship=relationship, + ) + # Attach as a child on the root link_mapping (the identifying tree). + if cm.link_mappings: + root = cm.link_mappings[0] + if root.children is None: + root.children = [] + root.children.append(child) + else: + if not cm.object_mappings: + raise ValueError( + f"Cannot attach link '{relationship.full_name}': concept " + f"'{mot_concept.name}' has no identifying object mapping " + f"to use as the link root." + ) + root_om = cm.object_mappings[0] + cm.link_mappings.append(LinkMapping( + object_mapping=ObjectMapping( + concept=root_om.concept, + referent_mappings=root_om.referent_mappings, + ), + children=[child], + )) + + @staticmethod + def _find_concept_mapping( + concept_mappings: list[ConceptMapping], + concept: Concept, + dataset: Dataset, + ) -> ConceptMapping | None: + """Resolve the ConceptMapping built for this (concept, dataset). + + When multiple datasets feed the same concept we get one ConceptMapping + per dataset; pick the one whose referent expressions reference + `dataset`, falling back to the first candidate.""" + candidates = [cm for cm in concept_mappings if cm.concept is concept] + if len(candidates) <= 1: + return candidates[0] if candidates else None + return next( + (cm for cm in candidates if PalantirToOsiConverter._references_dataset(cm, dataset)), + candidates[0], + ) + + @staticmethod + def _references_dataset(cm: ConceptMapping, dataset: Dataset) -> bool: + """True iff any referent expression in `cm` points to a field of `dataset`.""" + return any( + isinstance(rm.expression, DatasetField) and rm.expression.dataset is dataset + for om in cm.object_mappings + for rm in (om.referent_mappings or []) + ) + + @staticmethod + def _convert_many_to_many(ontology: OntologyComponent, rel: ManyToManyRelation) -> None: + aot = rel.role_a_player() + aot_concept = ontology.lookup_concept(PalantirToOsiConverter._concept_name(aot)) + bot = rel.role_b_player() + bot_concept = ontology.lookup_concept(PalantirToOsiConverter._concept_name(bot)) + if aot_concept is None or bot_concept is None: + return + rel_name = PalantirToOsiConverter._attribute_name(rel) + + if aot_concept is bot_concept: + verbalize = f"{{{aot_concept}}} {rel_name} {{{bot_concept}:snd}}" + relates = [(bot_concept, "snd")] + else: + verbalize = f"{{{aot_concept}}} {rel_name} {{{bot_concept}}}" + relates = [(bot_concept, None)] + + relationship = Relationship( + name=rel_name, + container=aot_concept, + relates=relates, + verbalizes=[verbalize], + multiplicity=None, + ) + ontology.add_relationship(relationship) + + @staticmethod + def _convert_intermediary_relation( + ontology: OntologyComponent, + palantir_ontology: PalantirOntology, + rel: IntermediaryRelation, + ) -> None: + aot = rel.role_a_player() + aot_name = PalantirToOsiConverter._concept_name(aot) + aot_concept = ontology.lookup_concept(aot_name) + bot = rel.role_b_player() + bot_name = PalantirToOsiConverter._concept_name(bot) + bot_concept = ontology.lookup_concept(bot_name) + if aot_concept is None or bot_concept is None: + return + rel_name = PalantirToOsiConverter._attribute_name(rel) + + if aot_concept is bot_concept: + verbalize = f"{{{aot_concept}}} {rel_name} {{{bot_concept}:snd}}" + relates: list[tuple[Concept, str | None]] = [(bot_concept, "snd")] + else: + verbalize = f"{{{aot_concept}}} {rel_name} {{{bot_concept}}}" + relates = [(bot_concept, None)] + + relationship = Relationship( + name=rel_name, + container=aot_concept, + relates=relates, + verbalizes=[verbalize], + ) + ontology.add_relationship(relationship) + + rel_a = palantir_ontology.relations()[rel.relation_a()] + rel_a_name = PalantirToOsiConverter._attribute_name(rel_a) + rel_b = palantir_ontology.relations()[rel.relation_b()] + rel_b_name = PalantirToOsiConverter._attribute_name(rel_b) + + fp_a = PalantirToOsiConverter._concept_name( + rel_a.many_object_type() if isinstance(rel_a, ManyToOneRelation) else rel_a.role_a_player() + ) + sp_a = PalantirToOsiConverter._concept_name( + rel_a.one_object_type() if isinstance(rel_a, ManyToOneRelation) else rel_a.role_b_player() + ) + fp_b = PalantirToOsiConverter._concept_name( + rel_b.many_object_type() if isinstance(rel_b, ManyToOneRelation) else rel_b.role_a_player() + ) + sp_b = PalantirToOsiConverter._concept_name( + rel_b.one_object_type() if isinstance(rel_b, ManyToOneRelation) else rel_b.role_b_player() + ) + + assert (aot_name == fp_a and bot_name == fp_b) or ( + aot_name == sp_a and bot_name == sp_b + ), f"Invalid intermediary relation '{rel_name}' arguments." + + join_condition = ( + f"{fp_a}.{rel_a_name}({relationship.first_role.name}) AND " + f"{fp_b}.{rel_b_name}({relationship.last_role.name})" + ) + formula = Formula(raw_expr=join_condition, parent=relationship) + relationship.add_derived_by(formula) + ontology.add_rule(formula) + + # ------------------------------------------------------------------ + # Datasets + # ------------------------------------------------------------------ + + @staticmethod + def _convert_dataset( + semantic_model: SemanticModel, + ontology: OntologyComponent, + ot: ObjectType, + palantir_ds: PalantirDataSet, + db_name: str, + schema_name: str, + ) -> Dataset: + ds_name = f"{PalantirToOsiConverter._concept_name(ot)}_{palantir_ds.readable_id()}" + existing = semantic_model.lookup_dataset(ds_name) + if existing is not None: + return existing + + fields: list[DatasetField] = [] + for column in palantir_ds.columns(): + if column.type().upper() == "ARRAY": + continue + field_name = PalantirToOsiConverter._normalize_field_name(column.name()) + fields.append( + DatasetField( + name=field_name, + expression=DialectExpressionSet( + dialects=[ + DialectExpression(dialect=_DEFAULT_DIALECT, expression=field_name) + ] + ), + type=PalantirToOsiConverter._resolve_field_type(ontology, palantir_ds, column), + ) + ) + + dataset = Dataset( + name=ds_name, + source=f"{db_name}.{schema_name}.{palantir_ds.readable_id()}", + fields=fields, + description=palantir_ds.description(), + ) + semantic_model.add_dataset(dataset) + return dataset + + @staticmethod + def _resolve_field_type( + ontology: OntologyComponent, palantir_ds: PalantirDataSet, column: DataSetColumn + ) -> Concept: + type_str = ( + DataType.parse_datatype(column.type()).to_type() if column.type() else "String" + ) + concept = ontology.lookup_concept(type_str) + if not concept: + raise ValueError( + f"Concept '{type_str}' is not defined in the ontology but used in the " + f"DatasetField '{palantir_ds.readable_id()}.{column.name()}'." + ) + return concept + + # ------------------------------------------------------------------ + # Naming / typing helpers + # ------------------------------------------------------------------ + + @staticmethod + def _attribute_name(prop: PalantirProperty | Relation) -> str: + return to_verbalization_string(prop.readable_id()) + + @staticmethod + def _concept_name(ot: ObjectType) -> str: + return to_pascal_case(ot.name()) + + @staticmethod + def _type_to_madlib_suffix(type_, arr_depth: int = 1) -> str: + if isinstance(type_, ArrayDataType): + depth = arr_depth + return ( + f"{{Integer:{PalantirToOsiConverter._depth_role_name(depth)}}} maps to " + f"{PalantirToOsiConverter._type_to_madlib_suffix(type_.base_type(), depth + 1)}" + ) + return f"{{{type_.to_type()}}}" + + @staticmethod + def _convert_property_type_roles( + ontology: OntologyComponent, roles: list[tuple[Concept, str | None]], type_, arr_depth: int = 1 + ) -> list[tuple[Concept, str | None]]: + if isinstance(type_, ArrayDataType): + integer = ontology.lookup_concept("Integer") + if integer is None: + raise ValueError("Builtin 'Integer' could not be resolved for array role.") + roles.append((integer, PalantirToOsiConverter._depth_role_name(arr_depth))) + PalantirToOsiConverter._convert_property_type_roles( + ontology, roles, type_.base_type(), arr_depth + 1 + ) + else: + target = ontology.lookup_concept(type_.to_type()) + if target is None: + raise ValueError( + f"Type concept '{type_.to_type()}' is not defined in the ontology." + ) + roles.append((target, None)) + return roles + + @staticmethod + def _depth_role_name(depth: int) -> str: + name = PalantirToOsiConverter.depths_role_names.get(depth) + if not name: + raise Exception(f"Array types of depth {depth} are not supported") + return name + + @staticmethod + def _get_dataset_field_by_palantir_property( + prop: PalantirProperty, palantir_ds: PalantirDataSet, dataset: Dataset + ) -> DatasetField | None: + column_name = prop.column_name() + pk_mapping = prop.pk_mapping() + ds_guid = palantir_ds.guid() + if pk_mapping: + if ds_guid not in pk_mapping: + raise ValueError( + f"Primary key mapping for Palantir DataSet '{palantir_ds.readable_id()}' " + f"is missing property '{PalantirToOsiConverter._attribute_name(prop)}'" + ) + column_name = pk_mapping[ds_guid] + if not column_name: + return None + field = dataset.field(PalantirToOsiConverter._normalize_field_name(column_name)) + if not field: + warnings.warn(f"Dataset '{dataset.name}' does not contain a field named '{column_name}'") + return field + + @staticmethod + def _normalize_field_name(name: str) -> str: + normalized = name.replace("-", "_") + if normalized and normalized[0].isdigit(): + normalized = f"_{normalized}" + return normalized diff --git a/converters/ontology/src/osi/converter/spec_to_osi/__init__.py b/converters/ontology/src/osi/converter/spec_to_osi/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py new file mode 100644 index 00000000..a11a39c5 --- /dev/null +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -0,0 +1,499 @@ +"""Converter from OsiSpec (Pydantic DTOs) to OsiOntology (runtime semantic model).""" + +from __future__ import annotations + +import re + +from osi.common.graph import topological_sort +from osi.model import ( + Concept, + ConceptMapping, + ConceptType, + CustomExtension, + Dataset, + DatasetField, + DialectExpression, + DialectExpressionSet, + Dimension, + Formula, + JoinPath, + LinkMapping, + SemanticModel, + Metric, + ObjectMapping, + OntologyComponent, + OntologyMapping, + ReferentMapping, + Relationship, + RelationshipMultiplicity, + OsiOntology +) +from osi.spec import ( + Concept as SpecConcept, + ConceptMapping as SpecConceptMapping, + CustomExtension as SpecCustomExtension, + Dataset as SpecDataset, + DatasetField as SpecDatasetField, + DialectExpression as SpecDialectExpression, + Dimension as SpecDimension, + Expression as SpecExpression, + JoinPath as SpecJoinPath, + LinkMapping as SpecLinkMapping, + SemanticModel as SpecSemanticModel, + Metric as SpecMetric, + ObjectMapping as SpecObjectMapping, + OntologyMapping as SpecOntologyMapping, + OsiSpec, + ReferentMapping as SpecReferentMapping, + Relationship as SpecRelationship, +) +Container = Concept | Relationship + +# A mapping expression is treated as a single field reference when it matches +# `DATASET.field` or a bare `field` identifier — no parsing, just a pattern check. +_QUALIFIED_FIELD_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*\.\s*([A-Za-z_][A-Za-z0-9_]*)\s*$") +_BARE_FIELD_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*$") + + +class SpecToOsiConverter: + """Top-level converter. Use `convert(spec)` to obtain a OsiOntology.""" + + @staticmethod + def convert(spec: OsiSpec) -> OsiOntology: + ontology = OntologyComponent() + model = OsiOntology( + name=spec.name, + ontology=ontology, + description=spec.description, + ai_context=spec.ai_context, + version=spec.version, + ) + + SpecToOsiConverter._populate_ontology(ontology, spec) + + for om_spec in spec.ontology_mappings: + SpecToOsiConverter._convert_ontology_mapping(model, om_spec) + + return model + + # ----- Ontology ------------------------------------------------------ + + @staticmethod + def _populate_ontology(ontology: OntologyComponent, spec: OsiSpec) -> None: + + concept_specs = {concept_component.concept.name: concept_component.concept for concept_component in spec.ontology} + sorted_names = SpecToOsiConverter._sort_spec_dependency_graph(list(concept_specs.values())) + for name in sorted_names: + concept_spec = concept_specs[name] + extends: list[Concept] = [] + if concept_spec.extends: + for ext in concept_spec.extends: + parent = ontology.lookup_concept(ext) + if not parent: + raise ValueError( + f"Subtype '{ext}' is not declared in ontology '{spec.name}'." + ) + extends.append(parent) + ontology.add_concept( + Concept( + name=concept_spec.name, + type=ConceptType.from_value(concept_spec.type), + description=concept_spec.description, + extends=extends, + ) + ) + + for concept_component in spec.ontology: + container = ontology.lookup_concept(concept_component.concept.name) + if container is None: + raise ValueError(f"Internal: container concept '{concept_component.concept.name}' not found") + for rel_spec in concept_component.relationships: + SpecToOsiConverter._convert_relationship(ontology, container, rel_spec) + + # Identifiers: now that all relationships exist, resolve identify_by. + for concept_component in spec.ontology: + concept_spec = concept_component.concept + concept = ontology.lookup_concept(concept_spec.name) + if concept is None: + continue + identifiers: dict[str, Relationship] = {} + for ref_name in concept_spec.identify_by: + rel = ontology.lookup_concept_relationship(concept, ref_name) + if rel is None: + raise ValueError( + f"identify_by '{ref_name}' on concept '{concept.name}' refers to an " + f"unknown relationship in ontology '{spec.name}'." + ) + identifiers[rel.full_name] = rel + concept.set_identify_by(identifiers) + + # Formulas: derived_by + requires (after concepts/relationships exist). + for concept_component in spec.ontology: + concept_spec = concept_component.concept + concept = ontology.lookup_concept(concept_spec.name) + if concept is None: + continue + for raw in concept_spec.requires: + req = _build_rule(raw, concept) + if req: + concept.add_require(req) + ontology.add_require(req) + for raw in concept_spec.derived_by: + rule = _build_rule(raw, concept) + if rule: + concept.add_derived_by(rule) + ontology.add_rule(rule) + for rel_spec in concept_component.relationships: + rel = ontology.lookup_concept_relationship(concept, rel_spec.name) + if rel is None: + continue + for raw in rel_spec.requires: + req = _build_rule(raw, rel) + if req: + rel.add_require(req) + ontology.add_require(req) + for raw in rel_spec.derived_by: + rule = _build_rule(raw, rel) + if rule: + rel.add_derived_by(rule) + ontology.add_rule(rule) + + @staticmethod + def _convert_relationship( + ontology: OntologyComponent, container: Concept, rel_spec: SpecRelationship + ) -> None: + relates: list[tuple[Concept, str | None]] = [] + for role_spec in rel_spec.roles: + role_concept = ontology.lookup_concept(role_spec.concept) + if role_concept is None: + raise ValueError( + f"Role concept '{role_spec.concept}' in relationship '{container.name}.{rel_spec.name}' " + f"is not declared in the ontology." + ) + relates.append((role_concept, role_spec.name)) + + multiplicity = RelationshipMultiplicity.from_value(rel_spec.multiplicity) + relationship = Relationship( + name=rel_spec.name, + container=container, + relates=relates, + description=rel_spec.description, + verbalizes=list(rel_spec.verbalizes) if rel_spec.verbalizes else None, + multiplicity=multiplicity, + ) + ontology.add_relationship(relationship) + + # ----- Logical model ------------------------------------------------- + + @staticmethod + def _convert_semantic_model(lm_spec: SpecSemanticModel) -> SemanticModel: + semantic_model = SemanticModel( + name=lm_spec.name, + description=lm_spec.description, + ai_context=lm_spec.ai_context, + custom_extensions=[ + _convert_custom_extension(ce) for ce in lm_spec.custom_extensions + ], + ) + for ds_spec in lm_spec.datasets: + semantic_model.add_dataset(_convert_dataset(ds_spec)) + for jp_spec in lm_spec.relationships: + semantic_model.add_join_path(_convert_join_path(jp_spec, semantic_model)) + for m_spec in lm_spec.metrics: + semantic_model.add_metric(_convert_metric(m_spec)) + return semantic_model + + # ----- Ontology mapping --------------------------------------------- + + @staticmethod + def _convert_ontology_mapping(model: OsiOntology, om_spec: SpecOntologyMapping) -> None: + ontology = model.ontology + + semantic_model = SpecToOsiConverter._convert_semantic_model(om_spec.semantic_model) + + mapping = OntologyMapping( + name=om_spec.name, + ontology=ontology, + semantic_model=semantic_model, + description=om_spec.description, + ) + model.add_ontology_mapping(mapping) + + for cm_spec in om_spec.concept_mappings: + mapping.add_concept_mapping( + SpecToOsiConverter._convert_concept_mapping( + model, ontology, semantic_model, cm_spec + ) + ) + + @staticmethod + def _convert_concept_mapping( + model: OsiOntology, + ontology: OntologyComponent, + semantic_model: SemanticModel, + cm_spec: SpecConceptMapping, + ) -> ConceptMapping: + concept = ontology.lookup_concept(cm_spec.concept) + if concept is None: + raise ValueError( + f"ConceptMapping references unknown concept '{cm_spec.concept}' in ontology '{model.name}'." + ) + cm = ConceptMapping(concept=concept) + for object_mapping_spec in cm_spec.object_mappings: + cm.object_mappings.append( + SpecToOsiConverter._convert_object_mapping( + model, ontology, semantic_model, concept, object_mapping_spec + ) + ) + for link_mapping_spec in cm_spec.link_mappings: + cm.link_mappings.append( + SpecToOsiConverter._convert_link_mapping( + model, ontology, semantic_model, concept, link_mapping_spec + ) + ) + return cm + + @staticmethod + def _convert_object_mapping( + model: OsiOntology, + ontology: OntologyComponent, + semantic_model: SemanticModel, + container: Concept, + om_spec: SpecObjectMapping, + ) -> ObjectMapping: + concept: Concept | None = None + if om_spec.concept: + concept = ontology.lookup_concept(om_spec.concept) + if concept is None: + raise ValueError( + f"ObjectMapping references unknown concept '{om_spec.concept}' in ontology " + f"'{model.name}'." + ) + expression: DatasetField | Formula | None = None + if om_spec.expression is not None: + expression = _resolve_mapping_expression(om_spec.expression, semantic_model, concept) + referent_mappings = None + if om_spec.referent_mappings is not None: + rm_container = concept if concept is not None else container + referent_mappings = [ + SpecToOsiConverter._convert_referent_mapping( + model, ontology, semantic_model, rm_container, rm + ) + for rm in om_spec.referent_mappings + ] + return ObjectMapping(concept=concept, expression=expression, referent_mappings=referent_mappings) + + @staticmethod + def _convert_referent_mapping( + model: OsiOntology, + ontology: OntologyComponent, + semantic_model: SemanticModel, + container: Concept, + rm_spec: SpecReferentMapping, + ) -> ReferentMapping: + rel = ontology.lookup_concept_relationship(container, rm_spec.relationship) + if rel is None: + raise ValueError( + f"ReferentMapping references unknown relationship " + f"'{container.name}.{rm_spec.relationship}' in ontology '{model.name}'." + ) + sibling_player = rel.last_role.player + expression: DatasetField | Formula | None = None + if rm_spec.expression is not None: + expression = _resolve_mapping_expression(rm_spec.expression, semantic_model, sibling_player) + nested = None + if rm_spec.referent_mappings is not None: + nested = [ + SpecToOsiConverter._convert_referent_mapping( + model, ontology, semantic_model, sibling_player, child + ) + for child in rm_spec.referent_mappings + ] + return ReferentMapping(relationship=rel, expression=expression, referent_mappings=nested) + + @staticmethod + def _convert_link_mapping( + model: OsiOntology, + ontology: OntologyComponent, + semantic_model: SemanticModel, + container: Concept, + lm_spec: SpecLinkMapping, + ) -> LinkMapping: + object_mapping = SpecToOsiConverter._convert_object_mapping( + model, ontology, semantic_model, container, lm_spec.object_mapping + ) + relationship: Relationship | None = None + if lm_spec.relationship is not None: + relationship = ontology.lookup_concept_relationship(container, lm_spec.relationship) + if relationship is None: + raise ValueError( + f"LinkMapping references unknown relationship " + f"'{container.name}.{lm_spec.relationship}' in ontology '{model.name}'." + ) + children: list[LinkMapping] | None = None + if lm_spec.children is not None: + child_container = relationship.last_role.player if relationship is not None else container + children = [ + SpecToOsiConverter._convert_link_mapping( + model, ontology, semantic_model, child_container, child + ) + for child in lm_spec.children + ] + return LinkMapping(object_mapping=object_mapping, relationship=relationship, children=children) + + # ----- helpers ------------------------------------------------------- + + @staticmethod + def _sort_spec_dependency_graph(concepts: list[SpecConcept]) -> list[str]: + nodes = [concept.name for concept in concepts] + edges: list[tuple[str, str]] = [] + for concept in concepts: + if concept.extends: + for ext in concept.extends: + edges.append((ext, concept.name)) + return topological_sort(nodes, edges) + + +# --------------------------------------------------------------------------- +# Module-level helpers (logical-model + custom extensions) +# --------------------------------------------------------------------------- + +def _build_rule(raw: str | None, parent: Container) -> Formula | None: + if not raw: + return None + return Formula(raw_expr=raw, parent=parent) + + +def _resolve_mapping_expression( + expression: str, semantic_model: SemanticModel, expected_type: Concept | None +) -> DatasetField | Formula: + """Map a raw spec expression onto either a DatasetField (single + `DATASET.field` or bare `field` reference) or a Formula (anything else). + """ + qualified = _QUALIFIED_FIELD_RE.match(expression) + if qualified: + ds_name, field_name = qualified.group(1), qualified.group(2) + dataset = semantic_model.lookup_dataset(ds_name) + if dataset is not None: + field = dataset.field(field_name) + if field is not None: + _pin_field_type(field, expected_type) + return field + return Formula(raw_expr=expression) + + bare = _BARE_FIELD_RE.match(expression) + if bare: + field_name = bare.group(1) + for dataset in semantic_model.datasets: + field = dataset.field(field_name) + if field is not None: + _pin_field_type(field, expected_type) + return field + return Formula(raw_expr=expression) + + return Formula(raw_expr=expression) + + +def _pin_field_type(field: DatasetField, expected_type: Concept | None) -> None: + if expected_type is None: + return + if field.type is None: + field.type = expected_type + return + if field.type is not expected_type: + raise ValueError( + f"Field '{field.name}' is already mapped as concept " + f"'{field.type.name}' but this mapping expects " + f"'{expected_type.name}'. A dataset field can only be " + f"bound to one ontology concept type." + ) + + +def _convert_custom_extension(ce: SpecCustomExtension) -> CustomExtension: + return CustomExtension(vendor_name=ce.vendor_name, data=ce.data) + + +def _convert_expression(expr: SpecExpression) -> DialectExpressionSet: + return DialectExpressionSet( + dialects=[_convert_dialect_expression(d) for d in expr.dialects] + ) + + +def _convert_dialect_expression(dialect_expr: SpecDialectExpression) -> DialectExpression: + return DialectExpression(dialect=dialect_expr.dialect, expression=dialect_expr.expression) + + +def _convert_dimension(dim: SpecDimension | None) -> Dimension | None: + if dim is None: + return None + return Dimension(is_time=dim.is_time) + + +def _convert_dataset_field(fl: SpecDatasetField) -> DatasetField: + return DatasetField( + name=fl.name, + expression=_convert_expression(fl.expression), + dimension=_convert_dimension(fl.dimension), + label=fl.label, + description=fl.description, + ai_context=fl.ai_context, + custom_extensions=[_convert_custom_extension(ce) for ce in fl.custom_extensions], + ) + + +def _convert_dataset(ds: SpecDataset) -> Dataset: + fields = [_convert_dataset_field(fl) for fl in ds.fields] + return Dataset( + name=ds.name, + source=ds.source, + fields=fields, + primary_key=ds.primary_key, + unique_keys=ds.unique_keys, + description=ds.description, + ai_context=ds.ai_context, + custom_extensions=[_convert_custom_extension(ce) for ce in ds.custom_extensions], + ) + + +def _convert_join_path(jp: SpecJoinPath, lm: SemanticModel) -> JoinPath: + from_dataset = lm.lookup_dataset(jp.from_) + to_dataset = lm.lookup_dataset(jp.to) + if from_dataset is None: + raise ValueError(f"JoinPath '{jp.name}': unknown 'from' dataset '{jp.from_}'.") + if to_dataset is None: + raise ValueError(f"JoinPath '{jp.name}': unknown 'to' dataset '{jp.to}'.") + from_columns: list[DatasetField] = [] + for col in jp.from_columns: + field = from_dataset.field(col) + if field is None: + raise ValueError( + f"JoinPath '{jp.name}': column '{col}' not found in dataset '{from_dataset.name}'." + ) + from_columns.append(field) + to_columns: list[DatasetField] = [] + for col in jp.to_columns: + field = to_dataset.field(col) + if field is None: + raise ValueError( + f"JoinPath '{jp.name}': column '{col}' not found in dataset '{to_dataset.name}'." + ) + to_columns.append(field) + return JoinPath( + name=jp.name, + from_dataset=from_dataset, + to_dataset=to_dataset, + from_columns=from_columns, + to_columns=to_columns, + ai_context=jp.ai_context, + custom_extensions=[_convert_custom_extension(ce) for ce in jp.custom_extensions], + ) + + +def _convert_metric(m: SpecMetric) -> Metric: + return Metric( + name=m.name, + expression=_convert_expression(m.expression), + description=m.description, + ai_context=m.ai_context, + custom_extensions=[_convert_custom_extension(ce) for ce in m.custom_extensions], + ) \ No newline at end of file diff --git a/converters/ontology/src/osi/external/__init__.py b/converters/ontology/src/osi/external/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/ontology/src/osi/external/palantir/__init__.py b/converters/ontology/src/osi/external/palantir/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/ontology/src/osi/external/palantir/model.py b/converters/ontology/src/osi/external/palantir/model.py new file mode 100644 index 00000000..82a9b340 --- /dev/null +++ b/converters/ontology/src/osi/external/palantir/model.py @@ -0,0 +1,632 @@ +from __future__ import annotations + +from enum import Enum + + +class Status(Enum): + ACTIVE = 1 + DEPRECATED = 2 + EXPERIMENTAL = 3 + EXAMPLE = 4 + ENDORSED = 5 + INTERMEDIARY = 6 # This status had been introduced for testing purposes + +class DataType(Enum): + ANY = 1 + ATTACHMENT = 2 + BOOLEAN = 3 + DATE = 4 + DECIMAL = 5 + DOUBLE = 6 + FLOAT = 7 + GEOHASH = 8 + GEOPOINT = 9 + GEOSHAPE = 10 + INTEGER = 11 + LONG = 12 + SHORT = 13 + STRING = 14 + TIMESERIES = 15 + TIMESTAMP = 16 + TIME_DEPENDENT = 17 + STRUCT = 18 + VECTOR = 19 + MEDIA_REFERENCE = 20 + CIPHER_TEXT = 21 + + @staticmethod + def parse_datatype(name: str) -> DataType: + try: + return DataType[name.upper()] + except KeyError: + raise ValueError(f"Unrecognized data type: {name}") + + def to_type(self) -> str: + if self in (DataType.STRING, DataType.GEOHASH, DataType.GEOSHAPE, DataType.GEOPOINT, DataType.TIMESERIES): + return "String" + elif self == DataType.INTEGER: + return "Integer" + elif self == DataType.DECIMAL: + return "Decimal" + elif self in (DataType.FLOAT, DataType.DOUBLE, DataType.TIME_DEPENDENT): + return "Float" + elif self == DataType.BOOLEAN: + return "Boolean" + elif self == DataType.TIMESTAMP: + return "DateTime" + elif self == DataType.DATE: + return "Date" + else: + return "Integer" + + +class ArrayDataType: + + def __init__(self, t: DataType | ArrayDataType): + self._base_type = t + + def base_type(self): + return self._base_type + + def __str__(self): + return f"array[{str(self._base_type)}]" + +class Resource: + + def __init__(self, guid, rid): + self._guid = guid + self._readable_id = rid + self._status = Status.ACTIVE + + def active(self): + return self._status == Status.ACTIVE + + def experimental(self): + return self._status == Status.EXPERIMENTAL + + def endorsed(self): + return self._status == Status.ENDORSED + + def intermediary(self): + return self._status == Status.INTERMEDIARY + + def guid(self): + return self._guid + + def readable_id(self): + return self._readable_id + + def set_status(self, stat:Status): + self._status = stat + + def status(self): + return self._status + +class DataSetModel: + + _data_sets: dict[str, DataSet] + + def __init__(self): + self._data_sets = {} + + def data_sets(self): + return self._data_sets.values() + + def data_sets_map(self): + return self._data_sets + + def info(self) -> str: + result: list[str] = [] + for ds in self._data_sets.values(): + result.append(str(ds)) + return "\n".join(result) + +class DataSet(Resource): + + def __init__(self, guid, rid): + super().__init__(guid, rid) + self._description: str | None = None + self._path: str | None = None + self._columns: list[DataSetColumn] = [] + self._depends_on: list[DataSet] = [] + + def description(self): + return self._description + + def path(self): + return self._path + + def columns(self): + return self._columns + + def depends_on(self): + return self._depends_on + + def info(self, indent: int = 0, visited: set[str] | None = None) -> str: + """ + Pretty-print this dataset with indentation and handle dependency graph + (avoids infinite recursion on cycles by tracking visited mainDatasetIds). + """ + pad = " " * indent + if visited is None: + visited = set() + lines: list[str] = [] + ds_id = str(self._guid) if self._guid is not None else "None" + name = str(self._readable_id) if self._readable_id is not None else "None" + + header = f'{pad}DataSet(id="{ds_id}", name="{name}")' + lines.append(header) + if self._description: + lines.append(f"{pad} description: {self._description}") + if self._path: + lines.append(f"{pad} path: {self._path}") + + # Columns + if self._columns: + lines.append(f"{pad} columns:") + for col in self._columns: + lines.append(col.info(indent + 4)) + + # Dependencies + if self._depends_on: + lines.append(f"{pad} depends_on:") + if ds_id in visited: + lines.append(f"{pad} ") + else: + visited.add(ds_id) + for dep in self._depends_on: + # Safeguard when dep is None + if dep is None: + lines.append(f"{pad} ") + continue + lines.append(dep.info(indent + 4, visited)) + visited.remove(ds_id) + + return "\n".join(lines) + + def __str__(self) -> str: + return self.info() + +class DataSetColumn: + def __init__(self, name, t, ds: DataSet): + self._type = t + self._name = name + self._part_of = ds + + def name(self): + return self._name + + def type(self): + return self._type + + def part_of(self) -> DataSet: + return self._part_of + + def info(self, indent: int = 0) -> str: + return f'{" " * indent}Column(name="{self._name}", type="{self._type}", part_of="{self._part_of.readable_id()}")' + + def __str__(self) -> str: + return self.info() + +class Ontology: + + _data_sets: dict[str, DataSet] + _object_types: dict[str, ObjectType] + _object_types_by_readable_id: dict[str, ObjectType] + _relations: dict[str, Relation] + _relations_by_readable_id: dict[str, Relation] + _intermediary_relations: dict[str, IntermediaryRelation] + + def __init__(self): + self._data_sets = {} + self._object_types = {} + self._object_types_by_readable_id = {} + self._relations = {} + self._relations_by_readable_id = {} + self._intermediary_relations = {} + + def add_object_type(self, ot): + self._object_types[ot.guid()] = ot + self._object_types_by_readable_id[ot.readable_id()] = ot + return self + + def add_relation(self, rel): + self._relations[rel.guid()] = rel + self._relations_by_readable_id[rel.readable_id()] = rel + return self + + def object_types(self): + return self._object_types + + def object_type_by_readable_id(self, rid): + return self._object_types_by_readable_id[rid] + + def relations(self): + return self._relations + + def relation_by_readable_id(self, rid): + return self._relations_by_readable_id[rid] + + def intermediary_relations(self): + return self._intermediary_relations + + def data_sets(self): + return self._data_sets + + def set_data_sets(self, data_sets: dict[str, DataSet]): + self._data_sets = data_sets + + def info(self, indentation="") -> str: + result: list[str] = [] + + for ot in sorted(self._object_types.values(), key=lambda x: x.guid()): + result.append(ot.info()) + result.append("") + + for ds in sorted(self._data_sets.values(), key=lambda x: x.guid()): + ds_name = ds.readable_id() + result.append(indentation + f"Data set '{ds_name}':") + for col in sorted(ds.columns(), key=lambda x: x.name()): + result.append(indentation + f" Column '{col.name()}' of type '{col.type()}'") + result.append("") + + for rel in sorted(self._relations.values(), key=lambda x: x.guid()): + result.append(indentation + rel.info()) + result.append("") + + for ir in sorted(self._intermediary_relations.values(), key=lambda x: x.guid()): + result.append(indentation + ir.info()) + result.append("") + + return "\n".join(result) + + def subtypes_relations(self) -> dict[ObjectType, ManyToOneRelation]: + result: dict[ObjectType, ManyToOneRelation] = {} + for rel in self._relations.values(): + if not isinstance(rel, ManyToOneRelation): + continue + + rel_exp_eligible = ( + rel.experimental() + and rel.one_object_type().active() + and rel.many_object_type().active() + ) + if not (rel.active() or rel_exp_eligible): + continue + + one_ot = rel.one_object_type() + many_ot = rel.many_object_type() + + if not rel.property_map(): + continue + + is_subtype = all( + mprop in many_ot.primary_keys() and oprop in one_ot.primary_keys() + for mprop, oprop in rel.property_map().items() + ) + + if is_subtype: + result[one_ot] = rel + + return result + +# An ObjectType is Palantir's analog of an EntityType. Its instances are +# identified by its primary-key Properties, which appear in the JSON as +# ReadingIds in an array, e.g.: +# +# { +# "rid" : +# "primaryKeys" : [ +# +# ... +# ] +# } +# +class ObjectType(Resource): + + def __init__(self, guid, rid, name): + super().__init__(guid, rid) + self._name = name + self._type_groups = [] + self._syncs_from = [] + self._properties = {} + self._pk_properties = set() + self._data_sources = [] + + def lookup_property_by_reading(self, pname): + for prop in self._properties.values(): + if prop.readable_id() == pname: + return prop + return None + + + def name(self): + return self._name + + def type_groups(self): + return self._type_groups + + def syncs_from(self): + if not self._syncs_from: + raise RuntimeError(f"Mandatory constraint violation: ObjectType '{self.readable_id()}' must sync with some DataSet") + return self._syncs_from + + def sync_from_data_set(self, ds): + self._syncs_from.append(ds) + return self + + def properties(self): + return self._properties + + def primary_keys(self): + return self._pk_properties + + def data_sources(self) -> list[DataSource]: + if not self._data_sources: + raise RuntimeError(f"Mandatory constraint violation: ObjectType '{self.readable_id()}' must have some data source") + return self._data_sources + + def set_properties(self, properties): + self._properties = properties + + def set_primary_keys(self, pk_properties): + self._pk_properties = pk_properties + + def info(self, indent: int = 0) -> str: + keys = ", ".join([prop.readable_id() for prop in self._pk_properties]) + result: list[str] = [f'{" " * indent}Object type "{self._name}({keys})":'] + if self._syncs_from: + for ds in self._syncs_from: + result.append(f'{" " * (indent + 4)}Syncs from "{ds.readable_id()}"') + if self._properties: + for p in self._properties.values(): + result.append(p.info(indent + 4)) + if self._type_groups: + result.append(f'{" " * (indent + 4)}Belongs to type groups:') + for tg in self._type_groups: + result.append(f'{" " * (indent + 8)} "{tg}"') + return "\n".join(result) + + def __str__(self) -> str: + return self.info() + +class DataSource: + def __init__(self, backing_dataset_id, backing_datasource_id): + self._backing_dataset_id = backing_dataset_id + self._backing_datasource_id = backing_datasource_id + + def backing_dataset_id(self): + return self._backing_dataset_id + + def backing_datasource_id(self): + return self._backing_datasource_id + +class Property(Resource): + + def __init__(self, guid, rid, t, ot: ObjectType, column_name, datasource_resource_id): + super().__init__(guid, rid) + self._part_of = ot + self._type = t + self._column_name = column_name + self._datasource_resource_id = datasource_resource_id + self._pk_mapping = {} + + def part_of(self) -> ObjectType: + return self._part_of + + def type(self): + return self._type + + def column_name(self): + return self._column_name + + def datasource_resource_id(self): + return self._datasource_resource_id + + def pk_mapping(self): + return self._pk_mapping + + def info(self, indent: int = 0) -> str: + result = f'{" " * indent}Property "{self.readable_id()}" has data type "{str(self._type)}"' + if self._column_name and not self._pk_mapping: + result += f'\n{" " * (indent + 4)}Refers to "{str(self._column_name)}" of "{self._datasource_resource_id}" dataset' + if self._pk_mapping: + for k,v in self._pk_mapping.items(): + result += f'\n{" " * (indent + 4)}Maps to primary key column "{v}" of "{k}" dataset' + return result + + def __str__(self) -> str: + return self.info() + +# In Palantir, a Relation is a binary relation whose roles are played +# by ObjectTypes rather than DataTypes -- i.e., entity types rather +# than value types. They come in two forms: ManyToOne and ManyToMany. +# +# Palantir does not model roles or constraints directly. Instead, Relations +# represent roles using Properties of the ObjectTypes that play the role. +# +class Relation(Resource): + + def __init__(self, guid, rid): + super().__init__(guid, rid) + + def info(self) -> str: + return "" + +# A ManyToOneRelation is a binary relation with a uniqueness constraint that spans +# the "many" role. These objects are populated from a JSON message that looks like this: +# +# { +# "definition": { +# "type" : "oneToMany", +# "oneToMany" : { +# "objectTypeRidOneSide" : , +# "objectTypeIdOneSide" : , +# ... +# }, +# "objectTypeIdManySide" : , +# "objectTypeRidManySide" : , +# "oneSidePrimaryKeyToManySidePropertyMapping" : { +# : , // one-object-property -> many-object-fk-property +# ... +# : , // one-object-property -> many-object-fk-property +# }, +# "rid" : +# } +# } +# +# Consider the conceptual relationship "Subscription is part of Account" with a UC on +# the Subscription role. Subscription then plays the "many" role, and Account plays the +# "one" role. The resource ids for the role players can be found using these paths: +# +# - for Subscription, and +# - for Account +# +# Such relations are implemented using one or more Properties of the ObjectType that +# plays the many role. Each of these Properties is interpreted as a foreign-key +# reference to a Property of the ObjectType that plays the one role. Because an ObjectType +# might have a compound key, there will be as many properties in the ObjectType that +# plays the many role as there are key properties in the ObjectType that plays the one +# role. And while we might naturally think about representing the correspondence between +# FK properties of the "many" object type to properties of the "one" object type, for +# some reason Palantir represents this in the reverse direction, which is equivalent, +# just weird. This is captured in the "oneSidePrimaryKeyToManySidePropertyMapping" +# message. +# +class ManyToOneRelation(Relation): + + def __init__(self, guid, rid, many_object_type: ObjectType, one_object_type: ObjectType, + property_map: dict[Property, Property]): + super().__init__(guid, rid) + self._one_object_type = one_object_type + self._many_object_type = many_object_type + # We map the property from the many object type to the property of the one object type + self._property_map = property_map + + def info(self) -> str: + one_role = self._one_object_type + many_role = self._many_object_type + return f'Relation "{self.readable_id()}" maps "{many_role._name}" to "{one_role._name}"' + + def many_object_type(self): + return self._many_object_type + + def one_object_type(self): + return self._one_object_type + + def property_map(self): + return self._property_map + +# A ManyToManyRelation is a binary relation with a uniqueness constraint that spans +# both of its roles. These are populated from a JSON message that looks like this: +# +# { +# "type": "manyToMany", +# "objectTypeRidA" : , +# "objectTypeRidB" : , +# "objectTypeIdA" : , +# "objectTypeIdB" : , +# "objectTypeAPrimaryKeyPropertyMapping" : { +# : , // a-object-property -> join-table-property +# ... +# }, +# "objectTypeBPrimaryKeyPropertyMapping" : { +# : , // b-object-property -> join-table-property +# ... +# }, +# "joinTableDataSource": { +# "backingResourceRid" : , +# "datasourceRid" : +# } +# } +# +class ManyToManyRelation(Relation): + def __init__(self, guid, rid, role_a_object_type: ObjectType, role_b_object_type: ObjectType, + role_a_property_map: dict[Property, str], role_b_property_map: dict[Property, str]): + super().__init__(guid, rid) + self._role_a_object_type = role_a_object_type + self._role_b_object_type = role_b_object_type + # + # We map properties from each of the two roles' object types to the id of a property + # of the backing resource. Notice that we map to rather than Property + # because the JSON form we are using may not record property information about the + # backing resource. + # + self._role_a_property_map = role_a_property_map + self._role_b_property_map = role_b_property_map + # + self._backing_dataset_id = None + self._backing_datasource_id = None + # + self._data_set: DataSet | None = None + + def role_a_player(self): + return self._role_a_object_type + + def role_b_player(self): + return self._role_b_object_type + + def role_a_property_map(self): + return self._role_a_property_map + + def role_b_property_map(self): + return self._role_b_property_map + + def backing_dataset_id(self): + if self._backing_dataset_id is None: + raise RuntimeError(f"Mandatory constraint violation: ManyToManyRelation '{self.readable_id()}' must name a backing dataset resource-id") + return self._backing_dataset_id + + def backing_datasource_id(self): + if self._backing_datasource_id is None: + raise RuntimeError(f"Mandatory constraint violation: ManyToManyRelation '{self.readable_id()}' must name a backing datasource resource-id") + return self._backing_datasource_id + + def data_set(self): + if self._data_set is None: + raise RuntimeError(f"Mandatory constraint violation: ManyToManyRelation '{self.readable_id()}' must have some DataSet") + return self._data_set + + def set_backing_dataset_id(self, id): + self._backing_dataset_id = id + + def set_backing_datasource_id(self, id): + self._backing_datasource_id = id + + def info(self): + result = [f'Relation "{self.readable_id()}" associates "{self.role_a_player()._name}" with "{self.role_b_player()._name}"'] + if self._data_set: + result.append(f' DataSet "{self.data_set().readable_id()}"') + return "\n".join(result) + + +class IntermediaryRelation(Relation): + def __init__(self, guid, rid, role_a_object_type: ObjectType, role_b_object_type: ObjectType, + intermediary_object_type: ObjectType, relation_a_rid: str, relation_b_rid: str): + super().__init__(guid, rid) + self._role_a_object_type = role_a_object_type + self._role_b_object_type = role_b_object_type + self._intermediary_object_type = intermediary_object_type + self._relation_a = relation_a_rid + self._relation_b = relation_b_rid + + def role_a_player(self): + return self._role_a_object_type + + def role_b_player(self): + return self._role_b_object_type + + def intermediary_player(self): + return self._intermediary_object_type + + def relation_a(self): + return self._relation_a + + def relation_b(self): + return self._relation_b + + def info(self): + return (f'Relation "{self.readable_id()}" associates "{self.role_a_player()._name}" with ' + f'"{self.role_b_player()._name} via intermediary player "{self.intermediary_player()._name}" and ' + f'relations "{self.relation_a()}" and "{self.relation_b()}"') + diff --git a/converters/ontology/src/osi/external/palantir/parser/__init__.py b/converters/ontology/src/osi/external/palantir/parser/__init__.py new file mode 100644 index 00000000..30b31d57 --- /dev/null +++ b/converters/ontology/src/osi/external/palantir/parser/__init__.py @@ -0,0 +1,680 @@ +import io +import json +import warnings +import zipfile +from io import IOBase +from typing import Any + +from osi.common.utils import camel_to_snake +from osi.external.palantir.model import DataSet, DataSetColumn, DataSetModel, ObjectType, Ontology, DataType, \ + ArrayDataType, Property, Status, ManyToOneRelation, Relation, ManyToManyRelation, IntermediaryRelation, DataSource +from osi.common.file_utils import iter_json_files_from_dir_in_zip, open_top_level_file_from_zip + + +# Helper functions to aid in parsing. Palantir's JSON exports can be inconsistent in their formatting, especially +# across versions. For example, some fields that are expected to be strings may sometimes be empty strings or +# missing entirely, and some fields that are expected to be lists may sometimes be singletons or missing entirely. +# These helper functions normalize these inconsistencies to make parsing easier. + +def norm(v: Any) -> str | None: + if not isinstance(v, str): + return None + return v if v.strip() else None + +def set_if_value(curr: str | None, new_val: str | None) -> str | None: + # Only set when new_val is not None (i.e., not empty string or missing) + return new_val if new_val is not None else curr + +def get_dict(d, key): + v = d.get(key) + return v if isinstance(v, dict) else {} + +def get_list(d, key): + v = d.get(key) + return v if isinstance(v, list) else [] + +# DataSets in Palantir have their own JSON format that is separate from the Ontology JSON format. +class PalantirDataSetParser: + + _model: DataSetModel + + def __init__(self): + self._model = DataSetModel() + + def model(self): + return self._model + + def _dataset_from_dict(self, d: dict[str, Any], registry: dict[str, DataSet]) -> DataSet | None: + ds_id = norm(d.get("mainDatasetId")) + + if not ds_id: + return None + + # Reuse or create instance; do not return early to ensure children are populated + ds = registry.get(ds_id, None) + if ds is None: + ds = DataSet(ds_id, norm(d.get("datasetName"))) + registry[ds_id] = ds + + # Scalars without overriding with empty values + ds._path = set_if_value(ds.path(), norm(d.get("datasetPath"))) + ds._readable_id = set_if_value(ds.readable_id(), norm(d.get("datasetName"))) + ds._description = set_if_value(ds.description(), norm(d.get("description"))) + + # Columns + ds_schema = d.get("datasetSchema") + if isinstance(ds_schema, list): + cols: list[DataSetColumn] = [] + for item in ds_schema: + if isinstance(item, dict): + cols.append(DataSetColumn(item.get("name"), item.get("type"), ds)) + ds._columns = cols # only when provided as a proper list + + # Dependencies + raw_inputs = d.get("inputDatasetIds") + if isinstance(raw_inputs, list): + inputs: list[DataSet] = [] + for item in raw_inputs: + if not isinstance(item, dict): + continue + child = self._dataset_from_dict(item, registry) + if child is not None: + inputs.append(child) + ds._depends_on = inputs + + return ds + + def parse(self, file: IOBase): + data = json.load(file) + + if not isinstance(data, list): + raise ValueError("Top-level JSON must be an array of datasets") + + registry: dict[str, DataSet] = {} + for item in data: + if isinstance(item, dict): + ds = self._dataset_from_dict(item, registry) + if ds: + self.model().data_sets_map()[ds.guid()] = ds + +# +# The constructs declared within a Palantir ontology refer to one another using one +# or both of two different reference schemes: +# - Resource ids, which are essentially GUIDs, and +# - Readable ids, which are human-readable strings that are not guaranteed to be unique +# but are more stable across versions and easier to work with. +# The parser extracts both forms of identifiers for each construct and builds lookup maps +# keyed by both forms of identifier to make it easier to resolve references regardless of +# which form they use. In general, resource ids correspond to the 'rid' JSON key, while +# readable ids correspond to the 'id' or 'apiName' JSON keys. +# +class PalantirOntologyParser: + _model: Ontology + + def __init__(self): + self._model = Ontology() + + def model(self): + return self._model + + def parse(self, file: IOBase): + data = json.load(file) + + if not isinstance(data, dict): + raise ValueError("Top-level JSON must be a dictionary of Ontology data") + + # Object Types + object_types, object_types_by_readable_id = self._parse_object_types(data) + self._model._object_types = object_types + self._model._object_types_by_readable_id = object_types_by_readable_id + + # ManyToOneRelations + self._model._relations, self._model._intermediary_relations = self._parse_relations(data, object_types) + self.validate_intermediary_relations() + + self._parse_extra(data) + + # Given a Raw Palantir ObjectType, extract the string to use as its name regardless + # of JSON convention + def _parse_object_type_name(self, raw_ot): + # Newer JSONs contain a displayMetadata section with this information + display_metadata = get_dict(raw_ot, "displayMetadata") + if display_metadata: + ot_name = norm(display_metadata.get("displayName")) + else: + ot_name = norm(raw_ot.get("displayName")) + + if ot_name is None: + raise ValueError(f'Could not extract a name from ObjectType with rid: {raw_ot.get("rid")}') + return ot_name + + def _parse_property_backing_data(self, raw_prop, property_id, object_type_id): + # In the new exports this information stores in the `source` field, but in the old exports + # it leaves in the `column` and `datasourceRid` fields. We need to support both cases. + # Really old format doesn't even have column/datasource info - so use property name as column + source = get_dict(raw_prop, "source") + column_name = norm(source.get("columnName")) or norm(raw_prop.get("column")) or property_id + backing_datasource_id = norm(source.get("datasourceBackingResourceRid")) or norm(raw_prop.get("datasourceRid")) or object_type_id + + return (column_name, backing_datasource_id) + + def _parse_object_types(self, data: dict) -> tuple[dict[str, ObjectType], dict[str, ObjectType]]: + object_types = {} + object_types_by_readable_id = {} + for raw_ot in get_list(data, "objectTypes"): + guid = norm(raw_ot.get("rid")) + if not guid: + raise ValueError("Object type `rid` field must be non-empty") + # Support both formats: new (id) and old (apiName) + readable_id = norm(raw_ot.get("id")) or norm(raw_ot.get("apiName")) + + # Extract the ObjectType's name + ot_name = self._parse_object_type_name(raw_ot) + + object_type = ObjectType(guid, readable_id, ot_name) + + object_type._type_groups = get_list(raw_ot, "typeGroups") + + status_message = get_dict(raw_ot, "status") + if status_message: + object_type.set_status(self._get_status(norm(status_message.get("type")))) + + object_types[guid] = object_type + object_types_by_readable_id[readable_id] = object_type + + data_sources = get_list(raw_ot, "datasources") + if len(data_sources) < 1: + # No backing datasource? This is common in old versions of the JSON. + # Then create one that uses the same identifier as the ObjectType it backs. + object_type._data_sources.append(DataSource(readable_id, readable_id)) + else: + for data_source in data_sources: + datasource_rid = norm(data_source.get("datasourceRid")) + backing_resource_rid = norm(data_source.get("backingResourceRid")) + + if not datasource_rid or not backing_resource_rid: + raise ValueError("Object type fields `datasourceRid` and `backingResourceRid` must be non-empty") + + object_type._data_sources.append(DataSource(backing_resource_rid, datasource_rid)) + + + properties = {} + properties_by_readable_id = {} + + # Support both formats: list (new) and dict (old) + raw_properties = raw_ot.get("properties", []) + if isinstance(raw_properties, dict): + # Old format: properties is a dict keyed by property name + raw_properties = list(raw_properties.values()) + + for raw_prop in raw_properties: + # Parse type (supports nested arrays) + # Old format uses 'dataType', new uses 'baseType' + raw_base_type = get_dict(raw_prop, "baseType") or get_dict(raw_prop, "dataType") + prop_type = self._parse_datatype_node(raw_base_type) + + # Support both formats: new (id) and old (apiName) + prop_id = norm(raw_prop.get("id")) or norm(raw_prop.get("apiName")) + prop_guid = norm(raw_prop.get("rid")) + if not prop_guid or not prop_id: + warnings.warn(f"Skipping property with missing id/rid in object type '{ot_name}'") + continue + + (column_name, backing_datasource_id) = self._parse_property_backing_data(raw_prop, prop_id, guid) + + prop_name = prop_id + prop = Property(prop_guid, prop_name, prop_type, object_type, column_name, backing_datasource_id) + + status_message = get_dict(raw_prop, "status") + if status_message: + prop.set_status(self._get_status(norm(status_message.get("type")))) + + # This information exists only in the latest exports + primary_key_mapping = get_dict(raw_prop, "primaryKeyMapping") + if primary_key_mapping: + pk_mapping = {} + for k,v in primary_key_mapping.items(): + pk_column_name = norm(v.get("columnName")) + pk_mapping[k] = pk_column_name + prop._pk_mapping = pk_mapping + + properties[prop_guid] = prop + properties_by_readable_id[prop_name] = prop + + object_type._properties = properties + + pk_properties = set() + # Support both formats: primaryKeys (list) and primaryKey (string) + pk_list = get_list(raw_ot, "primaryKeys") + if not pk_list: + single_pk = norm(raw_ot.get("primaryKey")) + if single_pk: + pk_list = [single_pk] + for raw_pk_prop in pk_list: + pk_property = properties_by_readable_id.get(raw_pk_prop, None) + if pk_property is None: + warnings.warn(f"Property '{raw_pk_prop}' is not defined in object type '{ot_name}' - skipping as primary key") + continue + pk_properties.add(pk_property) + + object_type._pk_properties = pk_properties + + return object_types, object_types_by_readable_id + + + def _parse_raw_relation_id(self, raw_relation): + return norm(raw_relation.get("id")) or norm(raw_relation.get("apiName")) + + def _parse_raw_relation_guid(self, raw_relation): + return norm(raw_relation.get("rid")) or norm(raw_relation.get("linkTypeRid")) + + # Assumes raw_relation is a "MANY" relation and looks to make sure that it is + # an alternative reading of a "ONE" relation + def _verify_alternative_reading_of(self, raw_relation, all_relations): + id = raw_relation.get("linkTypeRid") + for r in all_relations: + if r.get("cardinality") == 'ONE': + if r.get("linkTypeRid") == id: + return True + return False + + def _parse_source_and_target(self, raw_relation, object_types): + # In the old format, sourceObjectType/targetObjectType name the source and target + # using readings rather than guids. + source_ot = norm(raw_relation.get("sourceObjectType")) + target_ot = norm(raw_relation.get("targetObjectType")) + + # Look up object types by apiName (readable_id) + source_object_type = None + target_object_type = None + for ot in object_types.values(): + if ot.readable_id() == source_ot: + source_object_type = ot + if ot.readable_id() == target_ot: + target_object_type = ot + return (source_object_type, target_object_type) + + # The old style JSON format supports only ManyToOne relations and uses a simpler format. + def _parse_old_style_relation(self, raw_relation, object_types): + id = self._parse_raw_relation_id(raw_relation) + (source_object_type, target_object_type) = self._parse_source_and_target(raw_relation, object_types) + + if not source_object_type or not target_object_type: + warnings.warn(f"Skipping relation {self._parse_raw_relation_id(raw_relation)}: source or target object type not found") + return None + + # target_object_type must comprise exactly one primary-key property + if len(target_object_type.primary_keys()) == 1: + + # Choose the lone property from the set of target_object_type's primary key properties + target_object_pk_property = next(iter(target_object_type.primary_keys())) + + # Look up the name of the source property that is a foreign key reference + # to target_object_type's primary key property + source_property_name = norm(raw_relation.get("foreignKeyPropertyApiName")) + if source_property_name is not None: + fk_property = source_object_type.lookup_property_by_reading(source_property_name) + if fk_property is not None: + # Build property mapping from foreign key + property_map = { fk_property: target_object_pk_property } + guid = self._parse_raw_relation_guid(raw_relation) + return ManyToOneRelation(guid, id, source_object_type, target_object_type, property_map) + + warnings.warn(f"Skipping relation {id}: no foreign key mapping available.") + return None + + def _parse_relations(self, data: dict, object_types: dict[str, ObjectType]) -> tuple[dict[str, Relation], dict[str, IntermediaryRelation]]: + relations = {} + intermediary_relations = {} + + all_relations = get_list(data, "relations") + for raw_relation in all_relations: + # Support both formats: new (id/rid) and old (apiName/linkTypeRid) + relation_id = self._parse_raw_relation_id(raw_relation) + relation_guid = self._parse_raw_relation_guid(raw_relation) + + if not relation_guid or not relation_id: + # Skip relations without proper identifiers (can happen with SDK-extracted ontologies) + warnings.warn(f"Skipping relation with missing id/rid: {raw_relation.get('apiName', 'unknown')}") + continue + + relation_type = None + definition = get_dict(raw_relation, "definition") + if definition: + relation_type = norm(definition.get("type")) + + relation: Relation | None = None + + if not definition: + cardinality = norm(raw_relation.get("cardinality")) + if cardinality == 'MANY': + if not self._verify_alternative_reading_of(raw_relation, all_relations): + warnings.warn(f'Encountered an unsupported ManyToMany relation {relation_id}') + continue + + # Otherwise, assume the cardinality is "ONE" + relation = self._parse_old_style_relation(raw_relation, object_types) + if relation is None: + continue + + elif relation_type and relation_type.lower() == "onetomany": + one_to_many_dict = get_dict(definition, "oneToMany") + relation = self._parse_many_to_one_relation(relation_guid, relation_id, one_to_many_dict, object_types) + elif relation_type and relation_type.lower() == "intermediary": + intermediary_dict = get_dict(definition, "intermediary") + relation = self._parse_intermediary_relation(relation_guid, relation_id, intermediary_dict, object_types) + else: + many_to_many_dict = get_dict(definition, "manyToMany") + relation = self._parse_many_to_many_relation(relation_guid, relation_id, many_to_many_dict, object_types) + + status_message = get_dict(raw_relation, "status") + if status_message: + relation.set_status(self._get_status(norm(status_message.get("type")))) + + if isinstance(relation, IntermediaryRelation): + intermediary_relations[relation_guid] = relation + else: + relations[relation_guid] = relation + + return relations, intermediary_relations + + def validate_intermediary_relations(self): + for r in self._model.intermediary_relations().values(): + # Validate that the intermediary relation's link types exist + if r.relation_a() not in self._model.relations().keys(): + raise ValueError( + f"Relation with rid {r.relation_a()} is not defined for intermediary relation {r.guid()}") + if r.relation_b() not in self._model.relations().keys(): + raise ValueError( + f"Relation with rid {r.relation_b()} is not defined for intermediary relation {r.guid()}") + + + def _parse_many_to_one_relation(self, guid: str, id: str, raw: dict[Any, Any], object_types: dict[str, ObjectType]) -> Relation: + + one_object_type_rid = norm(raw.get("objectTypeRidOneSide")) + many_object_type_rid = norm(raw.get("objectTypeRidManySide")) + if not one_object_type_rid or not many_object_type_rid: + raise ValueError("ManyToOne relation is missing objectTypeRid fields") + + try: + one_object_type = object_types[one_object_type_rid] + many_object_type = object_types[many_object_type_rid] + except KeyError as e: + raise ValueError(f"Object type {e.args[0]} is not defined") from None + + one_to_many_mapping = get_dict(raw, "oneSidePrimaryKeyToManySidePropertyMapping") + if not one_to_many_mapping: + raise ValueError("Relation definition must contain `oneSidePrimaryKeyToManySidePropertyMapping`") + + property_map: dict[Property, Property] = {} + for k, v in one_to_many_mapping.items(): + try: + one_property = one_object_type.properties()[k] + many_property = many_object_type.properties()[v] + except KeyError as e: + raise ValueError(f"Property {e.args[0]} is not defined in object type {e.args[1]}") from None + + property_map[many_property] = one_property + + return ManyToOneRelation(guid, id, many_object_type, one_object_type, property_map) + + def _parse_many_to_many_relation(self, guid: str, id: str, raw: dict[Any, Any], object_types: dict[str, ObjectType]) -> Relation: + + role_a_object_type_rid = norm(raw.get("objectTypeRidA")) + role_b_object_type_rid = norm(raw.get("objectTypeRidB")) + if not role_a_object_type_rid or not role_b_object_type_rid: + raise ValueError("ManyToMany relation is missing objectTypeRid fields") + + try: + role_a_object_type = object_types[role_a_object_type_rid] + role_b_object_type = object_types[role_b_object_type_rid] + except KeyError as e: + raise ValueError(f"Object type {e.args[0]} is not defined") from None + + def build_property_map(object_type, pk_mapping: dict[str, str]) -> dict[Property, str]: + prop_map: dict[Property, str] = {} + for src_prop_id, dst_prop_id in pk_mapping.items(): + try: + obj_prop = object_type.properties()[src_prop_id] + except KeyError as e: + raise ValueError(f"Property {e.args[0]} is not defined in object type {e.args[1]}") from None + prop_map[obj_prop] = dst_prop_id + return prop_map + + role_a_pk_mapping = get_dict(raw, "objectTypeAPrimaryKeyPropertyMapping") + if not role_a_pk_mapping: + raise ValueError("Relation definition must contain `objectTypeAPrimaryKeyPropertyMapping`") + + role_a_property_map: dict[Property, str] = build_property_map(role_a_object_type, role_a_pk_mapping) + + role_b_pk_mapping = get_dict(raw, "objectTypeBPrimaryKeyPropertyMapping") + if not role_b_pk_mapping: + raise ValueError("Relation definition must contain `objectTypeBPrimaryKeyPropertyMapping`") + + role_b_property_map: dict[Property, str] = build_property_map(role_b_object_type, role_b_pk_mapping) + + relation = ManyToManyRelation(guid, id, role_a_object_type, role_b_object_type, role_a_property_map, + role_b_property_map) + + join_table_data_source = get_list(raw, "joinTableDatasource") + if len(join_table_data_source) != 1: + raise ValueError("Relation definition must contain exactly one `joinTableDatasource`") + + datasource_rid = norm(join_table_data_source[0].get("datasourceRid")) + backing_resource_rid = norm(join_table_data_source[0].get("backingResourceRid")) + + if not datasource_rid or not backing_resource_rid: + raise ValueError("Relation fields `datasourceRid` and `backingResourceRid` must be non-empty") + + relation.set_backing_datasource_id(datasource_rid) + relation.set_backing_dataset_id(backing_resource_rid) + + return relation + + def _parse_intermediary_relation(self, guid: str, id: str, raw: dict[Any, Any], object_types: dict[str, ObjectType]) -> Relation: + role_a_object_type_rid = norm(raw.get("objectTypeRidA")) + role_b_object_type_rid = norm(raw.get("objectTypeRidB")) + intermediary_rid = norm(raw.get("intermediaryObjectTypeRid")) + if not role_a_object_type_rid or not role_b_object_type_rid or not intermediary_rid: + raise ValueError("Intermediary relation is missing objectTypeRid fields") + + try: + role_a_object_type = object_types[role_a_object_type_rid] + role_b_object_type = object_types[role_b_object_type_rid] + intermediary_object_type = object_types[intermediary_rid] + except KeyError as e: + raise ValueError(f"Object type {e.args[0]} is not defined") from None + + a_to_intermediary_link_rid = norm(raw.get("aToIntermediaryLinkTypeRid")) + intermediary_to_b_link_rid = norm(raw.get("intermediaryToBLinkTypeRid")) + if not a_to_intermediary_link_rid or not intermediary_to_b_link_rid: + raise ValueError("Intermediary relation is missing link type rid fields") + + return IntermediaryRelation(guid, id, role_a_object_type, role_b_object_type, intermediary_object_type, + a_to_intermediary_link_rid, intermediary_to_b_link_rid) + + def _get_status(self, status): + match status: + case "active": + return Status.ACTIVE + case "deprecated": + return Status.DEPRECATED + case "experimental": + return Status.EXPERIMENTAL + case "example": + return Status.EXAMPLE + case "endorsed": + return Status.ENDORSED + # This status had been introduced for testing purposes + case "intermediary": + return Status.INTERMEDIARY + case _: + raise ValueError(f"Unrecognized Resource status {status}") + + def _parse_datatype_node(self, node) -> DataType | ArrayDataType: + """ + node: {"type": "...", "subType": {...}} possibly nested arrays + Returns DataType or ArrayDataType wrapping. + """ + + t = norm(node.get("type")) + if not t: + # Default to STRING for missing types + return DataType.STRING + + if t.upper() == "ARRAY": + # Support both camelCase (subType) and snake_case (sub_type) + sub = get_dict(node, "subType") or get_dict(node, "sub_type") + if not sub: + # Default to STRING array if subType is missing + return ArrayDataType(DataType.STRING) + inner = self._parse_datatype_node(sub) + return ArrayDataType(inner) + + # Non-array primitive + return DataType.parse_datatype(t) + + + def _parse_extra(self, data: dict) -> None: + """Extension point for subclasses to parse additional data from the ontology JSON. + + Called at the end of :meth:`parse` with the fully-deserialized JSON dict. + The base implementation is a no-op; override in a subclass to add domain-specific + parsing (e.g. actions, workflows, validations) without re-reading the file. + """ + +class PalantirParser: + _model: Ontology + + def model(self): + if self._model is None: + raise RuntimeError("You must call 'parse()' first before calling 'model()'") + return self._model + + def _make_ontology_parser(self) -> PalantirOntologyParser: + return PalantirOntologyParser() + + def parse(self, file: IOBase): + raw = file.buffer if isinstance(file, io.TextIOWrapper) else file + + # Read all bytes and detect ZIP + data = raw.read() + bio = io.BytesIO(data) + if not zipfile.is_zipfile(bio): + raise ValueError("Unsupported archive format. Expected ZIP") + + bio.seek(0) + with zipfile.ZipFile(bio) as zf: + self._parse_from_zip(zf) + + def _parse_from_zip(self, zf: zipfile.ZipFile): + self._validate_archive(zf) + + any_json = False + data_sets: dict[str, DataSet] = {} + for name, fh in iter_json_files_from_dir_in_zip(zf, "data_sets"): + any_json = True + try: + parser = PalantirDataSetParser() + parser.parse(fh) + data_sets.update(parser.model().data_sets_map()) + finally: + fh.close() + if not any_json: + raise ValueError("'data_sets' folder contains no JSON files") + + try: + with open_top_level_file_from_zip(zf, self._get_ontology_json_file_path(zf)) as fh: + parser = self._make_ontology_parser() + parser.parse(fh) + model = parser.model() + model.set_data_sets(data_sets) + + for ot in model.object_types().values(): + for ds in ot.data_sources(): + data_set = data_sets.get(ds.backing_dataset_id(), None) + if data_set is None: + # For SDK-extracted ontologies with synthetic datasources, + # mainDatasetId in data_sets JSON matches the object type's RID + data_set = data_sets.get(ot.guid(), None) + if data_set: + ot.sync_from_data_set(data_set) + # For SDK-extracted ontologies, property column_name defaults to + # the apiName (camelCase), but dataset columns use snake_case. + # Cross-reference to use the actual dataset column names. + ds_col_names = {col.name() for col in data_set.columns()} + for prop in ot.properties().values(): + col_name = prop.column_name() + if col_name not in ds_col_names: + snake_name = camel_to_snake(col_name) + if snake_name in ds_col_names: + prop._column_name = snake_name + for rel in model.relations().values(): + if isinstance(rel, ManyToManyRelation): + rel._data_set = data_sets.get(rel.backing_dataset_id(), None) + + self._model = model + except FileNotFoundError as e: + raise FileNotFoundError(str(e)) from e + + def _validate_archive(self, zf: zipfile.ZipFile): + """ + Ensure the ZIP archive contains a required 'data_sets/' directory. + Accept either: + - Top-level 'data_sets/' folder, or + - A single-root folder with 'root/data_sets/' inside. + """ + names = zf.namelist() + + # Fast path: direct presence at top-level or files under 'data_sets/' + has_data_sets = any( + n.endswith("/") and n.rstrip("/").endswith("data_sets") for n in names + ) or any(n.startswith("data_sets/") for n in names) + if has_data_sets: + return + + # Single-root archives: if there's exactly one root folder, allow root/data_sets/ + roots = {n.split("/", 1)[0] for n in names if "/" in n} + if len(roots) == 1: + root = next(iter(roots)) + has_rooted_data_sets = any( + n.endswith("/") and n.rstrip("/").endswith(f"{root}/data_sets") for n in names + ) or any(n.startswith(f"{root}/data_sets/") for n in names) + if has_rooted_data_sets: + return + + raise ValueError("Archive does not contain required 'data_sets' folder") + + def _get_ontology_json_file_path(self, zf: zipfile.ZipFile) -> str: + """ + Find exactly one top-level JSON file and return its archive path. + + Rules: + - "Top-level" means entries without '/' in their name. + - If the archive is packaged under a single root directory, then + "top-level" means entries directly under that root (exactly one '/'). + - There must be exactly one JSON at this level; otherwise raise. + """ + names = zf.namelist() + + # Identify entries without any parent directory. + top_level = [n for n in names if "/" not in n] + + # If nothing is at the real top-level, accept the case of a single root folder. + if not top_level: + roots = {n.split("/", 1)[0] for n in names if "/" in n} + if len(roots) == 1: + root = next(iter(roots)) + # Entries directly under the single root (e.g., 'root/file.json') + top_level = [n for n in names if n.startswith(f"{root}/") and n.count("/") == 1] + + # Keep only JSON files + json_candidates = [n for n in top_level if n.lower().endswith(".json")] + + # Enforce exactly one ontology JSON at the top level + if len(json_candidates) == 0: + raise ValueError("Archive must contain exactly one top-level JSON file (none found)") + if len(json_candidates) > 1: + raise ValueError("Archive must contain exactly one top-level JSON file (multiple found)") + + return json_candidates[0] diff --git a/converters/ontology/src/osi/model.py b/converters/ontology/src/osi/model.py new file mode 100644 index 00000000..5b354920 --- /dev/null +++ b/converters/ontology/src/osi/model.py @@ -0,0 +1,1128 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Protocol + +# --------------------------------------------------------------------------- +# Builtin concept names. +# --------------------------------------------------------------------------- + +BUILTIN_CONCEPTS: frozenset[str] = frozenset({ + "Any", "AnyEntity", "Boolean", "Date", "DateTime", "Decimal", "Float", "Integer", "String" +}) + +# --------------------------------------------------------------------------- +# Free-form metadata mirroring spec +# --------------------------------------------------------------------------- + +AiContext = str | dict[str, Any] + + +@dataclass +class CustomExtension: + vendor_name: str + data: str + + +# --------------------------------------------------------------------------- +# Ontology (concepts + relationships grouped by container) +# --------------------------------------------------------------------------- + +class ConceptType(str, Enum): + ENTITY_TYPE = "EntityType" + VALUE_TYPE = "ValueType" + + @classmethod + def from_value(cls, value: str | None) -> ConceptType | None: + if value is None: + return None + if not isinstance(value, str): + raise TypeError("value must be a string") + for member in cls: + if member.value == value: + return member + raise ValueError(f"Unknown concept type: {value}") + + +class RelationshipMultiplicity(str, Enum): + """Spec-level multiplicity declared on a relationship. + + Allows OneToOne or ManyToOne (ManyToMany is no longer expressible + at the spec level — it becomes the default 'unconstrained' case). + """ + ONE_TO_ONE = "OneToOne" + MANY_TO_ONE = "ManyToOne" + + @classmethod + def from_value(cls, value: str | None) -> RelationshipMultiplicity | None: + if value is None: + return None + if not isinstance(value, str): + raise TypeError("value must be a string") + normalized = value.strip().lower() + for member in cls: + if member.value.lower() == normalized: + return member + raise ValueError(f"Unknown relationship multiplicity value: {value}") + + +class Concept: + """Type-like ontology node. May be an EntityType (real-world object, + referenced via identifying relationships) or a ValueType (primitive-ish, + transitively extending a built-in value type).""" + _name: str + _type: ConceptType | None + _description: str | None + _builtin: bool + _extends: list[Concept] + _identify_by: dict[str, Relationship] + _derived_by: list[Formula] + _requires: list[Formula] + + def __init__( + self, + name: str, + type: ConceptType | None = None, + description: str | None = None, + builtin: bool = False, + extends: list[Concept] | None = None, + identify_by: dict[str, Relationship] | None = None, + derived_by: list[Formula] | None = None, + requires: list[Formula] | None = None, + ): + self._name = name + self._type = type + self._description = description + self._builtin = builtin + self._extends = extends if extends else [] + self._identify_by = identify_by if identify_by else {} + self._derived_by = derived_by if derived_by else [] + self._requires = requires if requires else [] + + def add_require(self, require: Formula) -> None: + self._requires.append(require) + + def add_derived_by(self, rule: Formula) -> None: + self._derived_by.append(rule) + + def set_identify_by(self, identifiers: dict[str, Relationship]) -> None: + self._identify_by = identifiers + + def extend(self, parent: Concept) -> None: + self._extends.append(parent) + + @property + def name(self) -> str: + return self._name + + @property + def type(self) -> ConceptType | None: + return self._type + + @property + def description(self) -> str | None: + return self._description + + @property + def is_builtin(self) -> bool: + return self._builtin + + @property + def is_value_type(self) -> bool: + return self._type == ConceptType.VALUE_TYPE + + @property + def is_entity_type(self) -> bool: + return self._type == ConceptType.ENTITY_TYPE + + @property + def is_primitive(self) -> bool: + if self.is_builtin: + return True + if self._extends and len(self._extends) == 1: + return self._extends[0].is_primitive + return False + + @property + def is_derived(self) -> bool: + return bool(self._derived_by) + + @property + def extends(self) -> list[Concept]: + return list(self._extends) + + @property + def identify_by(self) -> dict[str, Relationship]: + return dict(self._identify_by) + + @property + def derived_by(self) -> list[Formula]: + return list(self._derived_by) + + @property + def requires(self) -> list[Formula]: + return list(self._requires) + + def __str__(self) -> str: + return self._name + + +class Relationship: + """A relationship grouped under its first-role concept (the container). + In this model class we choose to store all the roles explicitly, including the first implicit role from the OSI spec. + """ + _name: str + _container: Concept + _roles: tuple[Role, ...] + _description: str | None + _verbalizes_raw: list[str] | None + _verbalizations: list[RelationshipVerbalization] + _multiplicity: RelationshipMultiplicity | None + _derived_by: list[Formula] + _requires: list[Formula] + + def __init__( + self, + name: str, + container: Concept, + relates: list[tuple[Concept, str | None]], + description: str | None = None, + verbalizes: list[str] | None = None, + multiplicity: RelationshipMultiplicity | None = None, + ): + self._name = name + self._container = container + container_role = Role(self, container, 0, None) + additional = [Role(self, concept, idx + 1, role_name) for idx, (concept, role_name) in enumerate(relates)] + self._roles = tuple([container_role] + additional) + self._description = description + self._multiplicity = multiplicity + self._verbalizes_raw = list(verbalizes) if verbalizes else None + self._verbalizations = parse_verbalizations(self, verbalizes) + self._derived_by = [] + self._requires = [] + + @property + def name(self) -> str: + return self._name + + @property + def full_name(self) -> str: + return f"{self._container.name}.{self._name}" + + @property + def container(self) -> Concept: + return self._container + + @property + def description(self) -> str | None: + return self._description + + @property + def signature(self) -> list[Concept]: + return [role.player for role in self._roles] + + @property + def arity(self) -> int: + return len(self._roles) + + @property + def binary(self) -> bool: + return self.arity == 2 + + @property + def unary(self) -> bool: + return self.arity == 1 + + def role(self, pos: int | Concept | str) -> Role: + if isinstance(pos, int): + return self._roles[pos] + if isinstance(pos, Concept): + for role in self._roles: + if role.player == pos: + return role + elif isinstance(pos, str): + for role in self._roles: + if role.name == pos: + return role + raise ValueError(f"Role '{pos}' not found in relationship '{self.full_name}'") + + @property + def roles(self) -> tuple[Role, ...]: + return self._roles + + def set_multiplicity(self, mult: RelationshipMultiplicity) -> None: + if self._multiplicity is not None and self._multiplicity != mult: + raise ValueError( + f"Conflicting multiplicity settings for relationship {self}: " + f"{self._multiplicity} and {mult}" + ) + self._multiplicity = mult + + @property + def first_role(self) -> Role: + return self._roles[0] + + @property + def last_role(self) -> Role: + return self._roles[-1] + + @property + def verbalizations(self) -> list[RelationshipVerbalization]: + return self._verbalizations + + @property + def verbalizes_raw(self) -> list[str] | None: + return self._verbalizes_raw + + @property + def multiplicity(self) -> RelationshipMultiplicity | None: + return self._multiplicity + + @property + def derived_by(self) -> list[Formula]: + return list(self._derived_by) + + @property + def requires(self) -> list[Formula]: + return list(self._requires) + + def add_derived_by(self, rule: Formula) -> None: + self._derived_by.append(rule) + + def add_require(self, rule: Formula) -> None: + self._requires.append(rule) + + def __str__(self) -> str: + return self._name + + +class Role: + _part_of: Relationship + _player: Concept + _name: str | None + _sibling: Role | None + _idx: int + + def __init__(self, part_of: Relationship, player: Concept, idx: int, name: str | None = None): + self._part_of = part_of + self._player = player + self._idx = idx + self._name = name + self._sibling = None + + @property + def player(self) -> Concept: + return self._player + + @property + def idx(self) -> int: + return self._idx + + @property + def name(self) -> str: + return self._name or self._player.name + + @property + def explicit_name(self) -> str | None: + return self._name + + @property + def part_of(self) -> Relationship: + return self._part_of + + @property + def sibling(self) -> Role | None: + if self._part_of.binary and not self._sibling: + first_role, second_role = self._part_of.roles + sibling = second_role if self == first_role else first_role + self._sibling = sibling + return self._sibling + + @property + def madlib(self) -> str: + return f"{self._player.name}:{self._name}" if self._name else self._player.name + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Role): + return False + return ( + self._part_of == other._part_of + and self._player == other._player + and self._name == other._name + ) + + def __hash__(self) -> int: + return hash((self._part_of, self._player, self._name)) + + +# --------------------------------------------------------------------------- +# Formula — raw expression string only. +# --------------------------------------------------------------------------- + +FormulaParent = Concept | Relationship | tuple[Concept, Relationship] | None + + +class Formula: + _raw_expr: str + _parent: FormulaParent + + def __init__(self, raw_expr: str, parent: FormulaParent = None): + self._raw_expr = raw_expr + self._parent = parent + + @property + def raw_expr(self) -> str: + return self._raw_expr + + @property + def parent(self) -> FormulaParent: + return self._parent + + def __str__(self) -> str: + return self._raw_expr + + +# --------------------------------------------------------------------------- +# Semantic model (datasets, join paths, metrics) +# --------------------------------------------------------------------------- + +@dataclass +class DialectExpression: + dialect: str + expression: str + + +@dataclass +class DialectExpressionSet: + """Runtime equivalent of spec.Expression — same logical expression rendered + in one or more dialects.""" + dialects: list[DialectExpression] = field(default_factory=list) + + def by_dialect(self, dialect: str) -> DialectExpression | None: + for d in self.dialects: + if d.dialect == dialect: + return d + return None + + @property + def primary(self) -> DialectExpression | None: + return self.dialects[0] if self.dialects else None + + +@dataclass +class Dimension: + is_time: bool | None = None + + +@dataclass +class DatasetField: + name: str + expression: DialectExpressionSet + type: Concept | None = None + dimension: Dimension | None = None + label: str | None = None + description: str | None = None + ai_context: AiContext | None = None + custom_extensions: list[CustomExtension] = field(default_factory=list) + # Back-reference to the owning Dataset, wired by Dataset.__init__. Used by + # mapping-expression rendering to reconstruct `.` strings + # for round-trip output. Not in the spec — purely runtime metadata. + dataset: "Dataset | None" = field(default=None, repr=False, compare=False) + + def __str__(self) -> str: + return self.name + + +def sanitize_identifier(ref: str) -> str: + return re.sub(r"[^A-Za-z0-9_]", "_", ref) + + +class Dataset: + _name: str + _source: str + _primary_key: list[str] | None + _unique_keys: list[list[str]] | None + _description: str | None + _ai_context: AiContext | None + _fields: list[DatasetField] + _custom_extensions: list[CustomExtension] + _field_name_map: dict[str, DatasetField] + + def __init__( + self, + name: str, + source: str, + fields: list[DatasetField], + primary_key: list[str] | None = None, + unique_keys: list[list[str]] | None = None, + description: str | None = None, + ai_context: AiContext | None = None, + custom_extensions: list[CustomExtension] | None = None, + ): + self._name = name + self._source = source + self._fields = fields + self._primary_key = primary_key + self._unique_keys = unique_keys + self._description = description + self._ai_context = ai_context + self._custom_extensions = custom_extensions or [] + self._field_name_map = {fl.name: fl for fl in fields} + # Wire the back-reference so each field knows its owning Dataset — + # the mapping-expression renderer needs it to reconstruct + # `.` strings on reverse conversion. + for fl in fields: + fl.dataset = self + + @property + def name(self) -> str: + return self._name + + @property + def source(self) -> str: + return self._source + + @property + def primary_key(self) -> list[str] | None: + return self._primary_key + + @property + def unique_keys(self) -> list[list[str]] | None: + return self._unique_keys + + @property + def description(self) -> str | None: + return self._description + + @property + def ai_context(self) -> AiContext | None: + return self._ai_context + + def field(self, name: str) -> DatasetField | None: + return self._field_name_map.get(name) + + @property + def fields(self) -> list[DatasetField]: + return list(self._fields) + + @property + def custom_extensions(self) -> list[CustomExtension]: + return list(self._custom_extensions) + + @property + def schema(self) -> dict[str, Concept | None]: + return {fl.name: fl.type for fl in self._fields} + + def __str__(self) -> str: + return self._name + + +class JoinPath: + """Runtime equivalent of spec.JoinPath — a foreign-key style join + between two Datasets, matching `from_columns` against `to_columns`.""" + _name: str + _from_dataset: Dataset + _to_dataset: Dataset + _from_columns: list[DatasetField] + _to_columns: list[DatasetField] + _ai_context: AiContext | None + _custom_extensions: list[CustomExtension] + + def __init__( + self, + name: str, + from_dataset: Dataset, + to_dataset: Dataset, + from_columns: list[DatasetField], + to_columns: list[DatasetField], + ai_context: AiContext | None = None, + custom_extensions: list[CustomExtension] | None = None, + ): + if len(from_columns) != len(to_columns): + raise ValueError( + f"JoinPath '{name}': from_columns/to_columns arity mismatch " + f"({len(from_columns)} vs {len(to_columns)})" + ) + self._name = name + self._from_dataset = from_dataset + self._to_dataset = to_dataset + self._from_columns = from_columns + self._to_columns = to_columns + self._ai_context = ai_context + self._custom_extensions = custom_extensions or [] + + @property + def name(self) -> str: + return self._name + + @property + def from_dataset(self) -> Dataset: + return self._from_dataset + + @property + def to_dataset(self) -> Dataset: + return self._to_dataset + + @property + def from_columns(self) -> list[DatasetField]: + return list(self._from_columns) + + @property + def to_columns(self) -> list[DatasetField]: + return list(self._to_columns) + + @property + def ai_context(self) -> AiContext | None: + return self._ai_context + + @property + def custom_extensions(self) -> list[CustomExtension]: + return list(self._custom_extensions) + + def __str__(self) -> str: + return self._name + + +class Metric: + """Logical-model-level metric defined as a multi-dialect aggregate expression.""" + _name: str + _expression: DialectExpressionSet + _description: str | None + _ai_context: AiContext | None + _custom_extensions: list[CustomExtension] + + def __init__( + self, + name: str, + expression: DialectExpressionSet, + description: str | None = None, + ai_context: AiContext | None = None, + custom_extensions: list[CustomExtension] | None = None, + ): + self._name = name + self._expression = expression + self._description = description + self._ai_context = ai_context + self._custom_extensions = custom_extensions or [] + + @property + def name(self) -> str: + return self._name + + @property + def expression(self) -> DialectExpressionSet: + return self._expression + + @property + def description(self) -> str | None: + return self._description + + @property + def ai_context(self) -> AiContext | None: + return self._ai_context + + @property + def custom_extensions(self) -> list[CustomExtension]: + return list(self._custom_extensions) + + +class SemanticModel: + """Bundle of datasets, join paths and metrics. One or more SemanticModels + can feed a single OntologyMapping (see spec).""" + _name: str + _description: str | None + _ai_context: AiContext | None + _datasets: list[Dataset] + _join_paths: list[JoinPath] + _metrics: list[Metric] + _custom_extensions: list[CustomExtension] + _dataset_name_map: dict[str, Dataset] + _join_path_name_map: dict[str, JoinPath] + _metric_name_map: dict[str, Metric] + + def __init__( + self, + name: str, + description: str | None = None, + ai_context: AiContext | None = None, + custom_extensions: list[CustomExtension] | None = None, + ): + self._name = name + self._description = description + self._ai_context = ai_context + self._datasets = [] + self._join_paths = [] + self._metrics = [] + self._custom_extensions = custom_extensions or [] + self._dataset_name_map = {} + self._join_path_name_map = {} + self._metric_name_map = {} + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str | None: + return self._description + + @property + def ai_context(self) -> AiContext | None: + return self._ai_context + + @property + def datasets(self) -> list[Dataset]: + return list(self._datasets) + + @property + def join_paths(self) -> list[JoinPath]: + return list(self._join_paths) + + @property + def metrics(self) -> list[Metric]: + return list(self._metrics) + + @property + def custom_extensions(self) -> list[CustomExtension]: + return list(self._custom_extensions) + + def add_dataset(self, dataset: Dataset) -> None: + if dataset.name in self._dataset_name_map: + raise ValueError(f"Dataset '{dataset.name}' already exists in logical model '{self._name}'") + self._datasets.append(dataset) + self._dataset_name_map[dataset.name] = dataset + + def add_join_path(self, join_path: JoinPath) -> None: + if join_path.name in self._join_path_name_map: + raise ValueError(f"JoinPath '{join_path.name}' already exists in logical model '{self._name}'") + self._join_paths.append(join_path) + self._join_path_name_map[join_path.name] = join_path + + def add_metric(self, metric: Metric) -> None: + if metric.name in self._metric_name_map: + raise ValueError(f"Metric '{metric.name}' already exists in logical model '{self._name}'") + self._metrics.append(metric) + self._metric_name_map[metric.name] = metric + + def lookup_dataset(self, name: str) -> Dataset | None: + return self._dataset_name_map.get(name) + + def lookup_join_path(self, name: str) -> JoinPath | None: + return self._join_path_name_map.get(name) + + def lookup_metric(self, name: str) -> Metric | None: + return self._metric_name_map.get(name) + + +# --------------------------------------------------------------------------- +# Ontology mapping (tree-shaped) +# --------------------------------------------------------------------------- + +@dataclass +class ObjectMapping: + """Maps to objects of some concept — either a direct `expression` (value + types / simple-id entities) or `referent_mappings` (compound id). XOR — + never both. + + `expression` carries the *parsed* mapping expression: a `DatasetField` + when it resolves to a single field reference, or a `Formula` for richer + expressions. The forward converter parses the spec's raw string and the + reverse converter reconstructs it — storing the parsed form rather than the + raw string lets callers introspect the mapping target.""" + concept: Concept | None = None + expression: DatasetField | Formula | None = None + referent_mappings: list[ReferentMapping] | None = None + + def __post_init__(self) -> None: + has_expr = self.expression is not None + has_refs = self.referent_mappings is not None + if has_expr and has_refs: + raise ValueError("ObjectMapping must not have both expression and referent_mappings") + if not has_expr and not has_refs: + raise ValueError("ObjectMapping must have either expression or referent_mappings") + + +@dataclass +class ReferentMapping: + """Locates an entity object by walking one of its identifying relationships. + + `expression`, like ObjectMapping's, is the parsed result — a `DatasetField` + for simple references or a `Formula` for richer expressions. Nested + `referent_mappings` descend into compound identifiers.""" + relationship: Relationship + expression: DatasetField | Formula | None = None + referent_mappings: list[ReferentMapping] | None = None + + def __post_init__(self) -> None: + has_expr = self.expression is not None + has_refs = self.referent_mappings is not None + if has_expr and has_refs: + raise ValueError("ReferentMapping must not have both expression and referent_mappings") + if not has_expr and not has_refs: + raise ValueError("ReferentMapping must have either expression or referent_mappings") + + +@dataclass +class LinkMapping: + """A node in the link-mapping tree. The arity of `relationship` equals the + node's depth (top-level = unary, depth 2 = binary, ...). `children` extend + the mapped tuple by one role each, sharing this node's `object_mapping`.""" + object_mapping: ObjectMapping + relationship: Relationship | None = None + children: list[LinkMapping] | None = None + + +@dataclass +class ConceptMapping: + """Mappings that populate one concept and the relationships under it.""" + concept: Concept + object_mappings: list[ObjectMapping] = field(default_factory=list) + link_mappings: list[LinkMapping] = field(default_factory=list) + + +class OntologyMapping: + """Binds a logical model to an ontology and declares how its fields + populate the ontology's concepts and relationships.""" + _name: str + _description: str | None + _ontology: OntologyComponent + _semantic_model: SemanticModel + _concept_mappings: list[ConceptMapping] + + def __init__( + self, + name: str, + ontology: OntologyComponent, + semantic_model: SemanticModel, + description: str | None = None, + ): + self._name = name + self._description = description + self._ontology = ontology + self._semantic_model = semantic_model + self._concept_mappings = [] + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str | None: + return self._description + + @property + def ontology(self) -> OntologyComponent: + return self._ontology + + @property + def semantic_model(self) -> SemanticModel: + return self._semantic_model + + @property + def concept_mappings(self) -> list[ConceptMapping]: + return list(self._concept_mappings) + + def add_concept_mapping(self, cm: ConceptMapping) -> None: + self._concept_mappings.append(cm) + + +# --------------------------------------------------------------------------- +# Observer protocol + Ontology component (mirrors OntologyComponent in spec) +# --------------------------------------------------------------------------- + +class OntologyObserver(Protocol): + """Structural interface for objects that want to be notified when concepts + or requires are added to an OntologyComponent. Implement both methods + and pass an instance to OntologyComponent.register().""" + + def on_concept_added(self, concept: Concept) -> None: ... + + def on_require_added(self, require: Formula) -> None: ... + + +class OntologyComponent: + """Structural container for concepts, relationships, constraints, and rules. + Document-level metadata (name, description, ai_context) lives on OsiOntology.""" + _concepts: list[Concept] + _relationships: list[Relationship] + _rules: list[Formula] + _requires: list[Formula] + _concept_name_map: dict[str, Concept] + _relationship_name_map: dict[str, Relationship] + _observers: list[OntologyObserver] + + def __init__(self): + self._concepts = [] + self._relationships = [] + self._rules = [] + self._requires = [] + self._concept_name_map = {} + self._relationship_name_map = {} + self._observers = [] + + def register(self, observer: OntologyObserver) -> None: + self._observers.append(observer) + for concept in self._concepts: + observer.on_concept_added(concept) + for require in self._requires: + observer.on_require_added(require) + + def add_concept(self, concept: Concept) -> None: + if concept.name in self._concept_name_map: + raise ValueError(f"Concept '{concept.name}' already exists in the ontology") + self._concepts.append(concept) + self._concept_name_map[concept.name] = concept + for obs in self._observers: + obs.on_concept_added(concept) + + def add_relationship(self, relationship: Relationship) -> None: + full_name = relationship.full_name + if full_name in self._relationship_name_map: + raise ValueError(f"Relationship '{full_name}' already exists in the ontology") + self._relationships.append(relationship) + self._relationship_name_map[full_name] = relationship + + def add_rule(self, rule: Formula) -> None: + self._rules.append(rule) + + def add_require(self, require: Formula) -> None: + self._requires.append(require) + for obs in self._observers: + obs.on_require_added(require) + + def concepts(self, exclude_builtin: bool = False) -> list[Concept]: + if exclude_builtin: + return [c for c in self._concepts if not c.is_builtin] + return list(self._concepts) + + @property + def relationships(self) -> list[Relationship]: + return list(self._relationships) + + @property + def rules(self) -> list[Formula]: + return list(self._rules) + + @property + def requires(self) -> list[Formula]: + return list(self._requires) + + def lookup_concept(self, name: str | None) -> Concept | None: + if not name: + return None + if name in self._concept_name_map: + return self._concept_name_map[name] + if name in BUILTIN_CONCEPTS: + concept = Concept(name=name, builtin=True) + self.add_concept(concept) + return concept + return None + + def lookup_concept_relationship(self, concept: Concept, name: str) -> Relationship | None: + rel = self._relationship_name_map.get(f"{concept.name}.{name}") + if rel: + return rel + for ext in concept.extends: + rel = self.lookup_concept_relationship(ext, name) + if rel: + return rel + return None + + +# --------------------------------------------------------------------------- +# Root semantic model (per OsiSpec) +# --------------------------------------------------------------------------- + +class OsiOntology: + _name: str + _description: str | None + _ai_context: AiContext | None + _version: str | None + _ontology: OntologyComponent + _ontology_mappings: list[OntologyMapping] + _ontology_mapping_index: dict[str, OntologyMapping] + + def __init__( + self, + name: str, + ontology: OntologyComponent, + description: str | None = None, + ai_context: AiContext | None = None, + version: str | None = None, + ): + self._name = name + self._description = description + self._ai_context = ai_context + self._version = version + self._ontology = ontology + self._ontology_mappings = [] + self._ontology_mapping_index = {} + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str | None: + return self._description + + @property + def ai_context(self) -> AiContext | None: + return self._ai_context + + @property + def version(self) -> str | None: + return self._version + + @property + def ontology(self) -> OntologyComponent: + return self._ontology + + def add_ontology_mapping(self, mapping: OntologyMapping) -> None: + if mapping.name in self._ontology_mapping_index: + raise ValueError(f"OntologyMapping '{mapping.name}' already exists in model") + self._ontology_mappings.append(mapping) + self._ontology_mapping_index[mapping.name] = mapping + + @property + def ontology_mappings(self) -> list[OntologyMapping]: + return list(self._ontology_mappings) + + +# --------------------------------------------------------------------------- +# Verbalization parser (handles a list of verbalization patterns) +# --------------------------------------------------------------------------- + +@dataclass +class Verbalization: + text: str + + +@dataclass +class RelationshipVerbalization(Verbalization): + roles: list[VerbalizationRole] + + +@dataclass +class VerbalizationRole: + concept: Concept + name: str | None = None + preceding_text: str | None = None + prefix: str | None = None + following_text: str | None = None + postfix: str | None = None + + def verbalization_name(self) -> str: + return f"{{{self.concept.name}:{self.name}}}" if self.name else f"{{{self.concept.name}}}" + + +_CONCEPT_TOKEN_RE = re.compile(r"\{([^:}]+?)(?::([^}]+))?\}") + + +def parse_verbalizations( + relationship: Relationship, verbalizations: list[str] | None +) -> list[RelationshipVerbalization]: + if not verbalizations: + return [_build_verbalization(relationship)] + return [_parse_verbalization(relationship, v) for v in verbalizations] + + +def _build_verbalization(relationship: Relationship) -> RelationshipVerbalization: + roles: list[VerbalizationRole] = [] + parts: list[str] = [] + for role in relationship.roles: + vr = VerbalizationRole(concept=role.player, name=role.explicit_name) + roles.append(vr) + parts.append(vr.verbalization_name()) + if relationship.unary: + return RelationshipVerbalization(text=f"{relationship.name} {parts[0]}", roles=roles) + return RelationshipVerbalization(text=" has ".join(parts), roles=roles) + + +def _parse_verbalization(relationship: Relationship, verbalization: str) -> RelationshipVerbalization: + """ + Parse a verbalization string into an ordered list of :class:`VerbalizationRole` objects. + + Format example: + + 'every chain- super {Store} reports returns of {Item} big -box for average- {Amount:amt}' + + The string may contain any number of ``{Concept}`` / ``{Concept:roleName}`` tokens. + Text between tokens is split uniformly by :func:`_split_segment` into + ``(postfix, middle, prefix)`` and assigned to the adjacent roles: + + +-----------+------------------------+------------------------------+----------------------+ + | position | postfix | middle | prefix | + +===========+========================+==============================+======================+ + | segment 0 | *ignored* | → roles[0].preceding_text | → roles[0].prefix | + +-----------+------------------------+------------------------------+----------------------+ + | segment i | → roles[i-1].postfix | → roles[i-1].following_text | → roles[i].prefix | + +-----------+------------------------+------------------------------+----------------------+ + | last seg | → roles[-1].postfix | → roles[-1].following_text | *ignored* | + +-----------+------------------------+------------------------------+----------------------+ + """ + tokens = list(_CONCEPT_TOKEN_RE.finditer(verbalization)) + if len(tokens) != relationship.arity: + raise ValueError( + f"Number of roles in verbalization '{verbalization}' for relationship " + f"{relationship.full_name} don't match" + ) + segments: list[str] = [] + roles: list[VerbalizationRole] = [] + prev_end = 0 + for idx, m in enumerate(tokens): + role = relationship.role(idx) + segments.append(verbalization[prev_end:m.start()].strip()) + verb_concept_name = m.group(1).strip() + rel_role_name = role.explicit_name + verb_role_name = m.group(2).strip() if m.group(2) else None + if rel_role_name != verb_role_name or role.player.name != verb_concept_name: + raise ValueError( + f"Role {idx}: '{role.player.name}:{role.name}' " + f"does not match verbalization role '{verb_concept_name}:{verb_role_name}'" + ) + roles.append(VerbalizationRole(concept=role.player, name=verb_role_name)) + prev_end = m.end() + segments.append(verbalization[prev_end:].strip()) + for i, seg in enumerate(segments): + if not seg: + continue + postfix, middle, prefix = _split_segment(seg) + if i == 0: + roles[0].preceding_text = middle + roles[0].prefix = prefix + elif i == len(tokens): + roles[-1].postfix = postfix + roles[-1].following_text = middle + else: + roles[i - 1].postfix = postfix + roles[i - 1].following_text = middle + roles[i].prefix = prefix + return RelationshipVerbalization(text=verbalization, roles=roles) + + +def _split_segment(segment: str) -> tuple[str | None, str | None, str | None]: + words = segment.split() + if not words: + return None, None, None + + postfix_end = 0 + if any(w.startswith("-") for w in words): + postfix_end = 1 + while postfix_end < len(words) and words[postfix_end].startswith("-"): + postfix_end += 1 + + prefix_start = len(words) + for i in range(postfix_end, len(words)): + if words[i].endswith("-"): + prefix_start = i + break + + postfix = " ".join(w.lstrip("-") for w in words[:postfix_end]) if postfix_end > 0 else None + prefix = " ".join(w.rstrip("-") for w in words[prefix_start:]) if prefix_start < len(words) else None + middle = " ".join(words[postfix_end:prefix_start]) if postfix_end < prefix_start else None + + return postfix, middle, prefix diff --git a/converters/ontology/src/osi/parser/__init__.py b/converters/ontology/src/osi/parser/__init__.py new file mode 100644 index 00000000..7c6d8463 --- /dev/null +++ b/converters/ontology/src/osi/parser/__init__.py @@ -0,0 +1,49 @@ +"""Entrypoint: read a YAML/JSON OSI spec and produce a OsiOntology.""" + +from __future__ import annotations + +import json +from io import IOBase + +import yaml + +from osi.converter.spec_to_osi.converter import SpecToOsiConverter +from osi.model import OsiOntology +from osi.spec import OsiSpec + + +class OsiParser: + _model: OsiOntology | None + _spec: OsiSpec | None + _debug: bool + + def __init__(self, debug: bool = False): + self._debug = debug + self._model = None + self._spec = None + + def parse(self, file: IOBase) -> None: + raw = OsiParser.load_data(file) + self._spec = OsiSpec.model_validate(raw) + self._model = SpecToOsiConverter.convert(self._spec) + + @staticmethod + def load_data(file: IOBase): + content = file.read() + file.seek(0) + name = (getattr(file, "name", "") or "").lower() + if name.endswith(".json"): + return json.loads(content) + return yaml.safe_load(content) + + def spec(self) -> OsiSpec: + spec = self._spec + if spec is None: + raise RuntimeError("You must call 'parse()' before accessing 'spec()'") + return spec + + def model(self) -> OsiOntology: + model = self._model + if model is None: + raise RuntimeError("You must call 'parse()' before accessing 'model()'") + return model \ No newline at end of file diff --git a/converters/ontology/src/osi/spec.py b/converters/ontology/src/osi/spec.py new file mode 100644 index 00000000..200f78e0 --- /dev/null +++ b/converters/ontology/src/osi/spec.py @@ -0,0 +1,248 @@ +from __future__ import annotations + +from typing import Any, Literal + +import yaml +from pydantic import BaseModel, ConfigDict, Field + + +class OsiObject(BaseModel): + """Base for all OSI DTOs. Strict (`extra=forbid`) to surface spec drift early.""" + model_config = ConfigDict( + populate_by_name=True, + arbitrary_types_allowed=True, + extra="forbid", + ) + + +# Free-form AI context: either a bare string or a structured object with keys +# like `instructions`, `synonyms`, `examples` (per core.md). +AiContext = str | dict[str, Any] + + +class CustomExtension(OsiObject): + """Vendor-specific metadata attached to any logical-model element (core.md). + + `data` is a JSON-encoded string so vendors can carry arbitrary payloads + without extending the core schema. + """ + vendor_name: str + data: str + + +# ---------- Ontology ---------- + +class Role(OsiObject): + """An additional role in a Relationship (the first role is implicit — the + container concept). `name` is only required to disambiguate when the same + concept plays multiple roles in the same relationship.""" + concept: str + name: str | None = None + + +class Relationship(OsiObject): + """A relationship grouped under its first-role concept. + + `roles` enumerates the *additional* roles (the first is the container + concept). `multiplicity` constrains the last role; `OneToOne` is only + valid for binary relationships. `verbalizes` is a list of natural-language + patterns with `{Concept}` or `{Concept:role_name}` placeholders. + `derived_by` and `requires` are raw expression strings (parsed elsewhere). + """ + name: str + description: str | None = None + roles: list[Role] = Field(default_factory=list) + verbalizes: list[str] = Field(default_factory=list) + multiplicity: Literal["OneToOne", "ManyToOne"] | None = None + derived_by: list[str] = Field(default_factory=list) + requires: list[str] = Field(default_factory=list) + + +class Concept(OsiObject): + """A type-like node in the ontology — either an `EntityType` (real-world + object referenced via other relationships) or a `ValueType` (a data type + with extra semantics, must transitively extend a built-in value type). + + `identify_by` lists the names of relationships (declared under this + concept) whose values uniquely reference its objects. + """ + name: str + type: Literal["EntityType", "ValueType"] | None = None + description: str | None = None + extends: list[str] | None = None + identify_by: list[str] = Field(default_factory=list) + derived_by: list[str] = Field(default_factory=list) + requires: list[str] = Field(default_factory=list) + + +class ConceptComponent(OsiObject): + """Envelope for a concept and the relationships nested under it. + + Mirrors the YAML shape `{ concept: {...}, relationships: [...] }` where + every relationship in the list takes the enclosing concept as its + implicit first role. + """ + concept: Concept + relationships: list[Relationship] = Field(default_factory=list) + + +# ---------- Logical model (per osi/core.md) ---------- + +class DialectExpression(OsiObject): + """A scalar (non-aggregating) SQL/expression in a specific dialect.""" + dialect: str + expression: str + + +class Expression(OsiObject): + """Multi-dialect expression carrier — same logical expression rendered in + one or more dialects (e.g. ANSI_SQL + SNOWFLAKE).""" + dialects: list[DialectExpression] = Field(default_factory=list) + + +class Dimension(OsiObject): + """Dimensional metadata on a DatasetField.""" + is_time: bool | None = None + + +class DatasetField(OsiObject): + """A row-level attribute of a Dataset. `expression` is scalar (no + aggregations); use Metric for aggregates.""" + name: str + expression: Expression + dimension: Dimension | None = None + label: str | None = None + description: str | None = None + ai_context: AiContext | None = None + custom_extensions: list[CustomExtension] = Field(default_factory=list) + + +class Dataset(OsiObject): + """A logical dataset (fact or dimension table) backed by `source` — a + physical table/view reference or a query.""" + name: str + source: str + primary_key: list[str] | None = None + unique_keys: list[list[str]] | None = None + description: str | None = None + ai_context: AiContext | None = None + fields: list[DatasetField] = Field(default_factory=list) + custom_extensions: list[CustomExtension] = Field(default_factory=list) + + +class JoinPath(OsiObject): + """A foreign-key style join between two Datasets: rows in `from` reference + rows in `to` by matching `from_columns` against `to_columns` in order. + Same arity required on both sides.""" + name: str + from_: str = Field(alias="from") + to: str + from_columns: list[str] + to_columns: list[str] + ai_context: AiContext | None = None + custom_extensions: list[CustomExtension] = Field(default_factory=list) + + +class Metric(OsiObject): + """A model-level quantitative measure defined as an aggregate expression. + Can reference fields across multiple Datasets.""" + name: str + expression: Expression + description: str | None = None + ai_context: AiContext | None = None + custom_extensions: list[CustomExtension] = Field(default_factory=list) + + +class SemanticModel(OsiObject): + """A complete logical/semantic model (the body that the core spec calls + `semantic_model`): datasets plus the join paths and metrics defined over + them. One or more SemanticModels can feed a single OntologyMapping.""" + name: str + description: str | None = None + ai_context: AiContext | None = None + datasets: list[Dataset] = Field(default_factory=list) + relationships: list[JoinPath] = Field(default_factory=list) + metrics: list[Metric] = Field(default_factory=list) + custom_extensions: list[CustomExtension] = Field(default_factory=list) + + +# ---------- Ontology mapping ---------- + +class ReferentMapping(OsiObject): + """Locates an entity object by walking one of its identifying + relationships. Carries either a leaf `expression` (SQL over dataset + fields) or a nested `referent_mappings` list when the referenced concept + is itself an entity with a compound/recursive identifier.""" + relationship: str + expression: str | None = None + referent_mappings: list[ReferentMapping] | None = None + + +class ObjectMapping(OsiObject): + """Maps to objects of some concept. Either a direct scalar `expression` + (for value types or simple-id entities) or `referent_mappings` (for + entities with compound identifiers). XOR — never both.""" + concept: str | None = None + expression: str | None = None + referent_mappings: list[ReferentMapping] | None = None + + +class LinkMapping(OsiObject): + """A node in the link-mapping tree. The arity of `relationship` equals + the node's depth (top-level = unary, depth 2 = binary, etc.). `children` + extend the mapped tuple by one role each, sharing this node's + `object_mapping` as their prefix to avoid duplication.""" + object_mapping: ObjectMapping + relationship: str | None = None + children: list[LinkMapping] | None = None + + +class ConceptMapping(OsiObject): + """Mappings that populate one concept and the relationships grouped under + it. `object_mappings` populate the concept's objects; `link_mappings` is + a forest of trees populating its relationships.""" + concept: str + object_mappings: list[ObjectMapping] = Field(default_factory=list) + link_mappings: list[LinkMapping] = Field(default_factory=list) + + +class OntologyMapping(OsiObject): + """Binds a semantic model to the document ontology, then declares how its + fields populate the ontology's concepts and relationships.""" + name: str + description: str | None = None + semantic_model: SemanticModel + concept_mappings: list[ConceptMapping] = Field(default_factory=list) + + +# ---------- Root ---------- + +class OsiSpec(OsiObject): + """Root OSI document: a single ontology definition and the ontology + mappings that wire semantic models into it.""" + version: str | None = None + name: str + description: str | None = None + ai_context: AiContext | None = None + ontology: list[ConceptComponent] = Field(default_factory=list) + ontology_mappings: list[OntologyMapping] = Field(default_factory=list) + + @classmethod + def load_yaml(cls, text: str) -> OsiSpec: + return cls.model_validate(yaml.safe_load(text)) + + def dump_dict(self) -> dict: + return self.model_dump(exclude_none=True, exclude_defaults=True, by_alias=True) + + def dump_yaml(self) -> str: + return yaml.safe_dump(self.dump_dict(), sort_keys=False) + + +# `ReferentMapping` and `LinkMapping` are self-referential (each can contain a +# list of itself). Combined with `from __future__ import annotations`, every +# annotation is a string at class-definition time, so the self-reference is an +# unresolved forward ref. `model_rebuild()` re-walks the schema once the class +# is fully defined and pins the forward ref to the real type — without it, +# validating a payload with nested children raises PydanticUndefinedAnnotation. +ReferentMapping.model_rebuild() +LinkMapping.model_rebuild() \ No newline at end of file From 9a8a8bbe429acdd6932546a81788daec1870de96 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Thu, 18 Jun 2026 13:29:27 +0200 Subject: [PATCH 02/29] Updated OSI Spec --- converters/ontology/src/osi/spec.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/converters/ontology/src/osi/spec.py b/converters/ontology/src/osi/spec.py index 200f78e0..68e75dd4 100644 --- a/converters/ontology/src/osi/spec.py +++ b/converters/ontology/src/osi/spec.py @@ -223,6 +223,7 @@ class OsiSpec(OsiObject): version: str | None = None name: str description: str | None = None + requires: list[str] = Field(default_factory=list) ai_context: AiContext | None = None ontology: list[ConceptComponent] = Field(default_factory=list) ontology_mappings: list[OntologyMapping] = Field(default_factory=list) @@ -245,4 +246,4 @@ def dump_yaml(self) -> str: # is fully defined and pins the forward ref to the real type — without it, # validating a payload with nested children raises PydanticUndefinedAnnotation. ReferentMapping.model_rebuild() -LinkMapping.model_rebuild() \ No newline at end of file +LinkMapping.model_rebuild() From d69f2022cdb2c7a86dd71f8940e202f67fcf2349 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Thu, 18 Jun 2026 15:21:43 +0200 Subject: [PATCH 03/29] Added `is_component` to the Concept --- converters/ontology/src/osi/model.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/converters/ontology/src/osi/model.py b/converters/ontology/src/osi/model.py index 5b354920..7461cc51 100644 --- a/converters/ontology/src/osi/model.py +++ b/converters/ontology/src/osi/model.py @@ -80,6 +80,7 @@ class Concept: _identify_by: dict[str, Relationship] _derived_by: list[Formula] _requires: list[Formula] + _is_component: bool def __init__( self, @@ -91,6 +92,7 @@ def __init__( identify_by: dict[str, Relationship] | None = None, derived_by: list[Formula] | None = None, requires: list[Formula] | None = None, + is_component: bool = True ): self._name = name self._type = type @@ -100,6 +102,7 @@ def __init__( self._identify_by = identify_by if identify_by else {} self._derived_by = derived_by if derived_by else [] self._requires = requires if requires else [] + self._is_component = is_component def add_require(self, require: Formula) -> None: self._requires.append(require) @@ -129,6 +132,11 @@ def description(self) -> str | None: def is_builtin(self) -> bool: return self._builtin + # True if this concept identifies a concept component in an OSI ontology + @property + def is_component(self) -> bool: + return self._is_component + @property def is_value_type(self) -> bool: return self._type == ConceptType.VALUE_TYPE @@ -916,7 +924,7 @@ def lookup_concept(self, name: str | None) -> Concept | None: if name in self._concept_name_map: return self._concept_name_map[name] if name in BUILTIN_CONCEPTS: - concept = Concept(name=name, builtin=True) + concept = Concept(name=name, builtin=True, is_component=False) self.add_concept(concept) return concept return None From 7f42fcc618e7f90da6bf7f7363f7f7c2f68a8233 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Mon, 22 Jun 2026 15:09:47 +0200 Subject: [PATCH 04/29] Added FormulaFactory protocol as an entry point for some FormulaParser --- .../ontology/scripts/palantir_to_osi.py | 2 +- converters/ontology/src/osi/__init__.py | 2 + .../converter/palantir_to_osi/converter.py | 87 +++++---- .../osi/converter/spec_to_osi/converter.py | 165 ++++++++---------- converters/ontology/src/osi/model.py | 11 ++ .../ontology/src/osi/parser/__init__.py | 2 +- 6 files changed, 135 insertions(+), 134 deletions(-) diff --git a/converters/ontology/scripts/palantir_to_osi.py b/converters/ontology/scripts/palantir_to_osi.py index ecdab07a..c1ed4242 100644 --- a/converters/ontology/scripts/palantir_to_osi.py +++ b/converters/ontology/scripts/palantir_to_osi.py @@ -48,7 +48,7 @@ with open(path, mode) as file: parser.parse(file) - ontology_model = PalantirToOsiConverter.convert(parser.model(), db_name, schema_name) + ontology_model = PalantirToOsiConverter().convert(parser.model(), db_name, schema_name) osi_spec = OsiToSpecConverter.convert(ontology_model) print(osi_spec.dump_yaml()) diff --git a/converters/ontology/src/osi/__init__.py b/converters/ontology/src/osi/__init__.py index 5aa05c23..e81bd319 100644 --- a/converters/ontology/src/osi/__init__.py +++ b/converters/ontology/src/osi/__init__.py @@ -14,6 +14,7 @@ DialectExpression, DialectExpressionSet, Formula, + FormulaFactory, JoinPath, LinkMapping, Metric, @@ -60,6 +61,7 @@ "OsiOntology", # Supporting types "CustomExtension", + "FormulaFactory", # Spec DTO "OsiSpec", # Parsers diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index a3578828..7e9f8499 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -28,6 +28,7 @@ DialectExpression, DialectExpressionSet, Formula, + FormulaFactory, LinkMapping, SemanticModel, ObjectMapping, @@ -44,16 +45,27 @@ class PalantirToOsiConverter: - """Top-level converter. Use `convert(ontology_model)` to obtain an `OsiOntology`.""" + """Converts a Palantir Ontology to OsiOntology. + + Pass a *formula_factory* to control how Formula objects are created. + The default produces plain ``Formula`` instances; downstream packages can + inject a factory that returns enriched subclasses (e.g. with an AST). + + model = PalantirToOsiConverter().convert(palantir_ontology) + model = PalantirToOsiConverter(formula_factory=my_parser).convert(palantir_ontology) + """ depths_role_names = {1: "fst", 2: "snd", 3: "thd", 4: "frt"} + def __init__(self, formula_factory: FormulaFactory = Formula): + self._formula_factory = formula_factory + # ------------------------------------------------------------------ # Entry point # ------------------------------------------------------------------ - @staticmethod def convert( + self, palantir_ontology: PalantirOntology, db_name: str = "palantir", schema_name: str = "palantir", @@ -70,12 +82,8 @@ def convert( # get created; emitted into the OntologyMapping at the end so they appear in a stable order. concept_mappings: list[ConceptMapping] = [] - PalantirToOsiConverter._convert_concepts( - ontology, semantic_model, palantir_ontology, concept_mappings, db_name, schema_name - ) - PalantirToOsiConverter._convert_relationships( - ontology, palantir_ontology, concept_mappings, semantic_model - ) + self._convert_concepts(ontology, semantic_model, palantir_ontology, concept_mappings, db_name, schema_name) + self._convert_relationships(ontology, palantir_ontology, concept_mappings, semantic_model) for cm in concept_mappings: ontology_mapping.add_concept_mapping(cm) @@ -86,8 +94,8 @@ def convert( # Concepts # ------------------------------------------------------------------ - @staticmethod def _convert_concepts( + self, ontology: OntologyComponent, semantic_model: SemanticModel, palantir_ontology: PalantirOntology, @@ -116,7 +124,7 @@ def _convert_concepts( for ot_guid in order: ot = palantir_ontology.object_types()[ot_guid] if ot.active() or ot.endorsed() or ot.intermediary(): - PalantirToOsiConverter._convert_object_type( + self._convert_object_type( ontology, semantic_model, ot, @@ -127,8 +135,8 @@ def _convert_concepts( schema_name, ) - @staticmethod def _convert_object_type( + self, ontology: OntologyComponent, semantic_model: SemanticModel, ot: ObjectType, @@ -162,7 +170,7 @@ def _convert_object_type( ontology.add_concept(concept) for prop in relevant_props: - PalantirToOsiConverter._convert_property(ontology, concept, prop) + self._convert_property(ontology, concept, prop) if not is_subtype or ignore_subtype: identifiers: dict[str, Relationship] = {} @@ -200,12 +208,11 @@ def _convert_object_type( f"contain the '{prop_name}' property." ) - PalantirToOsiConverter._convert_mappings( + self._convert_mappings( ontology, semantic_model, ot, subtype_relations, concept, concept_mappings, db_name, schema_name ) - @staticmethod - def _convert_property(ontology: OntologyComponent, concept: Concept, prop: PalantirProperty) -> None: + def _convert_property(self, ontology: OntologyComponent, concept: Concept, prop: PalantirProperty) -> None: def madlib_decl(c: Concept, p: PalantirProperty) -> str: return ( f"{{{c}}} {p.readable_id()} " @@ -217,7 +224,7 @@ def madlib_decl(c: Concept, p: PalantirProperty) -> str: return relates: list[tuple[Concept, str | None]] = [] - relates = PalantirToOsiConverter._convert_property_type_roles(ontology, relates, prop.type()) + relates = self._convert_property_type_roles(ontology, relates, prop.type()) ontology.add_relationship(Relationship( name=prop_name, @@ -230,8 +237,8 @@ def madlib_decl(c: Concept, p: PalantirProperty) -> str: # Mappings: ConceptMapping per (concept, dataset) # ------------------------------------------------------------------ - @staticmethod def _convert_mappings( + self, ontology: OntologyComponent, semantic_model: SemanticModel, ot: ObjectType, @@ -264,9 +271,7 @@ def resolve(p: PalantirProperty) -> PalantirProperty: return p for palantir_ds in ot.syncs_from(): - dataset = PalantirToOsiConverter._convert_dataset( - semantic_model, ontology, ot, palantir_ds, db_name, schema_name - ) + dataset = self._convert_dataset(semantic_model, ontology, ot, palantir_ds, db_name, schema_name) # Build referent_mappings that locate `concept` instances by # walking the (effective) identifying relationships against this @@ -349,8 +354,8 @@ def resolve(p: PalantirProperty) -> PalantirProperty: # Relations (M:1, M:M, intermediary) # ------------------------------------------------------------------ - @staticmethod def _convert_relationships( + self, ontology: OntologyComponent, palantir_ontology: PalantirOntology, concept_mappings: list[ConceptMapping], @@ -358,42 +363,40 @@ def _convert_relationships( ) -> None: for rel in palantir_ontology.relations().values(): if rel.active() or rel.intermediary(): - PalantirToOsiConverter._convert_relation(ontology, rel, concept_mappings, semantic_model) + self._convert_relation(ontology, rel, concept_mappings, semantic_model) elif ( isinstance(rel, ManyToOneRelation) and rel.experimental() and rel.one_object_type().active() and rel.many_object_type().active() ): - PalantirToOsiConverter._convert_relation(ontology, rel, concept_mappings, semantic_model) + self._convert_relation(ontology, rel, concept_mappings, semantic_model) for ir in palantir_ontology.intermediary_relations().values(): if ir.active() or ir.intermediary(): - PalantirToOsiConverter._convert_intermediary_relation(ontology, palantir_ontology, ir) + self._convert_intermediary_relation(ontology, palantir_ontology, ir) elif ( ir.experimental() and ir.role_a_player().active() and ir.role_b_player().active() and ir.intermediary_player().active() ): - PalantirToOsiConverter._convert_intermediary_relation(ontology, palantir_ontology, ir) + self._convert_intermediary_relation(ontology, palantir_ontology, ir) - @staticmethod def _convert_relation( + self, ontology: OntologyComponent, relation: Relation, concept_mappings: list[ConceptMapping], semantic_model: SemanticModel, ) -> None: if isinstance(relation, ManyToOneRelation): - PalantirToOsiConverter._convert_many_to_one( - ontology, relation, concept_mappings, semantic_model - ) + self._convert_many_to_one(ontology, relation, concept_mappings, semantic_model) elif isinstance(relation, ManyToManyRelation): - PalantirToOsiConverter._convert_many_to_many(ontology, relation) + self._convert_many_to_many(ontology, relation) - @staticmethod def _convert_many_to_one( + self, ontology: OntologyComponent, rel: ManyToOneRelation, concept_mappings: list[ConceptMapping], @@ -426,7 +429,7 @@ def _convert_many_to_one( ontology.add_relationship(relationship) if mot._syncs_from: - PalantirToOsiConverter._attach_link_to_concept_mappings( + self._attach_link_to_concept_mappings( ontology, rel, relationship, mot, mot_concept, oot_concept, concept_mappings, semantic_model ) else: @@ -438,12 +441,12 @@ def _convert_many_to_one( for mprop, oprop in rel.property_map().items() ] if frags: - formula = Formula(raw_expr=" AND ".join(frags), parent=relationship) + formula = self._formula_factory(raw_expr=" AND ".join(frags), parent=relationship) relationship.add_derived_by(formula) ontology.add_rule(formula) - @staticmethod def _attach_link_to_concept_mappings( + self, ontology: OntologyComponent, rel: ManyToOneRelation, relationship: Relationship, @@ -553,8 +556,7 @@ def _references_dataset(cm: ConceptMapping, dataset: Dataset) -> bool: for rm in (om.referent_mappings or []) ) - @staticmethod - def _convert_many_to_many(ontology: OntologyComponent, rel: ManyToManyRelation) -> None: + def _convert_many_to_many(self, ontology: OntologyComponent, rel: ManyToManyRelation) -> None: aot = rel.role_a_player() aot_concept = ontology.lookup_concept(PalantirToOsiConverter._concept_name(aot)) bot = rel.role_b_player() @@ -579,8 +581,8 @@ def _convert_many_to_many(ontology: OntologyComponent, rel: ManyToManyRelation) ) ontology.add_relationship(relationship) - @staticmethod def _convert_intermediary_relation( + self, ontology: OntologyComponent, palantir_ontology: PalantirOntology, rel: IntermediaryRelation, @@ -636,7 +638,7 @@ def _convert_intermediary_relation( f"{fp_a}.{rel_a_name}({relationship.first_role.name}) AND " f"{fp_b}.{rel_b_name}({relationship.last_role.name})" ) - formula = Formula(raw_expr=join_condition, parent=relationship) + formula = self._formula_factory(raw_expr=join_condition, parent=relationship) relationship.add_derived_by(formula) ontology.add_rule(formula) @@ -644,8 +646,8 @@ def _convert_intermediary_relation( # Datasets # ------------------------------------------------------------------ - @staticmethod def _convert_dataset( + self, semantic_model: SemanticModel, ontology: OntologyComponent, ot: ObjectType, @@ -721,18 +723,15 @@ def _type_to_madlib_suffix(type_, arr_depth: int = 1) -> str: ) return f"{{{type_.to_type()}}}" - @staticmethod def _convert_property_type_roles( - ontology: OntologyComponent, roles: list[tuple[Concept, str | None]], type_, arr_depth: int = 1 + self, ontology: OntologyComponent, roles: list[tuple[Concept, str | None]], type_, arr_depth: int = 1 ) -> list[tuple[Concept, str | None]]: if isinstance(type_, ArrayDataType): integer = ontology.lookup_concept("Integer") if integer is None: raise ValueError("Builtin 'Integer' could not be resolved for array role.") roles.append((integer, PalantirToOsiConverter._depth_role_name(arr_depth))) - PalantirToOsiConverter._convert_property_type_roles( - ontology, roles, type_.base_type(), arr_depth + 1 - ) + self._convert_property_type_roles(ontology, roles, type_.base_type(), arr_depth + 1) else: target = ontology.lookup_concept(type_.to_type()) if target is None: diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index a11a39c5..ad8d5c58 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -16,6 +16,7 @@ DialectExpressionSet, Dimension, Formula, + FormulaFactory, JoinPath, LinkMapping, SemanticModel, @@ -56,10 +57,20 @@ class SpecToOsiConverter: - """Top-level converter. Use `convert(spec)` to obtain a OsiOntology.""" + """Converts OsiSpec (Pydantic DTOs) to OsiOntology (runtime model). - @staticmethod - def convert(spec: OsiSpec) -> OsiOntology: + Pass a *formula_factory* to control how Formula objects are created. + The default produces plain ``Formula`` instances; downstream packages can + inject a factory that returns enriched subclasses (e.g. with an AST). + + model = SpecToOsiConverter().convert(spec) + model = SpecToOsiConverter(formula_factory=my_parser).convert(spec) + """ + + def __init__(self, formula_factory: FormulaFactory = Formula): + self._formula_factory = formula_factory + + def convert(self, spec: OsiSpec) -> OsiOntology: ontology = OntologyComponent() model = OsiOntology( name=spec.name, @@ -69,20 +80,19 @@ def convert(spec: OsiSpec) -> OsiOntology: version=spec.version, ) - SpecToOsiConverter._populate_ontology(ontology, spec) + self._populate_ontology(ontology, spec) for om_spec in spec.ontology_mappings: - SpecToOsiConverter._convert_ontology_mapping(model, om_spec) + self._convert_ontology_mapping(model, om_spec) return model # ----- Ontology ------------------------------------------------------ - @staticmethod - def _populate_ontology(ontology: OntologyComponent, spec: OsiSpec) -> None: + def _populate_ontology(self, ontology: OntologyComponent, spec: OsiSpec) -> None: concept_specs = {concept_component.concept.name: concept_component.concept for concept_component in spec.ontology} - sorted_names = SpecToOsiConverter._sort_spec_dependency_graph(list(concept_specs.values())) + sorted_names = self._sort_spec_dependency_graph(list(concept_specs.values())) for name in sorted_names: concept_spec = concept_specs[name] extends: list[Concept] = [] @@ -108,7 +118,7 @@ def _populate_ontology(ontology: OntologyComponent, spec: OsiSpec) -> None: if container is None: raise ValueError(f"Internal: container concept '{concept_component.concept.name}' not found") for rel_spec in concept_component.relationships: - SpecToOsiConverter._convert_relationship(ontology, container, rel_spec) + self._convert_relationship(ontology, container, rel_spec) # Identifiers: now that all relationships exist, resolve identify_by. for concept_component in spec.ontology: @@ -134,12 +144,12 @@ def _populate_ontology(ontology: OntologyComponent, spec: OsiSpec) -> None: if concept is None: continue for raw in concept_spec.requires: - req = _build_rule(raw, concept) + req = self._build_rule(raw, concept) if req: concept.add_require(req) ontology.add_require(req) for raw in concept_spec.derived_by: - rule = _build_rule(raw, concept) + rule = self._build_rule(raw, concept) if rule: concept.add_derived_by(rule) ontology.add_rule(rule) @@ -148,19 +158,18 @@ def _populate_ontology(ontology: OntologyComponent, spec: OsiSpec) -> None: if rel is None: continue for raw in rel_spec.requires: - req = _build_rule(raw, rel) + req = self._build_rule(raw, rel) if req: rel.add_require(req) ontology.add_require(req) for raw in rel_spec.derived_by: - rule = _build_rule(raw, rel) + rule = self._build_rule(raw, rel) if rule: rel.add_derived_by(rule) ontology.add_rule(rule) - @staticmethod def _convert_relationship( - ontology: OntologyComponent, container: Concept, rel_spec: SpecRelationship + self, ontology: OntologyComponent, container: Concept, rel_spec: SpecRelationship ) -> None: relates: list[tuple[Concept, str | None]] = [] for role_spec in rel_spec.roles: @@ -185,8 +194,7 @@ def _convert_relationship( # ----- Logical model ------------------------------------------------- - @staticmethod - def _convert_semantic_model(lm_spec: SpecSemanticModel) -> SemanticModel: + def _convert_semantic_model(self, lm_spec: SpecSemanticModel) -> SemanticModel: semantic_model = SemanticModel( name=lm_spec.name, description=lm_spec.description, @@ -205,11 +213,10 @@ def _convert_semantic_model(lm_spec: SpecSemanticModel) -> SemanticModel: # ----- Ontology mapping --------------------------------------------- - @staticmethod - def _convert_ontology_mapping(model: OsiOntology, om_spec: SpecOntologyMapping) -> None: + def _convert_ontology_mapping(self, model: OsiOntology, om_spec: SpecOntologyMapping) -> None: ontology = model.ontology - semantic_model = SpecToOsiConverter._convert_semantic_model(om_spec.semantic_model) + semantic_model = self._convert_semantic_model(om_spec.semantic_model) mapping = OntologyMapping( name=om_spec.name, @@ -220,14 +227,10 @@ def _convert_ontology_mapping(model: OsiOntology, om_spec: SpecOntologyMapping) model.add_ontology_mapping(mapping) for cm_spec in om_spec.concept_mappings: - mapping.add_concept_mapping( - SpecToOsiConverter._convert_concept_mapping( - model, ontology, semantic_model, cm_spec - ) - ) + mapping.add_concept_mapping(self._convert_concept_mapping(model, ontology, semantic_model, cm_spec)) - @staticmethod def _convert_concept_mapping( + self, model: OsiOntology, ontology: OntologyComponent, semantic_model: SemanticModel, @@ -241,20 +244,16 @@ def _convert_concept_mapping( cm = ConceptMapping(concept=concept) for object_mapping_spec in cm_spec.object_mappings: cm.object_mappings.append( - SpecToOsiConverter._convert_object_mapping( - model, ontology, semantic_model, concept, object_mapping_spec - ) + self._convert_object_mapping(model, ontology, semantic_model, concept, object_mapping_spec) ) for link_mapping_spec in cm_spec.link_mappings: cm.link_mappings.append( - SpecToOsiConverter._convert_link_mapping( - model, ontology, semantic_model, concept, link_mapping_spec - ) + self._convert_link_mapping(model, ontology, semantic_model, concept, link_mapping_spec) ) return cm - @staticmethod def _convert_object_mapping( + self, model: OsiOntology, ontology: OntologyComponent, semantic_model: SemanticModel, @@ -271,20 +270,18 @@ def _convert_object_mapping( ) expression: DatasetField | Formula | None = None if om_spec.expression is not None: - expression = _resolve_mapping_expression(om_spec.expression, semantic_model, concept) + expression = self._resolve_mapping_expression(om_spec.expression, semantic_model, concept) referent_mappings = None if om_spec.referent_mappings is not None: rm_container = concept if concept is not None else container referent_mappings = [ - SpecToOsiConverter._convert_referent_mapping( - model, ontology, semantic_model, rm_container, rm - ) + self._convert_referent_mapping(model, ontology, semantic_model, rm_container, rm) for rm in om_spec.referent_mappings ] return ObjectMapping(concept=concept, expression=expression, referent_mappings=referent_mappings) - @staticmethod def _convert_referent_mapping( + self, model: OsiOntology, ontology: OntologyComponent, semantic_model: SemanticModel, @@ -300,26 +297,24 @@ def _convert_referent_mapping( sibling_player = rel.last_role.player expression: DatasetField | Formula | None = None if rm_spec.expression is not None: - expression = _resolve_mapping_expression(rm_spec.expression, semantic_model, sibling_player) + expression = self._resolve_mapping_expression(rm_spec.expression, semantic_model, sibling_player) nested = None if rm_spec.referent_mappings is not None: nested = [ - SpecToOsiConverter._convert_referent_mapping( - model, ontology, semantic_model, sibling_player, child - ) + self._convert_referent_mapping(model, ontology, semantic_model, sibling_player, child) for child in rm_spec.referent_mappings ] return ReferentMapping(relationship=rel, expression=expression, referent_mappings=nested) - @staticmethod def _convert_link_mapping( + self, model: OsiOntology, ontology: OntologyComponent, semantic_model: SemanticModel, container: Concept, lm_spec: SpecLinkMapping, ) -> LinkMapping: - object_mapping = SpecToOsiConverter._convert_object_mapping( + object_mapping = self._convert_object_mapping( model, ontology, semantic_model, container, lm_spec.object_mapping ) relationship: Relationship | None = None @@ -334,14 +329,48 @@ def _convert_link_mapping( if lm_spec.children is not None: child_container = relationship.last_role.player if relationship is not None else container children = [ - SpecToOsiConverter._convert_link_mapping( - model, ontology, semantic_model, child_container, child - ) + self._convert_link_mapping(model, ontology, semantic_model, child_container, child) for child in lm_spec.children ] return LinkMapping(object_mapping=object_mapping, relationship=relationship, children=children) - # ----- helpers ------------------------------------------------------- + # ----- Formula helpers ----------------------------------------------- + + def _build_rule(self, raw: str | None, parent: Container) -> Formula | None: + if not raw: + return None + return self._formula_factory(raw_expr=raw, parent=parent) + + def _resolve_mapping_expression( + self, expression: str, semantic_model: SemanticModel, expected_type: Concept | None + ) -> DatasetField | Formula: + """Map a raw spec expression onto either a DatasetField (single + `DATASET.field` or bare `field` reference) or a Formula (anything else). + """ + qualified = _QUALIFIED_FIELD_RE.match(expression) + if qualified: + ds_name, field_name = qualified.group(1), qualified.group(2) + dataset = semantic_model.lookup_dataset(ds_name) + if dataset is not None: + field = dataset.field(field_name) + if field is not None: + _pin_field_type(field, expected_type) + return field + return self._formula_factory(raw_expr=expression) + + bare = _BARE_FIELD_RE.match(expression) + if bare: + field_name = bare.group(1) + for dataset in semantic_model.datasets: + field = dataset.field(field_name) + if field is not None: + _pin_field_type(field, expected_type) + return field + return self._formula_factory(raw_expr=expression) + + return self._formula_factory(raw_expr=expression) + + # ----- Structural helpers -------------------------- @staticmethod def _sort_spec_dependency_graph(concepts: list[SpecConcept]) -> list[str]: @@ -354,46 +383,6 @@ def _sort_spec_dependency_graph(concepts: list[SpecConcept]) -> list[str]: return topological_sort(nodes, edges) -# --------------------------------------------------------------------------- -# Module-level helpers (logical-model + custom extensions) -# --------------------------------------------------------------------------- - -def _build_rule(raw: str | None, parent: Container) -> Formula | None: - if not raw: - return None - return Formula(raw_expr=raw, parent=parent) - - -def _resolve_mapping_expression( - expression: str, semantic_model: SemanticModel, expected_type: Concept | None -) -> DatasetField | Formula: - """Map a raw spec expression onto either a DatasetField (single - `DATASET.field` or bare `field` reference) or a Formula (anything else). - """ - qualified = _QUALIFIED_FIELD_RE.match(expression) - if qualified: - ds_name, field_name = qualified.group(1), qualified.group(2) - dataset = semantic_model.lookup_dataset(ds_name) - if dataset is not None: - field = dataset.field(field_name) - if field is not None: - _pin_field_type(field, expected_type) - return field - return Formula(raw_expr=expression) - - bare = _BARE_FIELD_RE.match(expression) - if bare: - field_name = bare.group(1) - for dataset in semantic_model.datasets: - field = dataset.field(field_name) - if field is not None: - _pin_field_type(field, expected_type) - return field - return Formula(raw_expr=expression) - - return Formula(raw_expr=expression) - - def _pin_field_type(field: DatasetField, expected_type: Concept | None) -> None: if expected_type is None: return diff --git a/converters/ontology/src/osi/model.py b/converters/ontology/src/osi/model.py index 7461cc51..77e84017 100644 --- a/converters/ontology/src/osi/model.py +++ b/converters/ontology/src/osi/model.py @@ -393,6 +393,17 @@ def __str__(self) -> str: return self._raw_expr +class FormulaFactory(Protocol): + """Callable that produces a Formula (or a subclass) from a raw expression. + + The default implementation is the Formula constructor itself. Override in + a downstream package to return an enriched subclass — e.g. one carrying an + AST produced by a FormulaParser — without touching the converters. + """ + + def __call__(self, raw_expr: str, parent: FormulaParent = None) -> Formula: ... + + # --------------------------------------------------------------------------- # Semantic model (datasets, join paths, metrics) # --------------------------------------------------------------------------- diff --git a/converters/ontology/src/osi/parser/__init__.py b/converters/ontology/src/osi/parser/__init__.py index 7c6d8463..911407af 100644 --- a/converters/ontology/src/osi/parser/__init__.py +++ b/converters/ontology/src/osi/parser/__init__.py @@ -25,7 +25,7 @@ def __init__(self, debug: bool = False): def parse(self, file: IOBase) -> None: raw = OsiParser.load_data(file) self._spec = OsiSpec.model_validate(raw) - self._model = SpecToOsiConverter.convert(self._spec) + self._model = SpecToOsiConverter().convert(self._spec) @staticmethod def load_data(file: IOBase): From ec6283235785fda35fd4eb2bd70141a85fdc9f8d Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Tue, 23 Jun 2026 14:21:57 +0200 Subject: [PATCH 05/29] Extended FormulaFactory --- .../converter/palantir_to_osi/converter.py | 6 ++--- .../osi/converter/spec_to_osi/converter.py | 26 +++++++++---------- converters/ontology/src/osi/model.py | 15 ++++++----- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index 7e9f8499..45887a11 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -57,7 +57,7 @@ class PalantirToOsiConverter: depths_role_names = {1: "fst", 2: "snd", 3: "thd", 4: "frt"} - def __init__(self, formula_factory: FormulaFactory = Formula): + def __init__(self, formula_factory: FormulaFactory = FormulaFactory()): self._formula_factory = formula_factory # ------------------------------------------------------------------ @@ -441,7 +441,7 @@ def _convert_many_to_one( for mprop, oprop in rel.property_map().items() ] if frags: - formula = self._formula_factory(raw_expr=" AND ".join(frags), parent=relationship) + formula = self._formula_factory(raw_expr=" AND ".join(frags), parent=relationship, ontology=ontology) relationship.add_derived_by(formula) ontology.add_rule(formula) @@ -638,7 +638,7 @@ def _convert_intermediary_relation( f"{fp_a}.{rel_a_name}({relationship.first_role.name}) AND " f"{fp_b}.{rel_b_name}({relationship.last_role.name})" ) - formula = self._formula_factory(raw_expr=join_condition, parent=relationship) + formula = self._formula_factory(raw_expr=join_condition, parent=relationship, ontology=ontology) relationship.add_derived_by(formula) ontology.add_rule(formula) diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index ad8d5c58..9f835546 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -67,7 +67,7 @@ class SpecToOsiConverter: model = SpecToOsiConverter(formula_factory=my_parser).convert(spec) """ - def __init__(self, formula_factory: FormulaFactory = Formula): + def __init__(self, formula_factory: FormulaFactory = FormulaFactory()): self._formula_factory = formula_factory def convert(self, spec: OsiSpec) -> OsiOntology: @@ -144,12 +144,12 @@ def _populate_ontology(self, ontology: OntologyComponent, spec: OsiSpec) -> None if concept is None: continue for raw in concept_spec.requires: - req = self._build_rule(raw, concept) + req = self._build_rule(raw, concept, ontology) if req: concept.add_require(req) ontology.add_require(req) for raw in concept_spec.derived_by: - rule = self._build_rule(raw, concept) + rule = self._build_rule(raw, concept, ontology) if rule: concept.add_derived_by(rule) ontology.add_rule(rule) @@ -158,12 +158,12 @@ def _populate_ontology(self, ontology: OntologyComponent, spec: OsiSpec) -> None if rel is None: continue for raw in rel_spec.requires: - req = self._build_rule(raw, rel) + req = self._build_rule(raw, rel, ontology) if req: rel.add_require(req) ontology.add_require(req) for raw in rel_spec.derived_by: - rule = self._build_rule(raw, rel) + rule = self._build_rule(raw, rel, ontology) if rule: rel.add_derived_by(rule) ontology.add_rule(rule) @@ -270,7 +270,7 @@ def _convert_object_mapping( ) expression: DatasetField | Formula | None = None if om_spec.expression is not None: - expression = self._resolve_mapping_expression(om_spec.expression, semantic_model, concept) + expression = self._resolve_mapping_expression(om_spec.expression, semantic_model, concept, ontology) referent_mappings = None if om_spec.referent_mappings is not None: rm_container = concept if concept is not None else container @@ -297,7 +297,7 @@ def _convert_referent_mapping( sibling_player = rel.last_role.player expression: DatasetField | Formula | None = None if rm_spec.expression is not None: - expression = self._resolve_mapping_expression(rm_spec.expression, semantic_model, sibling_player) + expression = self._resolve_mapping_expression(rm_spec.expression, semantic_model, sibling_player, ontology) nested = None if rm_spec.referent_mappings is not None: nested = [ @@ -336,13 +336,13 @@ def _convert_link_mapping( # ----- Formula helpers ----------------------------------------------- - def _build_rule(self, raw: str | None, parent: Container) -> Formula | None: + def _build_rule(self, raw: str | None, parent: Container, ontology: OntologyComponent) -> Formula | None: if not raw: return None - return self._formula_factory(raw_expr=raw, parent=parent) + return self._formula_factory(raw_expr=raw, parent=parent, ontology=ontology) def _resolve_mapping_expression( - self, expression: str, semantic_model: SemanticModel, expected_type: Concept | None + self, expression: str, semantic_model: SemanticModel, expected_type: Concept | None, ontology: OntologyComponent ) -> DatasetField | Formula: """Map a raw spec expression onto either a DatasetField (single `DATASET.field` or bare `field` reference) or a Formula (anything else). @@ -356,7 +356,7 @@ def _resolve_mapping_expression( if field is not None: _pin_field_type(field, expected_type) return field - return self._formula_factory(raw_expr=expression) + return self._formula_factory(raw_expr=expression, ontology=ontology) bare = _BARE_FIELD_RE.match(expression) if bare: @@ -366,9 +366,9 @@ def _resolve_mapping_expression( if field is not None: _pin_field_type(field, expected_type) return field - return self._formula_factory(raw_expr=expression) + return self._formula_factory(raw_expr=expression, ontology=ontology) - return self._formula_factory(raw_expr=expression) + return self._formula_factory(raw_expr=expression, ontology=ontology) # ----- Structural helpers -------------------------- diff --git a/converters/ontology/src/osi/model.py b/converters/ontology/src/osi/model.py index 77e84017..9181c923 100644 --- a/converters/ontology/src/osi/model.py +++ b/converters/ontology/src/osi/model.py @@ -393,15 +393,18 @@ def __str__(self) -> str: return self._raw_expr -class FormulaFactory(Protocol): - """Callable that produces a Formula (or a subclass) from a raw expression. +class FormulaFactory: + """Base factory that produces a Formula from a raw expression. - The default implementation is the Formula constructor itself. Override in - a downstream package to return an enriched subclass — e.g. one carrying an - AST produced by a FormulaParser — without touching the converters. + Subclass and override __call__ to return an enriched Formula subclass, + e.g. one carrying an AST produced by a FormulaParser. + + The *ontology* parameter gives the factory access to the ontology being + built so that name resolution and validation can be performed. """ - def __call__(self, raw_expr: str, parent: FormulaParent = None) -> Formula: ... + def __call__(self, raw_expr: str, parent: FormulaParent = None, ontology: OntologyComponent | None = None) -> Formula: + return Formula(raw_expr=raw_expr, parent=parent) # --------------------------------------------------------------------------- From ff8495d20bfc8cff455f7f1273995af31bc1b69d Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Tue, 23 Jun 2026 14:37:57 +0200 Subject: [PATCH 06/29] Updated OsiParser --- converters/ontology/src/osi/parser/__init__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/converters/ontology/src/osi/parser/__init__.py b/converters/ontology/src/osi/parser/__init__.py index 911407af..f027269f 100644 --- a/converters/ontology/src/osi/parser/__init__.py +++ b/converters/ontology/src/osi/parser/__init__.py @@ -1,4 +1,4 @@ -"""Entrypoint: read a YAML/JSON OSI spec and produce a OsiOntology.""" +"""Entrypoint: read a YAML/JSON OSI spec and produce an OsiOntology.""" from __future__ import annotations @@ -8,7 +8,7 @@ import yaml from osi.converter.spec_to_osi.converter import SpecToOsiConverter -from osi.model import OsiOntology +from osi.model import OsiOntology, FormulaFactory from osi.spec import OsiSpec @@ -17,15 +17,16 @@ class OsiParser: _spec: OsiSpec | None _debug: bool - def __init__(self, debug: bool = False): + def __init__(self, debug: bool = False, formula_factory: FormulaFactory = FormulaFactory()): self._debug = debug self._model = None self._spec = None + self._formula_factory = formula_factory def parse(self, file: IOBase) -> None: raw = OsiParser.load_data(file) self._spec = OsiSpec.model_validate(raw) - self._model = SpecToOsiConverter().convert(self._spec) + self._model = SpecToOsiConverter(formula_factory=self._formula_factory).convert(self._spec) @staticmethod def load_data(file: IOBase): From f74cbc3d17155f6a86fc4c562337881d009719dd Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Tue, 23 Jun 2026 15:32:21 +0200 Subject: [PATCH 07/29] Exclude builtin concepts from top sort --- .../src/osi/converter/spec_to_osi/converter.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index 9f835546..9ec27e1a 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -27,7 +27,8 @@ ReferentMapping, Relationship, RelationshipMultiplicity, - OsiOntology + OsiOntology, + BUILTIN_CONCEPTS ) from osi.spec import ( Concept as SpecConcept, @@ -374,12 +375,15 @@ def _resolve_mapping_expression( @staticmethod def _sort_spec_dependency_graph(concepts: list[SpecConcept]) -> list[str]: - nodes = [concept.name for concept in concepts] + nodes: list[str] = [] edges: list[tuple[str, str]] = [] for concept in concepts: + name = concept.name + nodes.append(name) if concept.extends: for ext in concept.extends: - edges.append((ext, concept.name)) + if ext not in BUILTIN_CONCEPTS: + edges.append((ext, name)) return topological_sort(nodes, edges) From b94c5a9c1efa9f9bbcfe8c2eadeb340860463687 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Tue, 23 Jun 2026 20:58:15 +0200 Subject: [PATCH 08/29] Added parent for mappings --- .../osi/converter/spec_to_osi/converter.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index 9ec27e1a..84b1cc03 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -271,7 +271,8 @@ def _convert_object_mapping( ) expression: DatasetField | Formula | None = None if om_spec.expression is not None: - expression = self._resolve_mapping_expression(om_spec.expression, semantic_model, concept, ontology) + parent = concept if concept is not None else container + expression = self._resolve_mapping_expression(om_spec.expression, parent, semantic_model, concept, ontology) referent_mappings = None if om_spec.referent_mappings is not None: rm_container = concept if concept is not None else container @@ -298,7 +299,7 @@ def _convert_referent_mapping( sibling_player = rel.last_role.player expression: DatasetField | Formula | None = None if rm_spec.expression is not None: - expression = self._resolve_mapping_expression(rm_spec.expression, semantic_model, sibling_player, ontology) + expression = self._resolve_mapping_expression(rm_spec.expression, rel, semantic_model, sibling_player, ontology) nested = None if rm_spec.referent_mappings is not None: nested = [ @@ -315,9 +316,7 @@ def _convert_link_mapping( container: Concept, lm_spec: SpecLinkMapping, ) -> LinkMapping: - object_mapping = self._convert_object_mapping( - model, ontology, semantic_model, container, lm_spec.object_mapping - ) + object_mapping = self._convert_object_mapping(model, ontology, semantic_model, container, lm_spec.object_mapping) relationship: Relationship | None = None if lm_spec.relationship is not None: relationship = ontology.lookup_concept_relationship(container, lm_spec.relationship) @@ -342,9 +341,8 @@ def _build_rule(self, raw: str | None, parent: Container, ontology: OntologyCom return None return self._formula_factory(raw_expr=raw, parent=parent, ontology=ontology) - def _resolve_mapping_expression( - self, expression: str, semantic_model: SemanticModel, expected_type: Concept | None, ontology: OntologyComponent - ) -> DatasetField | Formula: + def _resolve_mapping_expression(self, expression: str, parent: Concept | Relationship, semantic_model: SemanticModel, + expected_type: Concept | None, ontology: OntologyComponent) -> DatasetField | Formula: """Map a raw spec expression onto either a DatasetField (single `DATASET.field` or bare `field` reference) or a Formula (anything else). """ @@ -357,7 +355,7 @@ def _resolve_mapping_expression( if field is not None: _pin_field_type(field, expected_type) return field - return self._formula_factory(raw_expr=expression, ontology=ontology) + return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology) bare = _BARE_FIELD_RE.match(expression) if bare: @@ -367,9 +365,9 @@ def _resolve_mapping_expression( if field is not None: _pin_field_type(field, expected_type) return field - return self._formula_factory(raw_expr=expression, ontology=ontology) + return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology) - return self._formula_factory(raw_expr=expression, ontology=ontology) + return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology) # ----- Structural helpers -------------------------- From d776c2edab863f3eac1e32e330fbd40c36de9de3 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Tue, 23 Jun 2026 21:24:24 +0200 Subject: [PATCH 09/29] Pass SemanticModel parameter to FormulaFactory --- .../src/osi/converter/palantir_to_osi/converter.py | 11 +++++++---- .../src/osi/converter/spec_to_osi/converter.py | 14 +++++++++----- converters/ontology/src/osi/model.py | 5 +++-- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index 45887a11..4f4a8dfc 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -374,14 +374,14 @@ def _convert_relationships( for ir in palantir_ontology.intermediary_relations().values(): if ir.active() or ir.intermediary(): - self._convert_intermediary_relation(ontology, palantir_ontology, ir) + self._convert_intermediary_relation(ontology, palantir_ontology, ir, semantic_model) elif ( ir.experimental() and ir.role_a_player().active() and ir.role_b_player().active() and ir.intermediary_player().active() ): - self._convert_intermediary_relation(ontology, palantir_ontology, ir) + self._convert_intermediary_relation(ontology, palantir_ontology, ir, semantic_model) def _convert_relation( self, @@ -441,7 +441,8 @@ def _convert_many_to_one( for mprop, oprop in rel.property_map().items() ] if frags: - formula = self._formula_factory(raw_expr=" AND ".join(frags), parent=relationship, ontology=ontology) + formula = self._formula_factory(raw_expr=" AND ".join(frags), parent=relationship, ontology=ontology, + semantic_model=semantic_model) relationship.add_derived_by(formula) ontology.add_rule(formula) @@ -586,6 +587,7 @@ def _convert_intermediary_relation( ontology: OntologyComponent, palantir_ontology: PalantirOntology, rel: IntermediaryRelation, + semantic_model: SemanticModel, ) -> None: aot = rel.role_a_player() aot_name = PalantirToOsiConverter._concept_name(aot) @@ -638,7 +640,8 @@ def _convert_intermediary_relation( f"{fp_a}.{rel_a_name}({relationship.first_role.name}) AND " f"{fp_b}.{rel_b_name}({relationship.last_role.name})" ) - formula = self._formula_factory(raw_expr=join_condition, parent=relationship, ontology=ontology) + formula = self._formula_factory(raw_expr=join_condition, parent=relationship, ontology=ontology, + semantic_model=semantic_model) relationship.add_derived_by(formula) ontology.add_rule(formula) diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index 84b1cc03..21f9c103 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -336,10 +336,11 @@ def _convert_link_mapping( # ----- Formula helpers ----------------------------------------------- - def _build_rule(self, raw: str | None, parent: Container, ontology: OntologyComponent) -> Formula | None: + def _build_rule(self, raw: str | None, parent: Container, ontology: OntologyComponent, + semantic_model: SemanticModel | None = None) -> Formula | None: if not raw: return None - return self._formula_factory(raw_expr=raw, parent=parent, ontology=ontology) + return self._formula_factory(raw_expr=raw, parent=parent, ontology=ontology, semantic_model=semantic_model) def _resolve_mapping_expression(self, expression: str, parent: Concept | Relationship, semantic_model: SemanticModel, expected_type: Concept | None, ontology: OntologyComponent) -> DatasetField | Formula: @@ -355,7 +356,8 @@ def _resolve_mapping_expression(self, expression: str, parent: Concept | Relatio if field is not None: _pin_field_type(field, expected_type) return field - return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology) + return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology, + semantic_model=semantic_model) bare = _BARE_FIELD_RE.match(expression) if bare: @@ -365,9 +367,11 @@ def _resolve_mapping_expression(self, expression: str, parent: Concept | Relatio if field is not None: _pin_field_type(field, expected_type) return field - return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology) + return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology, + semantic_model=semantic_model) - return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology) + return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology, + semantic_model=semantic_model) # ----- Structural helpers -------------------------- diff --git a/converters/ontology/src/osi/model.py b/converters/ontology/src/osi/model.py index 9181c923..39ec6fd9 100644 --- a/converters/ontology/src/osi/model.py +++ b/converters/ontology/src/osi/model.py @@ -399,11 +399,12 @@ class FormulaFactory: Subclass and override __call__ to return an enriched Formula subclass, e.g. one carrying an AST produced by a FormulaParser. - The *ontology* parameter gives the factory access to the ontology being + The *ontology* and *semantic_model* parameters give the factory access to the ontology being built so that name resolution and validation can be performed. """ - def __call__(self, raw_expr: str, parent: FormulaParent = None, ontology: OntologyComponent | None = None) -> Formula: + def __call__(self, raw_expr: str, parent: FormulaParent = None, + ontology: OntologyComponent | None = None, semantic_model: SemanticModel | None = None) -> Formula: return Formula(raw_expr=raw_expr, parent=parent) From 0564610386b5bf32d57a2bc411dcc13e452c8578 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 24 Jun 2026 09:54:15 +0200 Subject: [PATCH 10/29] Added MappingFormulaFactory --- .../converter/palantir_to_osi/converter.py | 11 +++----- .../osi/converter/spec_to_osi/converter.py | 28 +++++++++---------- converters/ontology/src/osi/model.py | 16 +++++++++-- 3 files changed, 30 insertions(+), 25 deletions(-) diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index 4f4a8dfc..45887a11 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -374,14 +374,14 @@ def _convert_relationships( for ir in palantir_ontology.intermediary_relations().values(): if ir.active() or ir.intermediary(): - self._convert_intermediary_relation(ontology, palantir_ontology, ir, semantic_model) + self._convert_intermediary_relation(ontology, palantir_ontology, ir) elif ( ir.experimental() and ir.role_a_player().active() and ir.role_b_player().active() and ir.intermediary_player().active() ): - self._convert_intermediary_relation(ontology, palantir_ontology, ir, semantic_model) + self._convert_intermediary_relation(ontology, palantir_ontology, ir) def _convert_relation( self, @@ -441,8 +441,7 @@ def _convert_many_to_one( for mprop, oprop in rel.property_map().items() ] if frags: - formula = self._formula_factory(raw_expr=" AND ".join(frags), parent=relationship, ontology=ontology, - semantic_model=semantic_model) + formula = self._formula_factory(raw_expr=" AND ".join(frags), parent=relationship, ontology=ontology) relationship.add_derived_by(formula) ontology.add_rule(formula) @@ -587,7 +586,6 @@ def _convert_intermediary_relation( ontology: OntologyComponent, palantir_ontology: PalantirOntology, rel: IntermediaryRelation, - semantic_model: SemanticModel, ) -> None: aot = rel.role_a_player() aot_name = PalantirToOsiConverter._concept_name(aot) @@ -640,8 +638,7 @@ def _convert_intermediary_relation( f"{fp_a}.{rel_a_name}({relationship.first_role.name}) AND " f"{fp_b}.{rel_b_name}({relationship.last_role.name})" ) - formula = self._formula_factory(raw_expr=join_condition, parent=relationship, ontology=ontology, - semantic_model=semantic_model) + formula = self._formula_factory(raw_expr=join_condition, parent=relationship, ontology=ontology) relationship.add_derived_by(formula) ontology.add_rule(formula) diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index 21f9c103..beaa2476 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -17,6 +17,7 @@ Dimension, Formula, FormulaFactory, + MappingFormulaFactory, JoinPath, LinkMapping, SemanticModel, @@ -68,8 +69,10 @@ class SpecToOsiConverter: model = SpecToOsiConverter(formula_factory=my_parser).convert(spec) """ - def __init__(self, formula_factory: FormulaFactory = FormulaFactory()): + def __init__(self, formula_factory: FormulaFactory = FormulaFactory(), + mapping_formula_factory: MappingFormulaFactory = MappingFormulaFactory()): self._formula_factory = formula_factory + self._mapping_formula_factory = mapping_formula_factory def convert(self, spec: OsiSpec) -> OsiOntology: ontology = OntologyComponent() @@ -271,8 +274,7 @@ def _convert_object_mapping( ) expression: DatasetField | Formula | None = None if om_spec.expression is not None: - parent = concept if concept is not None else container - expression = self._resolve_mapping_expression(om_spec.expression, parent, semantic_model, concept, ontology) + expression = self._resolve_mapping_expression(om_spec.expression, semantic_model, concept, ontology) referent_mappings = None if om_spec.referent_mappings is not None: rm_container = concept if concept is not None else container @@ -299,7 +301,7 @@ def _convert_referent_mapping( sibling_player = rel.last_role.player expression: DatasetField | Formula | None = None if rm_spec.expression is not None: - expression = self._resolve_mapping_expression(rm_spec.expression, rel, semantic_model, sibling_player, ontology) + expression = self._resolve_mapping_expression(rm_spec.expression, semantic_model, sibling_player, ontology) nested = None if rm_spec.referent_mappings is not None: nested = [ @@ -336,14 +338,13 @@ def _convert_link_mapping( # ----- Formula helpers ----------------------------------------------- - def _build_rule(self, raw: str | None, parent: Container, ontology: OntologyComponent, - semantic_model: SemanticModel | None = None) -> Formula | None: + def _build_rule(self, raw: str | None, parent: Container, ontology: OntologyComponent) -> Formula | None: if not raw: return None - return self._formula_factory(raw_expr=raw, parent=parent, ontology=ontology, semantic_model=semantic_model) + return self._formula_factory(raw_expr=raw, parent=parent, ontology=ontology) - def _resolve_mapping_expression(self, expression: str, parent: Concept | Relationship, semantic_model: SemanticModel, - expected_type: Concept | None, ontology: OntologyComponent) -> DatasetField | Formula: + def _resolve_mapping_expression(self, expression: str, semantic_model: SemanticModel, expected_type: Concept | None, + ontology: OntologyComponent) -> DatasetField | Formula: """Map a raw spec expression onto either a DatasetField (single `DATASET.field` or bare `field` reference) or a Formula (anything else). """ @@ -356,8 +357,7 @@ def _resolve_mapping_expression(self, expression: str, parent: Concept | Relatio if field is not None: _pin_field_type(field, expected_type) return field - return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology, - semantic_model=semantic_model) + return self._mapping_formula_factory(raw_expr=expression, ontology=ontology, semantic_model=semantic_model) bare = _BARE_FIELD_RE.match(expression) if bare: @@ -367,11 +367,9 @@ def _resolve_mapping_expression(self, expression: str, parent: Concept | Relatio if field is not None: _pin_field_type(field, expected_type) return field - return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology, - semantic_model=semantic_model) + return self._mapping_formula_factory(raw_expr=expression, ontology=ontology, semantic_model=semantic_model) - return self._formula_factory(raw_expr=expression, parent=parent, ontology=ontology, - semantic_model=semantic_model) + return self._mapping_formula_factory(raw_expr=expression, ontology=ontology, semantic_model=semantic_model) # ----- Structural helpers -------------------------- diff --git a/converters/ontology/src/osi/model.py b/converters/ontology/src/osi/model.py index 39ec6fd9..a0cab44b 100644 --- a/converters/ontology/src/osi/model.py +++ b/converters/ontology/src/osi/model.py @@ -394,13 +394,23 @@ def __str__(self) -> str: class FormulaFactory: - """Base factory that produces a Formula from a raw expression. + """Base factory for ontology-level formulas (derived_by / requires on concepts and relationships). Subclass and override __call__ to return an enriched Formula subclass, e.g. one carrying an AST produced by a FormulaParser. + """ + + def __call__(self, raw_expr: str, parent: FormulaParent = None, ontology: OntologyComponent | None = None) -> Formula: + return Formula(raw_expr=raw_expr, parent=parent) + + +class MappingFormulaFactory: + """Base factory for mapping-level formulas (object / referent mapping expressions). + + Receives both the ontology and the semantic model so that name resolution + and field-level validation can be performed against the full mapping context. - The *ontology* and *semantic_model* parameters give the factory access to the ontology being - built so that name resolution and validation can be performed. + Subclass and override __call__ to return an enriched Formula subclass. """ def __call__(self, raw_expr: str, parent: FormulaParent = None, From 1b85161fe789775bd34fdbdc9714fc99e06b5675 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 24 Jun 2026 10:25:56 +0200 Subject: [PATCH 11/29] Updated OsiParser --- converters/ontology/src/osi/parser/__init__.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/converters/ontology/src/osi/parser/__init__.py b/converters/ontology/src/osi/parser/__init__.py index f027269f..85690584 100644 --- a/converters/ontology/src/osi/parser/__init__.py +++ b/converters/ontology/src/osi/parser/__init__.py @@ -8,7 +8,7 @@ import yaml from osi.converter.spec_to_osi.converter import SpecToOsiConverter -from osi.model import OsiOntology, FormulaFactory +from osi.model import OsiOntology, FormulaFactory, MappingFormulaFactory from osi.spec import OsiSpec @@ -17,16 +17,21 @@ class OsiParser: _spec: OsiSpec | None _debug: bool - def __init__(self, debug: bool = False, formula_factory: FormulaFactory = FormulaFactory()): + def __init__(self, debug: bool = False, + formula_factory: FormulaFactory = FormulaFactory(), + mapping_formula_factory: MappingFormulaFactory = MappingFormulaFactory()): self._debug = debug self._model = None self._spec = None self._formula_factory = formula_factory + self._mapping_formula_factory = mapping_formula_factory def parse(self, file: IOBase) -> None: raw = OsiParser.load_data(file) self._spec = OsiSpec.model_validate(raw) - self._model = SpecToOsiConverter(formula_factory=self._formula_factory).convert(self._spec) + self._model = SpecToOsiConverter( + formula_factory=self._formula_factory, mapping_formula_factory=self._mapping_formula_factory + ).convert(self._spec) @staticmethod def load_data(file: IOBase): @@ -47,4 +52,4 @@ def model(self) -> OsiOntology: model = self._model if model is None: raise RuntimeError("You must call 'parse()' before accessing 'model()'") - return model \ No newline at end of file + return model From 774a3908306916309c8c85169305ee3b57f35472 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 24 Jun 2026 16:31:58 +0200 Subject: [PATCH 12/29] Updated OsiToSpecConverter to emit all non-built-in concepts. --- .../src/osi/converter/osi_to_spec/converter.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/converters/ontology/src/osi/converter/osi_to_spec/converter.py b/converters/ontology/src/osi/converter/osi_to_spec/converter.py index 6bc66562..3e1b833d 100644 --- a/converters/ontology/src/osi/converter/osi_to_spec/converter.py +++ b/converters/ontology/src/osi/converter/osi_to_spec/converter.py @@ -71,15 +71,14 @@ def convert(model: OsiOntology) -> OsiSpec: def _convert_ontology_concepts(ont: OntologyComponent) -> list[ConceptComponent]: components: list[ConceptComponent] = [] - for concept in ont.concepts(): + for concept in ont.concepts(exclude_builtin=True): rels = [rel for rel in ont.relationships if rel.container is concept] - if rels: - components.append( - ConceptComponent( - concept=_convert_concept(concept), - relationships=[_convert_relationship(rel) for rel in rels], - ) + components.append( + ConceptComponent( + concept=_convert_concept(concept), + relationships=[_convert_relationship(rel) for rel in rels], ) + ) return components From f0369dd379607a360621ba1cfc850a46ec7c5e9c Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Thu, 25 Jun 2026 16:09:55 +0200 Subject: [PATCH 13/29] Emmit only component concepts --- .../ontology/src/osi/converter/osi_to_spec/converter.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/converters/ontology/src/osi/converter/osi_to_spec/converter.py b/converters/ontology/src/osi/converter/osi_to_spec/converter.py index 3e1b833d..75ea58d1 100644 --- a/converters/ontology/src/osi/converter/osi_to_spec/converter.py +++ b/converters/ontology/src/osi/converter/osi_to_spec/converter.py @@ -71,7 +71,9 @@ def convert(model: OsiOntology) -> OsiSpec: def _convert_ontology_concepts(ont: OntologyComponent) -> list[ConceptComponent]: components: list[ConceptComponent] = [] - for concept in ont.concepts(exclude_builtin=True): + for concept in ont.concepts(): + if not concept.is_component: + continue rels = [rel for rel in ont.relationships if rel.container is concept] components.append( ConceptComponent( From 402efde1ee62fa0791d0649efc40080d49a00534 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Thu, 25 Jun 2026 16:41:13 +0200 Subject: [PATCH 14/29] Updated Palantir converter to handle cases where no identifying fields are found --- .../osi/converter/palantir_to_osi/converter.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index 45887a11..33ce56c9 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -291,6 +291,13 @@ def resolve(p: PalantirProperty) -> PalantirProperty: continue id_referents.append(ReferentMapping(relationship=rel, expression=field)) + if not id_referents: + warnings.warn( + f"No identifying fields found for concept '{concept.name}' in dataset " + f"'{dataset.name}'; skipping concept mapping." + ) + continue + cm = ConceptMapping(concept=concept) # object_mappings: how to construct/identify this concept's @@ -299,7 +306,7 @@ def resolve(p: PalantirProperty) -> PalantirProperty: cm.object_mappings.append( ObjectMapping( concept=parent_concept, - referent_mappings=list(id_referents) if id_referents else None, + referent_mappings=id_referents, ) ) @@ -332,19 +339,19 @@ def resolve(p: PalantirProperty) -> PalantirProperty: value_concept = relationship.last_role.player children.append( LinkMapping( - object_mapping=ObjectMapping(concept=value_concept,expression=field), + object_mapping=ObjectMapping(concept=value_concept, expression=field), relationship=relationship, ) ) - if id_referents or children: + if children: cm.link_mappings.append( LinkMapping( object_mapping=ObjectMapping( concept=parent_concept, - referent_mappings=list(id_referents) if id_referents else None, + referent_mappings=id_referents, ), - children=children if children else None, + children=children, ) ) From 29cfb36f69000e3953529da7f9f9a3a029e75249 Mon Sep 17 00:00:00 2001 From: Kurt Stirewalt Date: Thu, 2 Jul 2026 15:33:06 -0400 Subject: [PATCH 15/29] Added emission of ontology-level requires --- converters/ontology/src/osi/converter/osi_to_spec/converter.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/converters/ontology/src/osi/converter/osi_to_spec/converter.py b/converters/ontology/src/osi/converter/osi_to_spec/converter.py index 75ea58d1..f73d6f3b 100644 --- a/converters/ontology/src/osi/converter/osi_to_spec/converter.py +++ b/converters/ontology/src/osi/converter/osi_to_spec/converter.py @@ -60,6 +60,7 @@ def convert(model: OsiOntology) -> OsiSpec: name=model.name, description=model.description, ai_context=model.ai_context, + requires=[f.raw_expr for f in ont.requires], ontology=_convert_ontology_concepts(ont), ontology_mappings=ontology_mappings, ) @@ -269,4 +270,4 @@ def _convert_link_mapping(link_mapping: LinkMapping) -> SpecLinkMapping: object_mapping=_convert_object_mapping(link_mapping.object_mapping), relationship=link_mapping.relationship.name if link_mapping.relationship is not None else None, children=children, - ) \ No newline at end of file + ) From 90b8154063aeb6cd317fe315a67db4af147df315 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Mon, 20 Jul 2026 18:31:59 +0200 Subject: [PATCH 16/29] Updated dump_yaml method --- converters/ontology/src/osi/spec.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/converters/ontology/src/osi/spec.py b/converters/ontology/src/osi/spec.py index 68e75dd4..b21b28d2 100644 --- a/converters/ontology/src/osi/spec.py +++ b/converters/ontology/src/osi/spec.py @@ -236,7 +236,7 @@ def dump_dict(self) -> dict: return self.model_dump(exclude_none=True, exclude_defaults=True, by_alias=True) def dump_yaml(self) -> str: - return yaml.safe_dump(self.dump_dict(), sort_keys=False) + return yaml.safe_dump(self.dump_dict(), sort_keys=False, width=float("inf")) # `ReferentMapping` and `LinkMapping` are self-referential (each can contain a From 75fff33fe2e5a8099d3fee595bd55c97fa629ebb Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Tue, 21 Jul 2026 16:53:50 +0200 Subject: [PATCH 17/29] Updated parsers interface + convert top level spec requires --- converters/ontology/README.md | 4 +- .../ontology/scripts/palantir_to_osi.py | 16 +- .../ontology/src/osi/common/file_utils.py | 52 ++++++- .../converter/palantir_to_osi/converter.py | 2 - .../osi/converter/spec_to_osi/converter.py | 13 +- .../osi/external/palantir/parser/__init__.py | 137 +++++++++++------- converters/ontology/src/osi/model.py | 9 -- .../ontology/src/osi/parser/__init__.py | 18 ++- 8 files changed, 167 insertions(+), 84 deletions(-) diff --git a/converters/ontology/README.md b/converters/ontology/README.md index 0e09873c..94dfbf4c 100644 --- a/converters/ontology/README.md +++ b/converters/ontology/README.md @@ -62,12 +62,14 @@ from osi.converter.spec_to_osi.converter import SpecToOsiConverter ### `scripts/palantir_to_osi.py` -Converts a Palantir ontology export (`.zip` file containing a Palantir ontology JSON and one or more dataset spec JSON files) into an OSI-compliant YAML representation, printed to stdout. +Converts a Palantir ontology export (a `.zip` archive or an already extracted folder containing a Palantir ontology JSON and a `data_sets` folder of one or more dataset spec JSON files) into an OSI-compliant YAML representation, printed to stdout. **Usage:** ```bash python scripts/palantir_to_osi.py path/to/palantir_export.zip +# or an extracted folder: +python scripts/palantir_to_osi.py path/to/palantir_export/ ``` Warnings are written to stderr; the OSI YAML is written to stdout. diff --git a/converters/ontology/scripts/palantir_to_osi.py b/converters/ontology/scripts/palantir_to_osi.py index c1ed4242..0840bd78 100644 --- a/converters/ontology/scripts/palantir_to_osi.py +++ b/converters/ontology/scripts/palantir_to_osi.py @@ -1,14 +1,15 @@ # Description: # -# This script converts a zip file that contains: +# This script converts a Palantir ontology export into an OSI compliant YAML +# representation of that ontology, using environment variables to configure the +# Snowflake database and schema names. The export may be supplied either as a +# zip archive or as an already extracted folder, and must contain: # 1. A Palantir ontology (JSON file) and -# 2. A folder containing one or more Palantir dataset specs (JSON files) -# into an OSI compliant YAML representation of that ontology, using environment -# variables to configure the Snowflake database and schema names. +# 2. A 'data_sets' folder containing one or more Palantir dataset specs (JSON files) # # Usage: # -# $ python palantir_to_osi.py +# $ python palantir_to_osi.py # # Environment variables used: # @@ -43,10 +44,7 @@ path = Path(sys.argv[1]) parser = PalantirParser() - - mode = "rb" if path.suffix.lower() == ".zip" else "r" - with open(path, mode) as file: - parser.parse(file) + parser.parse(path) ontology_model = PalantirToOsiConverter().convert(parser.model(), db_name, schema_name) diff --git a/converters/ontology/src/osi/common/file_utils.py b/converters/ontology/src/osi/common/file_utils.py index 99590524..d9f70896 100644 --- a/converters/ontology/src/osi/common/file_utils.py +++ b/converters/ontology/src/osi/common/file_utils.py @@ -1,5 +1,6 @@ import io import zipfile +from pathlib import Path from typing import Iterable @@ -39,4 +40,53 @@ def _open_to_io_base(n: str) -> io.IOBase: if candidate in names: return _open_to_io_base(candidate) - raise FileNotFoundError(f"Missing required top-level file: {filename}") \ No newline at end of file + raise FileNotFoundError(f"Missing required top-level file: {filename}") + + +# The directory helpers below mirror the ZIP helpers above, but operate on an +# extracted folder rather than a ZIP archive. Like the ZIP variants, they accept +# either a folder whose top level directly contains the Palantir files, or a +# folder wrapping a single root directory that contains them. + +def _resolve_dir_root(base_dir: Path) -> Path: + """Return the directory that holds the top-level Palantir files. + + Either ``base_dir`` itself, or its single child directory when the export + was extracted under one wrapping folder (mirrors the single-root handling + used for ZIP archives). + """ + entries = list(base_dir.iterdir()) + if len(entries) == 1 and entries[0].is_dir(): + return entries[0] + return base_dir + + +def iter_json_files_from_dir(base_dir: Path, dir_prefix: str) -> Iterable[tuple[str, io.IOBase]]: + root = _resolve_dir_root(base_dir) + data_dir = root / dir_prefix.rstrip("/") + if not data_dir.is_dir(): + return + for path in sorted(data_dir.rglob("*")): + if path.is_file() and path.suffix.lower() == ".json": + with path.open("rb") as fp: + yield str(path), io.BytesIO(fp.read()) + + +def get_top_level_json_file_from_dir(base_dir: Path) -> Path: + root = _resolve_dir_root(base_dir) + candidates = [ + p for p in sorted(root.iterdir()) + if p.is_file() and p.suffix.lower() == ".json" + ] + if len(candidates) == 0: + raise FileNotFoundError("Directory must contain exactly one top-level JSON file (none found)") + if len(candidates) > 1: + raise ValueError("Directory must contain exactly one top-level JSON file (multiple found)") + return candidates[0] + + +def validate_dir(base_dir: Path) -> None: + """Ensure the extracted folder contains a required 'data_sets/' directory.""" + root = _resolve_dir_root(base_dir) + if not (root / "data_sets").is_dir(): + raise ValueError("Directory does not contain required 'data_sets' folder") \ No newline at end of file diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index 33ce56c9..3cc50d89 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -450,7 +450,6 @@ def _convert_many_to_one( if frags: formula = self._formula_factory(raw_expr=" AND ".join(frags), parent=relationship, ontology=ontology) relationship.add_derived_by(formula) - ontology.add_rule(formula) def _attach_link_to_concept_mappings( self, @@ -647,7 +646,6 @@ def _convert_intermediary_relation( ) formula = self._formula_factory(raw_expr=join_condition, parent=relationship, ontology=ontology) relationship.add_derived_by(formula) - ontology.add_rule(formula) # ------------------------------------------------------------------ # Datasets diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index beaa2476..92aeae00 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -151,12 +151,10 @@ def _populate_ontology(self, ontology: OntologyComponent, spec: OsiSpec) -> None req = self._build_rule(raw, concept, ontology) if req: concept.add_require(req) - ontology.add_require(req) for raw in concept_spec.derived_by: rule = self._build_rule(raw, concept, ontology) if rule: concept.add_derived_by(rule) - ontology.add_rule(rule) for rel_spec in concept_component.relationships: rel = ontology.lookup_concept_relationship(concept, rel_spec.name) if rel is None: @@ -165,12 +163,17 @@ def _populate_ontology(self, ontology: OntologyComponent, spec: OsiSpec) -> None req = self._build_rule(raw, rel, ontology) if req: rel.add_require(req) - ontology.add_require(req) for raw in rel_spec.derived_by: rule = self._build_rule(raw, rel, ontology) if rule: rel.add_derived_by(rule) - ontology.add_rule(rule) + + # Ontology-level requires are not scoped to any concept/relationship; + # they attach directly to the ontology component. + for raw in spec.requires: + req = self._build_rule(raw, None, ontology) + if req: + ontology.add_require(req) def _convert_relationship( self, ontology: OntologyComponent, container: Concept, rel_spec: SpecRelationship @@ -338,7 +341,7 @@ def _convert_link_mapping( # ----- Formula helpers ----------------------------------------------- - def _build_rule(self, raw: str | None, parent: Container, ontology: OntologyComponent) -> Formula | None: + def _build_rule(self, raw: str | None, parent: Container | None, ontology: OntologyComponent) -> Formula | None: if not raw: return None return self._formula_factory(raw_expr=raw, parent=parent, ontology=ontology) diff --git a/converters/ontology/src/osi/external/palantir/parser/__init__.py b/converters/ontology/src/osi/external/palantir/parser/__init__.py index 30b31d57..2dac186e 100644 --- a/converters/ontology/src/osi/external/palantir/parser/__init__.py +++ b/converters/ontology/src/osi/external/palantir/parser/__init__.py @@ -1,14 +1,15 @@ -import io import json import warnings import zipfile from io import IOBase -from typing import Any +from pathlib import Path +from typing import Any, Iterable from osi.common.utils import camel_to_snake from osi.external.palantir.model import DataSet, DataSetColumn, DataSetModel, ObjectType, Ontology, DataType, \ ArrayDataType, Property, Status, ManyToOneRelation, Relation, ManyToManyRelation, IntermediaryRelation, DataSource -from osi.common.file_utils import iter_json_files_from_dir_in_zip, open_top_level_file_from_zip +from osi.common.file_utils import iter_json_files_from_dir_in_zip, open_top_level_file_from_zip, \ + iter_json_files_from_dir, get_top_level_json_file_from_dir, validate_dir # Helper functions to aid in parsing. Palantir's JSON exports can be inconsistent in their formatting, especially @@ -554,25 +555,67 @@ def model(self): def _make_ontology_parser(self) -> PalantirOntologyParser: return PalantirOntologyParser() - def parse(self, file: IOBase): - raw = file.buffer if isinstance(file, io.TextIOWrapper) else file - - # Read all bytes and detect ZIP - data = raw.read() - bio = io.BytesIO(data) - if not zipfile.is_zipfile(bio): - raise ValueError("Unsupported archive format. Expected ZIP") - - bio.seek(0) - with zipfile.ZipFile(bio) as zf: - self._parse_from_zip(zf) + def parse(self, path: Path) -> Ontology: + # A Palantir export may arrive as a ZIP archive, an already extracted + # folder, or a folder that simply wraps a single ZIP archive. Detect + # which and process accordingly. + if path.is_dir(): + wrapped_zip = self._single_zip_in_dir(path) + if wrapped_zip is not None: + with zipfile.ZipFile(wrapped_zip) as zf: + self._parse_from_zip(zf) + else: + self._parse_from_dir(path) + elif path.is_file(): + if not zipfile.is_zipfile(path): + raise ValueError(f"Unsupported Palantir source '{path}'. Expected a ZIP archive or a directory") + with zipfile.ZipFile(path) as zf: + self._parse_from_zip(zf) + else: + raise FileNotFoundError(f"Palantir source '{path}' does not exist") + return self.model() + + @staticmethod + def _single_zip_in_dir(base_dir: Path) -> Path | None: + """Return the lone ZIP archive inside ``base_dir`` when the folder wraps + exactly one ``.zip`` file (and nothing else); otherwise ``None``. This + mirrors the convenience of accepting a folder that just contains a + Palantir export archive.""" + entries = list(base_dir.iterdir()) + if len(entries) == 1 and entries[0].is_file() and zipfile.is_zipfile(entries[0]): + return entries[0] + return None def _parse_from_zip(self, zf: zipfile.ZipFile): self._validate_archive(zf) + def _data_set_streams() -> Iterable[IOBase]: + for _name, fh in iter_json_files_from_dir_in_zip(zf, "data_sets"): + yield fh + + try: + with open_top_level_file_from_zip(zf, self._get_ontology_json_file_path(zf)) as ontology_fh: + self._build_model(_data_set_streams(), ontology_fh) + except FileNotFoundError as e: + raise FileNotFoundError(str(e)) from e + + def _parse_from_dir(self, base_dir: Path): + validate_dir(base_dir) + ontology_path = get_top_level_json_file_from_dir(base_dir) + + def _data_set_streams() -> Iterable[IOBase]: + for _name, fh in iter_json_files_from_dir(base_dir, "data_sets"): + yield fh + + with ontology_path.open("rb") as ontology_fh: + self._build_model(_data_set_streams(), ontology_fh) + + def _build_model(self, data_set_streams: Iterable[IOBase], ontology_stream: IOBase): + """Build the ontology model from a stream of data set JSON files and the + top-level ontology JSON stream. Shared by the ZIP and directory paths.""" any_json = False data_sets: dict[str, DataSet] = {} - for name, fh in iter_json_files_from_dir_in_zip(zf, "data_sets"): + for fh in data_set_streams: any_json = True try: parser = PalantirDataSetParser() @@ -583,39 +626,35 @@ def _parse_from_zip(self, zf: zipfile.ZipFile): if not any_json: raise ValueError("'data_sets' folder contains no JSON files") - try: - with open_top_level_file_from_zip(zf, self._get_ontology_json_file_path(zf)) as fh: - parser = self._make_ontology_parser() - parser.parse(fh) - model = parser.model() - model.set_data_sets(data_sets) - - for ot in model.object_types().values(): - for ds in ot.data_sources(): - data_set = data_sets.get(ds.backing_dataset_id(), None) - if data_set is None: - # For SDK-extracted ontologies with synthetic datasources, - # mainDatasetId in data_sets JSON matches the object type's RID - data_set = data_sets.get(ot.guid(), None) - if data_set: - ot.sync_from_data_set(data_set) - # For SDK-extracted ontologies, property column_name defaults to - # the apiName (camelCase), but dataset columns use snake_case. - # Cross-reference to use the actual dataset column names. - ds_col_names = {col.name() for col in data_set.columns()} - for prop in ot.properties().values(): - col_name = prop.column_name() - if col_name not in ds_col_names: - snake_name = camel_to_snake(col_name) - if snake_name in ds_col_names: - prop._column_name = snake_name - for rel in model.relations().values(): - if isinstance(rel, ManyToManyRelation): - rel._data_set = data_sets.get(rel.backing_dataset_id(), None) - - self._model = model - except FileNotFoundError as e: - raise FileNotFoundError(str(e)) from e + parser = self._make_ontology_parser() + parser.parse(ontology_stream) + model = parser.model() + model.set_data_sets(data_sets) + + for ot in model.object_types().values(): + for ds in ot.data_sources(): + data_set = data_sets.get(ds.backing_dataset_id(), None) + if data_set is None: + # For SDK-extracted ontologies with synthetic datasources, + # mainDatasetId in data_sets JSON matches the object type's RID + data_set = data_sets.get(ot.guid(), None) + if data_set: + ot.sync_from_data_set(data_set) + # For SDK-extracted ontologies, property column_name defaults to + # the apiName (camelCase), but dataset columns use snake_case. + # Cross-reference to use the actual dataset column names. + ds_col_names = {col.name() for col in data_set.columns()} + for prop in ot.properties().values(): + col_name = prop.column_name() + if col_name not in ds_col_names: + snake_name = camel_to_snake(col_name) + if snake_name in ds_col_names: + prop._column_name = snake_name + for rel in model.relations().values(): + if isinstance(rel, ManyToManyRelation): + rel._data_set = data_sets.get(rel.backing_dataset_id(), None) + + self._model = model def _validate_archive(self, zf: zipfile.ZipFile): """ diff --git a/converters/ontology/src/osi/model.py b/converters/ontology/src/osi/model.py index a0cab44b..98f9cbd1 100644 --- a/converters/ontology/src/osi/model.py +++ b/converters/ontology/src/osi/model.py @@ -881,7 +881,6 @@ class OntologyComponent: Document-level metadata (name, description, ai_context) lives on OsiOntology.""" _concepts: list[Concept] _relationships: list[Relationship] - _rules: list[Formula] _requires: list[Formula] _concept_name_map: dict[str, Concept] _relationship_name_map: dict[str, Relationship] @@ -890,7 +889,6 @@ class OntologyComponent: def __init__(self): self._concepts = [] self._relationships = [] - self._rules = [] self._requires = [] self._concept_name_map = {} self._relationship_name_map = {} @@ -918,9 +916,6 @@ def add_relationship(self, relationship: Relationship) -> None: self._relationships.append(relationship) self._relationship_name_map[full_name] = relationship - def add_rule(self, rule: Formula) -> None: - self._rules.append(rule) - def add_require(self, require: Formula) -> None: self._requires.append(require) for obs in self._observers: @@ -935,10 +930,6 @@ def concepts(self, exclude_builtin: bool = False) -> list[Concept]: def relationships(self) -> list[Relationship]: return list(self._relationships) - @property - def rules(self) -> list[Formula]: - return list(self._rules) - @property def requires(self) -> list[Formula]: return list(self._requires) diff --git a/converters/ontology/src/osi/parser/__init__.py b/converters/ontology/src/osi/parser/__init__.py index 85690584..1f95d943 100644 --- a/converters/ontology/src/osi/parser/__init__.py +++ b/converters/ontology/src/osi/parser/__init__.py @@ -3,7 +3,7 @@ from __future__ import annotations import json -from io import IOBase +from pathlib import Path import yaml @@ -26,19 +26,21 @@ def __init__(self, debug: bool = False, self._formula_factory = formula_factory self._mapping_formula_factory = mapping_formula_factory - def parse(self, file: IOBase) -> None: - raw = OsiParser.load_data(file) + def parse(self, path: Path) -> OsiOntology: + # OSI always expects a single spec file. + if not path.is_file(): + raise ValueError(f"Expected a single OSI spec file, but '{path}' is not a file") + raw = OsiParser.load_data(path) self._spec = OsiSpec.model_validate(raw) self._model = SpecToOsiConverter( formula_factory=self._formula_factory, mapping_formula_factory=self._mapping_formula_factory ).convert(self._spec) + return self._model @staticmethod - def load_data(file: IOBase): - content = file.read() - file.seek(0) - name = (getattr(file, "name", "") or "").lower() - if name.endswith(".json"): + def load_data(path: Path): + content = path.read_text() + if path.suffix.lower() == ".json": return json.loads(content) return yaml.safe_load(content) From 53bdca72323cbf9fe674be71a21dd7377b49f1b8 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Tue, 21 Jul 2026 17:36:57 +0200 Subject: [PATCH 18/29] Refactored after the Copilot code review --- converters/ontology/pyproject.toml | 3 +- .../ontology/scripts/palantir_to_osi.py | 2 +- .../osi/converter/spec_to_osi/converter.py | 20 +++++++++---- .../src/osi/external/palantir/model.py | 2 +- .../osi/external/palantir/parser/__init__.py | 28 +++++++++---------- .../ontology/src/osi/parser/__init__.py | 6 ---- 6 files changed, 32 insertions(+), 29 deletions(-) diff --git a/converters/ontology/pyproject.toml b/converters/ontology/pyproject.toml index 6a881478..da73d30d 100644 --- a/converters/ontology/pyproject.toml +++ b/converters/ontology/pyproject.toml @@ -26,4 +26,5 @@ where = ["src"] [tool.pytest.ini_options] testpaths = ["tests"] -pythonVersion = "3.11" \ No newline at end of file +[tool.pyright] +pythonVersion = "3.11" diff --git a/converters/ontology/scripts/palantir_to_osi.py b/converters/ontology/scripts/palantir_to_osi.py index 0840bd78..018629a3 100644 --- a/converters/ontology/scripts/palantir_to_osi.py +++ b/converters/ontology/scripts/palantir_to_osi.py @@ -39,7 +39,7 @@ schema_name = os.environ.get("SNOWFLAKE_SCHEMA_NAME", "PALANTIR") if len(sys.argv) != 2: - raise Exception(f"++ Usage: {sys.argv[0]} path to Palantir sources") + sys.exit(f"Usage: {sys.argv[0]} ") path = Path(sys.argv[1]) diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index 92aeae00..43381ba9 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -365,11 +365,21 @@ def _resolve_mapping_expression(self, expression: str, semantic_model: SemanticM bare = _BARE_FIELD_RE.match(expression) if bare: field_name = bare.group(1) - for dataset in semantic_model.datasets: - field = dataset.field(field_name) - if field is not None: - _pin_field_type(field, expected_type) - return field + matches = [ + (dataset, field) + for dataset in semantic_model.datasets + if (field := dataset.field(field_name)) is not None + ] + if len(matches) > 1: + owners = ", ".join(dataset.name for dataset, _ in matches) + raise ValueError( + f"Bare field reference '{field_name}' is ambiguous: it exists in multiple " + f"datasets ({owners}). Qualify it as 'DATASET.{field_name}'." + ) + if matches: + _, field = matches[0] + _pin_field_type(field, expected_type) + return field return self._mapping_formula_factory(raw_expr=expression, ontology=ontology, semantic_model=semantic_model) return self._mapping_formula_factory(raw_expr=expression, ontology=ontology, semantic_model=semantic_model) diff --git a/converters/ontology/src/osi/external/palantir/model.py b/converters/ontology/src/osi/external/palantir/model.py index 82a9b340..7db5eae3 100644 --- a/converters/ontology/src/osi/external/palantir/model.py +++ b/converters/ontology/src/osi/external/palantir/model.py @@ -627,6 +627,6 @@ def relation_b(self): def info(self): return (f'Relation "{self.readable_id()}" associates "{self.role_a_player()._name}" with ' - f'"{self.role_b_player()._name} via intermediary player "{self.intermediary_player()._name}" and ' + f'"{self.role_b_player()._name}" via intermediary player "{self.intermediary_player()._name}" and ' f'relations "{self.relation_a()}" and "{self.relation_b()}"') diff --git a/converters/ontology/src/osi/external/palantir/parser/__init__.py b/converters/ontology/src/osi/external/palantir/parser/__init__.py index 2dac186e..fd51c7c1 100644 --- a/converters/ontology/src/osi/external/palantir/parser/__init__.py +++ b/converters/ontology/src/osi/external/palantir/parser/__init__.py @@ -138,16 +138,16 @@ def parse(self, file: IOBase): # Given a Raw Palantir ObjectType, extract the string to use as its name regardless # of JSON convention def _parse_object_type_name(self, raw_ot): - # Newer JSONs contain a displayMetadata section with this information - display_metadata = get_dict(raw_ot, "displayMetadata") - if display_metadata: - ot_name = norm(display_metadata.get("displayName")) - else: - ot_name = norm(raw_ot.get("displayName")) + # Newer JSONs contain a displayMetadata section with this information + display_metadata = get_dict(raw_ot, "displayMetadata") + if display_metadata: + ot_name = norm(display_metadata.get("displayName")) + else: + ot_name = norm(raw_ot.get("displayName")) - if ot_name is None: - raise ValueError(f'Could not extract a name from ObjectType with rid: {raw_ot.get("rid")}') - return ot_name + if ot_name is None: + raise ValueError(f'Could not extract a name from ObjectType with rid: {raw_ot.get("rid")}') + return ot_name def _parse_property_backing_data(self, raw_prop, property_id, object_type_id): # In the new exports this information stores in the `source` field, but in the old exports @@ -545,12 +545,10 @@ def _parse_extra(self, data: dict) -> None: """ class PalantirParser: - _model: Ontology + _model: Ontology | None - def model(self): - if self._model is None: - raise RuntimeError("You must call 'parse()' first before calling 'model()'") - return self._model + def __init__(self): + self._model = None def _make_ontology_parser(self) -> PalantirOntologyParser: return PalantirOntologyParser() @@ -573,7 +571,7 @@ def parse(self, path: Path) -> Ontology: self._parse_from_zip(zf) else: raise FileNotFoundError(f"Palantir source '{path}' does not exist") - return self.model() + return self._model @staticmethod def _single_zip_in_dir(base_dir: Path) -> Path | None: diff --git a/converters/ontology/src/osi/parser/__init__.py b/converters/ontology/src/osi/parser/__init__.py index 1f95d943..9f560d61 100644 --- a/converters/ontology/src/osi/parser/__init__.py +++ b/converters/ontology/src/osi/parser/__init__.py @@ -49,9 +49,3 @@ def spec(self) -> OsiSpec: if spec is None: raise RuntimeError("You must call 'parse()' before accessing 'spec()'") return spec - - def model(self) -> OsiOntology: - model = self._model - if model is None: - raise RuntimeError("You must call 'parse()' before accessing 'model()'") - return model From 9b938b10f5caa1499c42cbfc39223d5d23d43ed5 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Tue, 21 Jul 2026 17:56:35 +0200 Subject: [PATCH 19/29] Refactored after the Copilot code review --- converters/ontology/pyproject.toml | 3 --- converters/ontology/requirements.lock | 6 +++--- converters/ontology/scripts/palantir_to_osi.py | 5 ++--- .../ontology/src/osi/converter/palantir_to_osi/converter.py | 4 +++- .../ontology/src/osi/external/palantir/parser/__init__.py | 4 ++++ 5 files changed, 12 insertions(+), 10 deletions(-) diff --git a/converters/ontology/pyproject.toml b/converters/ontology/pyproject.toml index da73d30d..485340aa 100644 --- a/converters/ontology/pyproject.toml +++ b/converters/ontology/pyproject.toml @@ -23,8 +23,5 @@ dev = [ [tool.setuptools.packages.find] where = ["src"] -[tool.pytest.ini_options] -testpaths = ["tests"] - [tool.pyright] pythonVersion = "3.11" diff --git a/converters/ontology/requirements.lock b/converters/ontology/requirements.lock index 39f9e31d..2b69d8bf 100644 --- a/converters/ontology/requirements.lock +++ b/converters/ontology/requirements.lock @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.12 +# This file is autogenerated by pip-compile with Python 3.11 # by the following command: # # pip-compile --output-file=requirements.lock pyproject.toml @@ -7,11 +7,11 @@ annotated-types==0.7.0 # via pydantic pydantic==2.13.4 - # via osi-ontology-converters (pyproject.toml) + # via ontology (pyproject.toml) pydantic-core==2.46.4 # via pydantic pyyaml==6.0.3 - # via osi-ontology-converters (pyproject.toml) + # via ontology (pyproject.toml) typing-extensions==4.15.0 # via # pydantic diff --git a/converters/ontology/scripts/palantir_to_osi.py b/converters/ontology/scripts/palantir_to_osi.py index 018629a3..1dcea201 100644 --- a/converters/ontology/scripts/palantir_to_osi.py +++ b/converters/ontology/scripts/palantir_to_osi.py @@ -44,10 +44,9 @@ path = Path(sys.argv[1]) parser = PalantirParser() - parser.parse(path) + palantir_model = parser.parse(path) - ontology_model = PalantirToOsiConverter().convert(parser.model(), db_name, schema_name) + ontology_model = PalantirToOsiConverter().convert(palantir_model, db_name, schema_name) osi_spec = OsiToSpecConverter.convert(ontology_model) print(osi_spec.dump_yaml()) - diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index 3cc50d89..c5d1f542 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -174,7 +174,9 @@ def _convert_object_type( if not is_subtype or ignore_subtype: identifiers: dict[str, Relationship] = {} - for prop in ot.primary_keys(): + # primary_keys() is a set; sort by readable_id so identify_by + # ordering (and the resulting YAML) is stable across runs. + for prop in sorted(ot.primary_keys(), key=lambda p: p.readable_id()): prop_name = PalantirToOsiConverter._attribute_name(prop) rel = ontology.lookup_concept_relationship(concept, prop_name) if rel is None: diff --git a/converters/ontology/src/osi/external/palantir/parser/__init__.py b/converters/ontology/src/osi/external/palantir/parser/__init__.py index fd51c7c1..25cb79ce 100644 --- a/converters/ontology/src/osi/external/palantir/parser/__init__.py +++ b/converters/ontology/src/osi/external/palantir/parser/__init__.py @@ -571,6 +571,10 @@ def parse(self, path: Path) -> Ontology: self._parse_from_zip(zf) else: raise FileNotFoundError(f"Palantir source '{path}' does not exist") + + if self._model is None: + raise RuntimeError(f"Parsing '{path}' did not produce an ontology model") + return self._model @staticmethod From 474995691aa33869f86f5624c80b9046dbf353b5 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Tue, 21 Jul 2026 18:21:27 +0200 Subject: [PATCH 20/29] Refactored after the Copilot code review --- .../converter/palantir_to_osi/converter.py | 4 +-- .../osi/converter/spec_to_osi/converter.py | 8 +++--- .../osi/external/palantir/parser/__init__.py | 28 ++++++++++++------- .../ontology/src/osi/parser/__init__.py | 8 +++--- converters/ontology/src/osi/spec.py | 3 +- 5 files changed, 30 insertions(+), 21 deletions(-) diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index c5d1f542..19dba400 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -57,8 +57,8 @@ class PalantirToOsiConverter: depths_role_names = {1: "fst", 2: "snd", 3: "thd", 4: "frt"} - def __init__(self, formula_factory: FormulaFactory = FormulaFactory()): - self._formula_factory = formula_factory + def __init__(self, formula_factory: FormulaFactory | None = None): + self._formula_factory = formula_factory or FormulaFactory() # ------------------------------------------------------------------ # Entry point diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index 43381ba9..0b709c3e 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -69,10 +69,10 @@ class SpecToOsiConverter: model = SpecToOsiConverter(formula_factory=my_parser).convert(spec) """ - def __init__(self, formula_factory: FormulaFactory = FormulaFactory(), - mapping_formula_factory: MappingFormulaFactory = MappingFormulaFactory()): - self._formula_factory = formula_factory - self._mapping_formula_factory = mapping_formula_factory + def __init__(self, formula_factory: FormulaFactory | None = None, + mapping_formula_factory: MappingFormulaFactory | None = None): + self._formula_factory = formula_factory or FormulaFactory() + self._mapping_formula_factory = mapping_formula_factory or MappingFormulaFactory() def convert(self, spec: OsiSpec) -> OsiOntology: ontology = OntologyComponent() diff --git a/converters/ontology/src/osi/external/palantir/parser/__init__.py b/converters/ontology/src/osi/external/palantir/parser/__init__.py index 25cb79ce..b599b3d6 100644 --- a/converters/ontology/src/osi/external/palantir/parser/__init__.py +++ b/converters/ontology/src/osi/external/palantir/parser/__init__.py @@ -168,6 +168,10 @@ def _parse_object_types(self, data: dict) -> tuple[dict[str, ObjectType], dict[s raise ValueError("Object type `rid` field must be non-empty") # Support both formats: new (id) and old (apiName) readable_id = norm(raw_ot.get("id")) or norm(raw_ot.get("apiName")) + if not readable_id: + raise ValueError( + f"Object type must have a non-empty `id` or `apiName` (rid: {guid})" + ) # Extract the ObjectType's name ot_name = self._parse_object_type_name(raw_ot) @@ -409,11 +413,12 @@ def _parse_many_to_one_relation(self, guid: str, id: str, raw: dict[Any, Any], o property_map: dict[Property, Property] = {} for k, v in one_to_many_mapping.items(): - try: - one_property = one_object_type.properties()[k] - many_property = many_object_type.properties()[v] - except KeyError as e: - raise ValueError(f"Property {e.args[0]} is not defined in object type {e.args[1]}") from None + one_property = one_object_type.properties().get(k) + if one_property is None: + raise ValueError(f"Property {k} is not defined in object type {one_object_type.readable_id()}") + many_property = many_object_type.properties().get(v) + if many_property is None: + raise ValueError(f"Property {v} is not defined in object type {many_object_type.readable_id()}") property_map[many_property] = one_property @@ -435,10 +440,11 @@ def _parse_many_to_many_relation(self, guid: str, id: str, raw: dict[Any, Any], def build_property_map(object_type, pk_mapping: dict[str, str]) -> dict[Property, str]: prop_map: dict[Property, str] = {} for src_prop_id, dst_prop_id in pk_mapping.items(): - try: - obj_prop = object_type.properties()[src_prop_id] - except KeyError as e: - raise ValueError(f"Property {e.args[0]} is not defined in object type {e.args[1]}") from None + obj_prop = object_type.properties().get(src_prop_id) + if obj_prop is None: + raise ValueError( + f"Property {src_prop_id} is not defined in object type {object_type.readable_id()}" + ) prop_map[obj_prop] = dst_prop_id return prop_map @@ -457,7 +463,9 @@ def build_property_map(object_type, pk_mapping: dict[str, str]) -> dict[Property relation = ManyToManyRelation(guid, id, role_a_object_type, role_b_object_type, role_a_property_map, role_b_property_map) - join_table_data_source = get_list(raw, "joinTableDatasource") + # Accept both key spellings seen across export versions: `joinTableDatasource` + # and `joinTableDataSource` (capital 'S'). + join_table_data_source = get_list(raw, "joinTableDatasource") or get_list(raw, "joinTableDataSource") if len(join_table_data_source) != 1: raise ValueError("Relation definition must contain exactly one `joinTableDatasource`") diff --git a/converters/ontology/src/osi/parser/__init__.py b/converters/ontology/src/osi/parser/__init__.py index 9f560d61..1e1f829a 100644 --- a/converters/ontology/src/osi/parser/__init__.py +++ b/converters/ontology/src/osi/parser/__init__.py @@ -18,13 +18,13 @@ class OsiParser: _debug: bool def __init__(self, debug: bool = False, - formula_factory: FormulaFactory = FormulaFactory(), - mapping_formula_factory: MappingFormulaFactory = MappingFormulaFactory()): + formula_factory: FormulaFactory | None = None, + mapping_formula_factory: MappingFormulaFactory | None = None): self._debug = debug self._model = None self._spec = None - self._formula_factory = formula_factory - self._mapping_formula_factory = mapping_formula_factory + self._formula_factory = formula_factory or FormulaFactory() + self._mapping_formula_factory = mapping_formula_factory or MappingFormulaFactory() def parse(self, path: Path) -> OsiOntology: # OSI always expects a single spec file. diff --git a/converters/ontology/src/osi/spec.py b/converters/ontology/src/osi/spec.py index b21b28d2..670496ad 100644 --- a/converters/ontology/src/osi/spec.py +++ b/converters/ontology/src/osi/spec.py @@ -1,5 +1,6 @@ from __future__ import annotations +import sys from typing import Any, Literal import yaml @@ -236,7 +237,7 @@ def dump_dict(self) -> dict: return self.model_dump(exclude_none=True, exclude_defaults=True, by_alias=True) def dump_yaml(self) -> str: - return yaml.safe_dump(self.dump_dict(), sort_keys=False, width=float("inf")) + return yaml.safe_dump(self.dump_dict(), sort_keys=False, width=sys.maxsize) # `ReferentMapping` and `LinkMapping` are self-referential (each can contain a From 517db4d7f4396e59925d864875d3841a535cb9c4 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 22 Jul 2026 00:53:37 +0200 Subject: [PATCH 21/29] Added tests --- converters/ontology/tests/__init__.py | 0 converters/ontology/tests/conftest.py | 34 + .../flights_roundtrip.yaml | 1207 +++++++++++++++++ .../flights_structure.txt | 166 +++ .../ontology/tests/test_flights_snapshot.py | 58 + converters/ontology/tests/test_osi_parser.py | 191 +++ 6 files changed, 1656 insertions(+) create mode 100644 converters/ontology/tests/__init__.py create mode 100644 converters/ontology/tests/conftest.py create mode 100644 converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml create mode 100644 converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_structure_snapshot/flights_structure.txt create mode 100644 converters/ontology/tests/test_flights_snapshot.py create mode 100644 converters/ontology/tests/test_osi_parser.py diff --git a/converters/ontology/tests/__init__.py b/converters/ontology/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/converters/ontology/tests/conftest.py b/converters/ontology/tests/conftest.py new file mode 100644 index 00000000..1fbc983d --- /dev/null +++ b/converters/ontology/tests/conftest.py @@ -0,0 +1,34 @@ +"""Shared fixtures for the ontology converter test suite.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from osi.model import OsiOntology +from osi.parser import OsiParser + +# Repo layout: /converters/ontology/tests/conftest.py -> /examples +_REPO_ROOT = Path(__file__).resolve().parents[3] +_EXAMPLES_DIR = _REPO_ROOT / "examples" + + +@pytest.fixture(scope="session") +def examples_dir() -> Path: + if not _EXAMPLES_DIR.is_dir(): + pytest.skip(f"examples directory not found at {_EXAMPLES_DIR}") + return _EXAMPLES_DIR + + +@pytest.fixture(scope="session") +def flights_path(examples_dir: Path) -> Path: + path = examples_dir / "flights.yaml" + if not path.is_file(): + pytest.skip(f"flights.yaml not found at {path}") + return path + + +@pytest.fixture +def flights_model(flights_path: Path) -> OsiOntology: + return OsiParser().parse(flights_path) \ No newline at end of file diff --git a/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml new file mode 100644 index 00000000..df0a042b --- /dev/null +++ b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml @@ -0,0 +1,1207 @@ +version: 0.2.0.dev0 +name: Flights +description: Ontology of flights into and out of airports. +requires: +- COUNT[Airport] > 0 +- COUNT[Carrier] > 0 +ontology: +- concept: + name: Route + type: EntityType + identify_by: + - id + relationships: + - name: average_departure_delay + roles: + - concept: Delay + verbalizes: + - '{Route} has average- departure {Delay}' + multiplicity: ManyToOne + derived_by: + - Delay == AVG[Flight.departure_delay WHERE Flight.route(Route) GROUP BY Route] + - name: average_arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Route} has average- arrival {Delay}' + multiplicity: ManyToOne + derived_by: + - Delay == AVG[Flight.arrival_delay WHERE Flight.route(Route) GROUP BY Route] + - name: distance + roles: + - concept: Distance + verbalizes: + - '{Route} spans {Distance}' + multiplicity: ManyToOne + - name: lies_in + roles: + - concept: DistanceGroup + verbalizes: + - '{Route} has {DistanceGroup}' + multiplicity: ManyToOne + - name: id + roles: + - concept: RouteId + verbalizes: + - '{Route} is identified by {RouteId}' + multiplicity: OneToOne + - name: route_name + roles: + - concept: String + verbalizes: + - '{Route} has name- {String}' + multiplicity: ManyToOne + - name: destination + roles: + - concept: Airport + verbalizes: + - '{Route} connects to destination- {Airport}' + multiplicity: ManyToOne + requires: + - NOT Route.departure(Airport) + - name: departure + roles: + - concept: Airport + verbalizes: + - '{Route} connects to departure- {Airport}' + multiplicity: ManyToOne + requires: + - NOT Route.destination(Airport) +- concept: + name: RouteId + type: ValueType + description: A unique identifier for a route between two airports. Constructed by concatenating the IATA codes of the departure and destination airports (e.g., 'ATL -> DCA') + extends: + - String +- concept: + name: Carrier + type: EntityType + identify_by: + - code + relationships: + - name: code + roles: + - concept: CarrierCode + verbalizes: + - '{Carrier} uses {CarrierCode}' + multiplicity: OneToOne + - name: name + roles: + - concept: CarrierName + verbalizes: + - '{Carrier} has {CarrierName}' + multiplicity: ManyToOne +- concept: + name: CarrierName + type: ValueType + extends: + - String +- concept: + name: CarrierCode + type: ValueType + description: The two-letter IATA code for the airline carrier. + extends: + - String +- concept: + name: Flight + type: EntityType + identify_by: + - id + relationships: + - name: departure_delay + roles: + - concept: Delay + verbalizes: + - '{Flight} has departure- {Delay}' + multiplicity: ManyToOne + - name: number + roles: + - concept: FlightNr + verbalizes: + - '{Flight} has {FlightNr}' + multiplicity: ManyToOne + - name: scheduled_departure + roles: + - concept: DateTime + verbalizes: + - '{Flight} is scheduled to depart at {DateTime}' + multiplicity: ManyToOne + requires: + - Flight.scheduled_departure < Flight.scheduled_arrival + - name: arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Flight} has arrival- {Delay}' + multiplicity: ManyToOne + - name: canceled + verbalizes: + - '{Flight} was canceled' + - name: canceled_due_to + roles: + - concept: CancelationReason + verbalizes: + - '{Flight} was canceled due to {CancelationReason}' + multiplicity: ManyToOne + - name: distance + roles: + - concept: Distance + verbalizes: + - '{Flight} spans actual- {Distance}' + multiplicity: ManyToOne + - name: id + roles: + - concept: FlightId + verbalizes: + - '{Flight} is identified by {FlightId}' + multiplicity: OneToOne + - name: scheduled_arrival + roles: + - concept: DateTime + verbalizes: + - '{Flight} is scheduled to arrive at {DateTime}' + multiplicity: ManyToOne + - name: registers_longitude_series + roles: + - concept: DateTime + - concept: DegreesLongitude + verbalizes: + - '{Flight} at {DateTime} registers {DegreesLongitude}' + multiplicity: ManyToOne + - name: registers_latitude_series + roles: + - concept: DateTime + - concept: DegreesLatitude + verbalizes: + - '{Flight} at {DateTime} registers {DegreesLatitude}' + multiplicity: ManyToOne + - name: date + roles: + - concept: Date + verbalizes: + - '{Flight} is scheduled to depart on {Date}' + multiplicity: ManyToOne + - name: departs_at + roles: + - concept: DateTime + verbalizes: + - '{Flight} departs at {DateTime}' + multiplicity: ManyToOne + requires: + - Flight.departs_at < Flight.arrives_at + - name: diverted + verbalizes: + - '{Flight} was diverted' + - name: arrives_at + roles: + - concept: DateTime + verbalizes: + - '{Flight} arrives at {DateTime}' + multiplicity: ManyToOne + - name: route + roles: + - concept: Route + verbalizes: + - '{Flight} traverses {Route}' + multiplicity: ManyToOne + - name: aircraft + roles: + - concept: Aircraft + verbalizes: + - '{Flight} uses {Aircraft}' + multiplicity: ManyToOne + requires: + - Flight.operated_by(Aircraft.carrier) + - name: operated_by + roles: + - concept: Carrier + verbalizes: + - '{Flight} is operated by {Carrier}' + multiplicity: ManyToOne +- concept: + name: FlightId + type: ValueType + description: A unique identifier for an instance of a flight + extends: + - String +- concept: + name: FlightNr + type: ValueType + description: The IATA flight number, which is typically a combination of the airline's IATA code and a numeric code (e.g., 'AA1234'). + extends: + - String +- concept: + name: Market + type: EntityType + identify_by: + - name + relationships: + - name: name + roles: + - concept: MarketName + verbalizes: + - '{Market} is identified by {MarketName}' + multiplicity: OneToOne +- concept: + name: MarketName + type: ValueType + extends: + - String +- concept: + name: Airport + type: EntityType + identify_by: + - code + requires: + - Airport.latitude + - Airport.longitude + relationships: + - name: city + roles: + - concept: City + verbalizes: + - '{Airport} is located in {City}' + multiplicity: ManyToOne + - name: serves + roles: + - concept: Market + verbalizes: + - '{Airport} serves {Market}' + multiplicity: ManyToOne + - name: average_departure_delay + roles: + - concept: Delay + verbalizes: + - '{Airport} has average- departure {Delay}' + multiplicity: ManyToOne + derived_by: + - Delay == AVG[Flight.departure_delay WHERE Airport == Flight.route.departure GROUP BY Airport] + - name: longitude + roles: + - concept: DegreesLongitude + verbalizes: + - '{Airport} centers at {DegreesLongitude}' + multiplicity: ManyToOne + - name: average_arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Airport} has average- arrival {Delay}' + multiplicity: ManyToOne + derived_by: + - Delay == AVG[Flight.arrival_delay WHERE Airport == Flight.route.destination GROUP BY Airport] + - name: name + roles: + - concept: AirportName + verbalizes: + - '{Airport} has {AirportName}' + multiplicity: ManyToOne + - name: code + roles: + - concept: AirportCode + verbalizes: + - '{Airport} has {AirportCode}' + multiplicity: OneToOne + - name: latitude + roles: + - concept: DegreesLatitude + verbalizes: + - '{Airport} centers at {DegreesLatitude}' + multiplicity: ManyToOne +- concept: + name: AirportId + type: ValueType + description: Five digit number used as an alternate identifier for airports. + extends: + - String +- concept: + name: AirportCode + type: ValueType + description: The three-letter IATA code for the airport. + extends: + - String +- concept: + name: AirportName + type: ValueType + extends: + - String +- concept: + name: Aircraft + type: EntityType + identify_by: + - tailnum + relationships: + - name: serial_number + roles: + - concept: SerialNr + verbalizes: + - '{Aircraft} has {SerialNr}' + multiplicity: ManyToOne + - name: name + roles: + - concept: String + verbalizes: + - '{Aircraft} has name {String}' + multiplicity: ManyToOne + - name: number_of_seats + roles: + - concept: Integer + verbalizes: + - '{Aircraft} has {Integer} seats' + multiplicity: ManyToOne + - name: tailnum + roles: + - concept: TailNr + verbalizes: + - '{Aircraft} is identified by {TailNr}' + multiplicity: OneToOne + - name: model + roles: + - concept: Model + verbalizes: + - '{Aircraft} has {Model}' + multiplicity: ManyToOne + - name: year_manufactured + roles: + - concept: Year + verbalizes: + - '{Aircraft} was manufactured in {Year}' + multiplicity: ManyToOne + - name: capacity + roles: + - concept: Capacity + verbalizes: + - '{Aircraft} has {Capacity}' + multiplicity: ManyToOne + - name: carrier + roles: + - concept: Carrier + verbalizes: + - '{Aircraft} is operated by {Carrier}' + multiplicity: ManyToOne +- concept: + name: Model + type: EntityType + identify_by: + - name + - manufacturer + relationships: + - name: name + roles: + - concept: ModelName + verbalizes: + - '{Model} has {ModelName}' + multiplicity: ManyToOne + - name: manufacturer + roles: + - concept: Manufacturer + verbalizes: + - '{Model} is manufactured by {Manufacturer}' + multiplicity: ManyToOne +- concept: + name: ModelName + type: ValueType + extends: + - String +- concept: + name: Manufacturer + type: EntityType + identify_by: + - name + relationships: + - name: name + roles: + - concept: ManufacturerName + verbalizes: + - '{Manufacturer} is identified by {ManufacturerName}' + multiplicity: OneToOne +- concept: + name: ManufacturerName + type: ValueType + extends: + - String +- concept: + name: Runway + type: EntityType + identify_by: + - designator + - airport + relationships: + - name: airport + roles: + - concept: Airport + verbalizes: + - '{Runway} belongs to {Airport}' + multiplicity: ManyToOne + - name: designator + roles: + - concept: RunwayDesignator + verbalizes: + - '{Runway} uses {RunwayDesignator}' + multiplicity: ManyToOne + - name: length + roles: + - concept: RunwayLength + verbalizes: + - '{Runway} has {RunwayLength}' + multiplicity: ManyToOne + - name: geometry + roles: + - concept: RunwayGeometry + verbalizes: + - '{Runway} has {RunwayGeometry}' + multiplicity: ManyToOne +- concept: + name: City + type: EntityType + identify_by: + - name + - state + relationships: + - name: name + roles: + - concept: CityName + verbalizes: + - '{City} has {CityName}' + multiplicity: ManyToOne + - name: state + roles: + - concept: State + verbalizes: + - '{City} is located in {State}' + multiplicity: ManyToOne +- concept: + name: State + type: EntityType + identify_by: + - code + relationships: + - name: code + roles: + - concept: StateCode + verbalizes: + - '{State} is identified by {StateCode}' + multiplicity: OneToOne + - name: name + roles: + - concept: StateName + verbalizes: + - '{State} has {StateName}' + multiplicity: ManyToOne +- concept: + name: CancelationReason + type: EntityType + description: A curated set of reasons that explain why a flight is canceled + identify_by: + - code + relationships: + - name: code + roles: + - concept: CancelationCode + verbalizes: + - '{CancelationReason} is identified by {CancelationCode}' + multiplicity: OneToOne + - name: description + roles: + - concept: String + verbalizes: + - '{CancelationReason} has description- {String}' + multiplicity: ManyToOne +- concept: + name: Year + type: ValueType + extends: + - String +- concept: + name: TailNr + type: ValueType + extends: + - String +- concept: + name: StateName + type: ValueType + extends: + - String +- concept: + name: StateCode + type: ValueType + extends: + - String +- concept: + name: SerialNr + type: ValueType + extends: + - String +- concept: + name: RunwayDesignator + type: ValueType + description: Used to distinguish runways within a given airport. + extends: + - String +- concept: + name: DistanceGroup + type: ValueType + description: A number used to group distances of different lengths, where 1 groups the shortest and 10 the longest. + extends: + - Integer + requires: + - 1 <= DistanceGroup + - DistanceGroup <= 10 +- concept: + name: CityName + type: ValueType + extends: + - String +- concept: + name: Polygon + type: ValueType + description: A polygon represented as a list of vertices, where each vertex is a pair of latitude and longitude coordinates. + extends: + - String +- concept: + name: RunwayGeometry + type: ValueType + description: A polygon that models the shape of a runway. + extends: + - Polygon +- concept: + name: DegreesLongitude + type: ValueType + extends: + - Decimal + requires: + - DegreesLongitude <= 180 + - DegreesLongitude >= -180 +- concept: + name: DegreesLatitude + type: ValueType + extends: + - Decimal + requires: + - DegreesLatitude <= 90 + - DegreesLatitude >= -90 +- concept: + name: CancelationCode + type: ValueType + description: The single character code that identifies the reason a flight is canceled + extends: + - String + requires: + - CancelationCode == 'A' OR CancelationCode == 'B' OR CancelationCode == 'C' OR CancelationCode == 'D' +- concept: + name: NrMinutes + type: ValueType + description: Unit of measure for time in minutes + extends: + - Decimal +- concept: + name: Delay + type: ValueType + extends: + - NrMinutes +- concept: + name: NrMiles + type: ValueType + description: Unit of measure for distance in miles + extends: + - Decimal +- concept: + name: Distance + type: ValueType + extends: + - NrMiles +- concept: + name: NrPounds + type: ValueType + description: Unit of measure for weight in pounds + extends: + - Integer +- concept: + name: Capacity + type: ValueType + description: The capacity of an aircraft, measured in pounds. + extends: + - NrPounds +- concept: + name: NrFeet + type: ValueType + description: Unit of measure for distance in feet + extends: + - Decimal +- concept: + name: RunwayLength + type: ValueType + description: The unit for measuring the lengths of runways in American airports. + extends: + - NrFeet +ontology_mappings: +- name: flights_mapping + semantic_model: + name: Flights semantic model + datasets: + - name: RUNWAY + source: DATABASE.SCHEMA.RUNWAYS + fields: + - name: airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: airport_code + - name: length + expression: + dialects: + - dialect: ANSI_SQL + expression: length + - name: shape + expression: + dialects: + - dialect: ANSI_SQL + expression: shape + - name: designator + expression: + dialects: + - dialect: ANSI_SQL + expression: designator + - name: AIRCRAFT + source: DATABASE.SCHEMA.AIRCRAFT + description: An airplane, helicopter, or other machine capable of flight + fields: + - name: serial_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: serial_nr + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: nr_seats + expression: + dialects: + - dialect: ANSI_SQL + expression: nr_seats + - name: tail_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: tail_nr + - name: carrier_code + expression: + dialects: + - dialect: ANSI_SQL + expression: carrier_code + - name: manufacturer + expression: + dialects: + - dialect: ANSI_SQL + expression: manufacturer + - name: model + expression: + dialects: + - dialect: ANSI_SQL + expression: model + - name: year + expression: + dialects: + - dialect: ANSI_SQL + expression: year + - name: capacity + expression: + dialects: + - dialect: ANSI_SQL + expression: capacity + - name: AIRPORT + source: DATABASE.SCHEMA.AIRPORTS + description: An airport that is identified by an IATA code. + fields: + - name: state_code + expression: + dialects: + - dialect: ANSI_SQL + expression: state_code + - name: market_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: market_nm + - name: longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: longitude + - name: opened + expression: + dialects: + - dialect: ANSI_SQL + expression: opened + - name: state_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: state_nm + - name: city_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: city_nm + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: code + expression: + dialects: + - dialect: ANSI_SQL + expression: code + - name: latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: latitude + - name: FLIGHT + source: DATABASE.SCHEMA.FLIGHTS + description: A commercial passenger flight. + fields: + - name: dep_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: dep_delay + - name: air_time + expression: + dialects: + - dialect: ANSI_SQL + expression: air_time + - name: nr + expression: + dialects: + - dialect: ANSI_SQL + expression: nr + - name: carrier_code + expression: + dialects: + - dialect: ANSI_SQL + expression: carrier_code + - name: duration + expression: + dialects: + - dialect: ANSI_SQL + expression: duration + - name: scheduled_departure + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_departure + - name: arr_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: arr_delay + - name: cancelled + expression: + dialects: + - dialect: ANSI_SQL + expression: cancelled + - name: cancel_code + expression: + dialects: + - dialect: ANSI_SQL + expression: cancel_code + - name: distance + expression: + dialects: + - dialect: ANSI_SQL + expression: distance + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + - name: scheduled_arrival + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_arrival + - name: wheels_on + expression: + dialects: + - dialect: ANSI_SQL + expression: wheels_on + - name: date + expression: + dialects: + - dialect: ANSI_SQL + expression: date + - name: wheels_off + expression: + dialects: + - dialect: ANSI_SQL + expression: wheels_off + - name: scheduled_duration + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_duration + - name: tail_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: tail_nr + - name: departure + expression: + dialects: + - dialect: ANSI_SQL + expression: departure + - name: route_id + expression: + dialects: + - dialect: ANSI_SQL + expression: route_id + - name: diverted + expression: + dialects: + - dialect: ANSI_SQL + expression: diverted + - name: arrival + expression: + dialects: + - dialect: ANSI_SQL + expression: arrival + - name: CARRIER + source: DATABASE.SCHEMA.CARRIERS + description: An airline, such as Delta, United, or American. + fields: + - name: code + expression: + dialects: + - dialect: ANSI_SQL + expression: code + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: ROUTE + source: DATABASE.SCHEMA.ROUTES + description: Represents the existence of one or more flights between a pair of departure arrival airports. + fields: + - name: orig_airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: orig_airport_code + - name: dest_airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: dest_airport_code + - name: distance + expression: + dialects: + - dialect: ANSI_SQL + expression: distance + - name: dist_grp + expression: + dialects: + - dialect: ANSI_SQL + expression: dist_grp + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + concept_mappings: + - concept: Runway + object_mappings: + - referent_mappings: + - relationship: designator + expression: RUNWAY.designator + - relationship: airport + referent_mappings: + - relationship: code + expression: RUNWAY.airport_code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: designator + expression: RUNWAY.designator + - relationship: airport + referent_mappings: + - relationship: code + expression: RUNWAY.airport_code + children: + - object_mapping: + concept: RunwayLength + expression: RUNWAY.length + relationship: length + - object_mapping: + concept: RunwayGeometry + expression: RUNWAY.shape + relationship: geometry + - concept: Manufacturer + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer + - concept: Model + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRCRAFT.model + - relationship: manufacturer + referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer + - concept: Aircraft + object_mappings: + - referent_mappings: + - relationship: tailnum + expression: AIRCRAFT.tail_nr + link_mappings: + - object_mapping: + referent_mappings: + - relationship: tailnum + expression: AIRCRAFT.tail_nr + children: + - object_mapping: + concept: SerialNr + expression: AIRCRAFT.serial_nr + relationship: serial_number + - object_mapping: + concept: String + expression: AIRCRAFT.name + relationship: name + - object_mapping: + concept: Integer + expression: AIRCRAFT.nr_seats + relationship: number_of_seats + - object_mapping: + concept: Model + referent_mappings: + - relationship: name + expression: AIRCRAFT.model + - relationship: manufacturer + referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer + relationship: model + - object_mapping: + concept: Year + expression: AIRCRAFT.year + relationship: year_manufactured + - object_mapping: + concept: Capacity + expression: AIRCRAFT.capacity + relationship: capacity + - object_mapping: + concept: Carrier + referent_mappings: + - relationship: code + expression: AIRCRAFT.carrier_code + relationship: carrier + - concept: State + object_mappings: + - referent_mappings: + - relationship: code + expression: AIRPORT.state_code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + children: + - object_mapping: + concept: StateName + expression: AIRPORT.state_nm + relationship: name + - concept: City + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRPORT.city_nm + - relationship: state + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + - concept: Market + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRPORT.market_nm + - concept: Airport + object_mappings: + - referent_mappings: + - relationship: code + expression: AIRPORT.code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: AIRPORT.code + children: + - object_mapping: + concept: City + referent_mappings: + - relationship: name + expression: AIRPORT.city_nm + - relationship: state + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + relationship: city + - object_mapping: + concept: Market + referent_mappings: + - relationship: name + expression: AIRPORT.market_nm + relationship: serves + - object_mapping: + concept: DegreesLongitude + expression: AIRPORT.longitude + relationship: longitude + - object_mapping: + concept: AirportName + expression: AIRPORT.name + relationship: name + - object_mapping: + concept: DegreesLatitude + expression: AIRPORT.latitude + relationship: latitude + - concept: Flight + object_mappings: + - referent_mappings: + - relationship: id + expression: FLIGHT.id + link_mappings: + - object_mapping: + referent_mappings: + - relationship: id + expression: FLIGHT.id WHERE ( FLIGHT.diverted == TRUE ) + relationship: diverted + - object_mapping: + referent_mappings: + - relationship: id + expression: FLIGHT.id WHERE ( FLIGHT.cancelled == TRUE ) + relationship: canceled + - object_mapping: + referent_mappings: + - relationship: id + expression: FLIGHT.id + children: + - object_mapping: + concept: Delay + expression: FLIGHT.arr_delay + relationship: arrival_delay + - object_mapping: + concept: Delay + expression: FLIGHT.dep_delay + relationship: departure_delay + - object_mapping: + concept: FlightNr + expression: FLIGHT.nr + relationship: number + - object_mapping: + concept: DateTime + expression: FLIGHT.scheduled_departure + relationship: scheduled_departure + - object_mapping: + concept: CancelationReason + referent_mappings: + - relationship: code + expression: FLIGHT.cancel_code + relationship: canceled_due_to + - object_mapping: + concept: Distance + expression: FLIGHT.distance + relationship: distance + - object_mapping: + concept: DateTime + expression: FLIGHT.scheduled_arrival + relationship: scheduled_arrival + - object_mapping: + concept: Date + expression: FLIGHT.date + relationship: date + - object_mapping: + concept: DateTime + expression: FLIGHT.departure + relationship: departs_at + - object_mapping: + concept: DateTime + expression: FLIGHT.arrival + relationship: arrives_at + - object_mapping: + concept: Route + referent_mappings: + - relationship: id + expression: FLIGHT.route_id + relationship: route + - object_mapping: + concept: Aircraft + referent_mappings: + - relationship: tailnum + expression: FLIGHT.tail_nr + relationship: aircraft + - object_mapping: + concept: Carrier + referent_mappings: + - relationship: code + expression: FLIGHT.carrier_code + relationship: operated_by + - concept: Carrier + object_mappings: + - referent_mappings: + - relationship: code + expression: CARRIER.code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: CARRIER.code + children: + - object_mapping: + concept: CarrierName + expression: CARRIER.name + relationship: name + - concept: Route + object_mappings: + - referent_mappings: + - relationship: id + expression: ROUTE.id + link_mappings: + - object_mapping: + referent_mappings: + - relationship: id + expression: ROUTE.id + children: + - object_mapping: + concept: Distance + expression: ROUTE.distance + relationship: distance + - object_mapping: + concept: DistanceGroup + expression: ROUTE.dist_grp + relationship: lies_in + - object_mapping: + concept: String + expression: ROUTE.name + relationship: route_name + - object_mapping: + concept: Airport + referent_mappings: + - relationship: code + expression: ROUTE.dest_airport_code + relationship: destination + - object_mapping: + concept: Airport + referent_mappings: + - relationship: code + expression: ROUTE.orig_airport_code + relationship: departure diff --git a/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_structure_snapshot/flights_structure.txt b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_structure_snapshot/flights_structure.txt new file mode 100644 index 00000000..65cf234b --- /dev/null +++ b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_structure_snapshot/flights_structure.txt @@ -0,0 +1,166 @@ +name: Flights +version: 0.2.0.dev0 +description: Ontology of flights into and out of airports. + +ontology requires: + - COUNT[Airport] > 0 + - COUNT[Carrier] > 0 + +concepts: + Aircraft (ENTITY_TYPE) + identify_by: Aircraft.tailnum + Airport (ENTITY_TYPE) + identify_by: Airport.code + requires: Airport.latitude + requires: Airport.longitude + AirportCode (VALUE_TYPE) + extends: String + AirportId (VALUE_TYPE) + extends: String + AirportName (VALUE_TYPE) + extends: String + CancelationCode (VALUE_TYPE) + extends: String + requires: CancelationCode == 'A' OR CancelationCode == 'B' OR CancelationCode == 'C' OR CancelationCode == 'D' + CancelationReason (ENTITY_TYPE) + identify_by: CancelationReason.code + Capacity (VALUE_TYPE) + extends: NrPounds + Carrier (ENTITY_TYPE) + identify_by: Carrier.code + CarrierCode (VALUE_TYPE) + extends: String + CarrierName (VALUE_TYPE) + extends: String + City (ENTITY_TYPE) + identify_by: City.name, City.state + CityName (VALUE_TYPE) + extends: String + DegreesLatitude (VALUE_TYPE) + extends: Decimal + requires: DegreesLatitude <= 90 + requires: DegreesLatitude >= -90 + DegreesLongitude (VALUE_TYPE) + extends: Decimal + requires: DegreesLongitude <= 180 + requires: DegreesLongitude >= -180 + Delay (VALUE_TYPE) + extends: NrMinutes + Distance (VALUE_TYPE) + extends: NrMiles + DistanceGroup (VALUE_TYPE) + extends: Integer + requires: 1 <= DistanceGroup + requires: DistanceGroup <= 10 + Flight (ENTITY_TYPE) + identify_by: Flight.id + FlightId (VALUE_TYPE) + extends: String + FlightNr (VALUE_TYPE) + extends: String + Manufacturer (ENTITY_TYPE) + identify_by: Manufacturer.name + ManufacturerName (VALUE_TYPE) + extends: String + Market (ENTITY_TYPE) + identify_by: Market.name + MarketName (VALUE_TYPE) + extends: String + Model (ENTITY_TYPE) + identify_by: Model.manufacturer, Model.name + ModelName (VALUE_TYPE) + extends: String + NrFeet (VALUE_TYPE) + extends: Decimal + NrMiles (VALUE_TYPE) + extends: Decimal + NrMinutes (VALUE_TYPE) + extends: Decimal + NrPounds (VALUE_TYPE) + extends: Integer + Polygon (VALUE_TYPE) + extends: String + Route (ENTITY_TYPE) + identify_by: Route.id + RouteId (VALUE_TYPE) + extends: String + Runway (ENTITY_TYPE) + identify_by: Runway.airport, Runway.designator + RunwayDesignator (VALUE_TYPE) + extends: String + RunwayGeometry (VALUE_TYPE) + extends: Polygon + RunwayLength (VALUE_TYPE) + extends: NrFeet + SerialNr (VALUE_TYPE) + extends: String + State (ENTITY_TYPE) + identify_by: State.code + StateCode (VALUE_TYPE) + extends: String + StateName (VALUE_TYPE) + extends: String + TailNr (VALUE_TYPE) + extends: String + Year (VALUE_TYPE) + extends: String + +relationships: + Aircraft.capacity [MANY_TO_ONE]: Aircraft -> Capacity + Aircraft.carrier [MANY_TO_ONE]: Aircraft -> Carrier + Aircraft.model [MANY_TO_ONE]: Aircraft -> Model + Aircraft.name [MANY_TO_ONE]: Aircraft -> String + Aircraft.number_of_seats [MANY_TO_ONE]: Aircraft -> Integer + Aircraft.serial_number [MANY_TO_ONE]: Aircraft -> SerialNr + Aircraft.tailnum [ONE_TO_ONE]: Aircraft -> TailNr + Aircraft.year_manufactured [MANY_TO_ONE]: Aircraft -> Year + Airport.average_arrival_delay [MANY_TO_ONE]: Airport -> Delay + Airport.average_departure_delay [MANY_TO_ONE]: Airport -> Delay + Airport.city [MANY_TO_ONE]: Airport -> City + Airport.code [ONE_TO_ONE]: Airport -> AirportCode + Airport.latitude [MANY_TO_ONE]: Airport -> DegreesLatitude + Airport.longitude [MANY_TO_ONE]: Airport -> DegreesLongitude + Airport.name [MANY_TO_ONE]: Airport -> AirportName + Airport.serves [MANY_TO_ONE]: Airport -> Market + CancelationReason.code [ONE_TO_ONE]: CancelationReason -> CancelationCode + CancelationReason.description [MANY_TO_ONE]: CancelationReason -> String + Carrier.code [ONE_TO_ONE]: Carrier -> CarrierCode + Carrier.name [MANY_TO_ONE]: Carrier -> CarrierName + City.name [MANY_TO_ONE]: City -> CityName + City.state [MANY_TO_ONE]: City -> State + Flight.aircraft [MANY_TO_ONE]: Flight -> Aircraft + Flight.arrival_delay [MANY_TO_ONE]: Flight -> Delay + Flight.arrives_at [MANY_TO_ONE]: Flight -> DateTime + Flight.canceled [None]: Flight + Flight.canceled_due_to [MANY_TO_ONE]: Flight -> CancelationReason + Flight.date [MANY_TO_ONE]: Flight -> Date + Flight.departs_at [MANY_TO_ONE]: Flight -> DateTime + Flight.departure_delay [MANY_TO_ONE]: Flight -> Delay + Flight.distance [MANY_TO_ONE]: Flight -> Distance + Flight.diverted [None]: Flight + Flight.id [ONE_TO_ONE]: Flight -> FlightId + Flight.number [MANY_TO_ONE]: Flight -> FlightNr + Flight.operated_by [MANY_TO_ONE]: Flight -> Carrier + Flight.registers_latitude_series [MANY_TO_ONE]: Flight -> DateTime -> DegreesLatitude + Flight.registers_longitude_series [MANY_TO_ONE]: Flight -> DateTime -> DegreesLongitude + Flight.route [MANY_TO_ONE]: Flight -> Route + Flight.scheduled_arrival [MANY_TO_ONE]: Flight -> DateTime + Flight.scheduled_departure [MANY_TO_ONE]: Flight -> DateTime + Manufacturer.name [ONE_TO_ONE]: Manufacturer -> ManufacturerName + Market.name [ONE_TO_ONE]: Market -> MarketName + Model.manufacturer [MANY_TO_ONE]: Model -> Manufacturer + Model.name [MANY_TO_ONE]: Model -> ModelName + Route.average_arrival_delay [MANY_TO_ONE]: Route -> Delay + Route.average_departure_delay [MANY_TO_ONE]: Route -> Delay + Route.departure [MANY_TO_ONE]: Route -> Airport + Route.destination [MANY_TO_ONE]: Route -> Airport + Route.distance [MANY_TO_ONE]: Route -> Distance + Route.id [ONE_TO_ONE]: Route -> RouteId + Route.lies_in [MANY_TO_ONE]: Route -> DistanceGroup + Route.route_name [MANY_TO_ONE]: Route -> String + Runway.airport [MANY_TO_ONE]: Runway -> Airport + Runway.designator [MANY_TO_ONE]: Runway -> RunwayDesignator + Runway.geometry [MANY_TO_ONE]: Runway -> RunwayGeometry + Runway.length [MANY_TO_ONE]: Runway -> RunwayLength + State.code [ONE_TO_ONE]: State -> StateCode + State.name [MANY_TO_ONE]: State -> StateName diff --git a/converters/ontology/tests/test_flights_snapshot.py b/converters/ontology/tests/test_flights_snapshot.py new file mode 100644 index 00000000..92ca1692 --- /dev/null +++ b/converters/ontology/tests/test_flights_snapshot.py @@ -0,0 +1,58 @@ +"""Snapshot tests for the flights ontology. + +These lock in the converted structure and the round-tripped YAML so that any +change in parsing/conversion output shows up as a reviewable diff. + +Regenerate the snapshots after an intentional change with: + + pytest tests/test_flights_snapshot.py --snapshot-update +""" + +from __future__ import annotations + +from osi.converter.osi_to_spec.converter import OsiToSpecConverter +from osi.model import OntologyComponent, OsiOntology + + +def _render_structure(model: OsiOntology) -> str: + """Render a compact, deterministic text summary of the ontology structure.""" + ontology: OntologyComponent = model.ontology + lines: list[str] = [ + f"name: {model.name}", + f"version: {model.version}", + f"description: {model.description}", + "", + "ontology requires:", + ] + for req in ontology.requires: + lines.append(f" - {req}") + + lines.append("") + lines.append("concepts:") + for concept in sorted(ontology.concepts(exclude_builtin=True), key=lambda c: c.name): + type_name = concept.type.name if concept.type else "None" + lines.append(f" {concept.name} ({type_name})") + if concept.extends: + lines.append(f" extends: {', '.join(p.name for p in concept.extends)}") + if concept.identify_by: + lines.append(f" identify_by: {', '.join(sorted(concept.identify_by))}") + for req in concept.requires: + lines.append(f" requires: {req}") + + lines.append("") + lines.append("relationships:") + for rel in sorted(ontology.relationships, key=lambda r: r.full_name): + mult = rel.multiplicity.name if rel.multiplicity else "None" + signature = " -> ".join(c.name for c in rel.signature) + lines.append(f" {rel.full_name} [{mult}]: {signature}") + + return "\n".join(lines) + "\n" + + +def test_flights_structure_snapshot(flights_model, snapshot): + snapshot.assert_match(_render_structure(flights_model), "flights_structure.txt") + + +def test_flights_roundtrip_yaml_snapshot(flights_model, snapshot): + spec = OsiToSpecConverter.convert(flights_model) + snapshot.assert_match(spec.dump_yaml(), "flights_roundtrip.yaml") \ No newline at end of file diff --git a/converters/ontology/tests/test_osi_parser.py b/converters/ontology/tests/test_osi_parser.py new file mode 100644 index 00000000..ab682222 --- /dev/null +++ b/converters/ontology/tests/test_osi_parser.py @@ -0,0 +1,191 @@ +"""Unit tests for OsiParser and the spec -> OsiOntology conversion, driven by +the `examples/flights.yaml` ontology.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +import yaml + +from osi.converter.osi_to_spec.converter import OsiToSpecConverter +from osi.model import ConceptType, OsiOntology, RelationshipMultiplicity +from osi.parser import OsiParser + + +# ----- Document-level metadata ------------------------------------------ + +def test_parse_returns_model_with_metadata(flights_model): + assert flights_model.name == "Flights" + assert flights_model.version == "0.2.0.dev0" + assert flights_model.description == "Ontology of flights into and out of airports." + + +def test_parse_returns_populated_ontology(flights_model): + ontology = flights_model.ontology + # Built-in concepts (String, Integer, Decimal, ...) are always present on top + # of the ones declared in the spec. + assert len(ontology.concepts(exclude_builtin=True)) == 44 + assert len(ontology.concepts()) > len(ontology.concepts(exclude_builtin=True)) + assert len(ontology.relationships) == 58 + + +# ----- Ontology-level requires ------------------------------------------ + +def test_ontology_level_requires(flights_model): + requires = [str(r) for r in flights_model.ontology.requires] + assert requires == ["COUNT[Airport] > 0", "COUNT[Carrier] > 0"] + + +# ----- Concept-level requires ------------------------------------------- + +@pytest.mark.parametrize( + "concept_name, expected", + [ + ("DegreesLatitude", ["DegreesLatitude <= 90", "DegreesLatitude >= -90"]), + ("DegreesLongitude", ["DegreesLongitude <= 180", "DegreesLongitude >= -180"]), + ( + "CancelationCode", + ["CancelationCode == 'A' OR CancelationCode == 'B' OR CancelationCode == 'C' OR CancelationCode == 'D'"], + ), + ], +) +def test_concept_requires(flights_model, concept_name, expected): + concept = flights_model.ontology.lookup_concept(concept_name) + assert concept is not None + assert [str(r) for r in concept.requires] == expected + + +# ----- Value-type inheritance ------------------------------------------- + +@pytest.mark.parametrize( + "concept_name, parent_name", + [ + ("NrFeet", "Decimal"), + ("NrPounds", "Integer"), + ("Capacity", "NrPounds"), + ("CancelationCode", "String"), + ("Delay", "NrMinutes"), + ], +) +def test_value_type_extends(flights_model, concept_name, parent_name): + concept = flights_model.ontology.lookup_concept(concept_name) + assert concept is not None + assert concept.type == ConceptType.VALUE_TYPE + assert [p.name for p in concept.extends] == [parent_name] + + +# ----- Identifiers ------------------------------------------------------- + +def test_identify_by(flights_model): + airport = flights_model.ontology.lookup_concept("Airport") + assert airport is not None + assert airport.type == ConceptType.ENTITY_TYPE + assert list(airport.identify_by.keys()) == ["Airport.code"] + + +# ----- Relationship multiplicity ---------------------------------------- + +def test_relationship_multiplicity(flights_model): + ontology = flights_model.ontology + airport = ontology.lookup_concept("Airport") + code_rel = ontology.lookup_concept_relationship(airport, "code") + assert code_rel is not None + assert code_rel.multiplicity == RelationshipMultiplicity.ONE_TO_ONE + + +# ----- Ontology mapping / semantic model -------------------------------- + +def test_ontology_mapping(flights_model): + assert len(flights_model.ontology_mappings) == 1 + mapping = flights_model.ontology_mappings[0] + semantic_model = mapping.semantic_model + dataset_names = {d.name for d in semantic_model.datasets} + assert {"AIRPORT", "FLIGHT", "CARRIER", "ROUTE"} <= dataset_names + assert len(mapping.concept_mappings) == 11 + + +# ----- load_data -------------------------------------------------------- + +def test_load_data_reads_yaml(tmp_path: Path): + path = tmp_path / "spec.yaml" + path.write_text("a: 1\nb:\n - x\n - y\n") + assert OsiParser.load_data(path) == {"a": 1, "b": ["x", "y"]} + + +def test_load_data_reads_json(tmp_path: Path): + path = tmp_path / "spec.json" + path.write_text(json.dumps({"a": 1, "b": ["x", "y"]})) + assert OsiParser.load_data(path) == {"a": 1, "b": ["x", "y"]} + + +def test_parse_of_flights_as_json(flights_path: Path, tmp_path: Path): + # The parser selects JSON vs YAML from the file suffix; a .json rendering of + # the same spec must produce an equivalent model. + json_path = tmp_path / "flights.json" + json_path.write_text(json.dumps(yaml.safe_load(flights_path.read_text()))) + model = OsiParser().parse(json_path) + assert model.name == "Flights" + assert len(model.ontology.concepts(exclude_builtin=True)) == 44 + + +# ----- Error handling --------------------------------------------------- + +def test_parse_rejects_directory(tmp_path: Path): + with pytest.raises(ValueError, match="is not a file"): + OsiParser().parse(tmp_path) + + +def test_parse_rejects_missing_file(tmp_path: Path): + with pytest.raises(ValueError, match="is not a file"): + OsiParser().parse(tmp_path / "does_not_exist.yaml") + + +def test_spec_requires_parse_first(): + parser = OsiParser() + with pytest.raises(RuntimeError): + parser.spec() + + +def test_parsers_do_not_share_formula_factories(): + a, b = OsiParser(), OsiParser() + assert a._formula_factory is not b._formula_factory + assert a._mapping_formula_factory is not b._mapping_formula_factory + + +# ----- Round-trip invariants -------------------------------------------- + +def _structure_sets(model: OsiOntology): + ontology = model.ontology + return ( + {c.name for c in ontology.concepts(exclude_builtin=True)}, + {r.full_name for r in ontology.relationships}, + {str(req) for req in ontology.requires}, + ) + + +def test_roundtrip_preserves_structure(flights_path, tmp_path: Path): + """Parsing -> spec -> YAML -> parsing preserves the ontology structure. + + Note: concept/relationship *emission order* is not guaranteed to be stable + across a round-trip (it depends on the topological tie-breaking of the input + order), so we compare the sets of concepts, relationships, and requires + rather than the raw YAML. + """ + model1 = OsiParser().parse(flights_path) + yaml1 = OsiToSpecConverter.convert(model1).dump_yaml() + + roundtrip_path = tmp_path / "roundtrip.yaml" + roundtrip_path.write_text(yaml1) + model2 = OsiParser().parse(roundtrip_path) + + assert _structure_sets(model1) == _structure_sets(model2) + + +def test_dump_yaml_is_deterministic_for_fixed_input(flights_path): + """The same input file always dumps to identical YAML (so the snapshot is + stable across runs).""" + yaml_a = OsiToSpecConverter.convert(OsiParser().parse(flights_path)).dump_yaml() + yaml_b = OsiToSpecConverter.convert(OsiParser().parse(flights_path)).dump_yaml() + assert yaml_a == yaml_b \ No newline at end of file From a47e2796084686d3b1b2b61551709b01a1aad6e4 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 22 Jul 2026 00:58:31 +0200 Subject: [PATCH 22/29] Updated pyproject.toml file --- converters/ontology/pyproject.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/converters/ontology/pyproject.toml b/converters/ontology/pyproject.toml index 485340aa..458ed8d8 100644 --- a/converters/ontology/pyproject.toml +++ b/converters/ontology/pyproject.toml @@ -16,12 +16,15 @@ dependencies = [ dev = [ "pytest==9.0.3", "pytest-snapshot", - "parameterized", "pip-tools", ] [tool.setuptools.packages.find] where = ["src"] +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] + [tool.pyright] pythonVersion = "3.11" From 613461be82a9b320f698725f3f1e7f76e07acc03 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 22 Jul 2026 12:02:18 +0200 Subject: [PATCH 23/29] Refactored after the Copilot code review --- .../ontology/src/osi/converter/spec_to_osi/converter.py | 8 ++++++++ .../ontology/src/osi/external/palantir/parser/__init__.py | 8 +++++++- converters/ontology/src/osi/parser/__init__.py | 3 ++- 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index 0b709c3e..a60c6db0 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -189,6 +189,14 @@ def _convert_relationship( relates.append((role_concept, role_spec.name)) multiplicity = RelationshipMultiplicity.from_value(rel_spec.multiplicity) + # OneToOne is only meaningful for binary relationships (the container + # concept plus exactly one additional role). + if multiplicity == RelationshipMultiplicity.ONE_TO_ONE and len(relates) != 1: + raise ValueError( + f"Relationship '{container.name}.{rel_spec.name}' declares OneToOne multiplicity, " + f"which is only valid for binary relationships (exactly one additional role), " + f"but it has {len(relates)}." + ) relationship = Relationship( name=rel_spec.name, container=container, diff --git a/converters/ontology/src/osi/external/palantir/parser/__init__.py b/converters/ontology/src/osi/external/palantir/parser/__init__.py index b599b3d6..c3bae0ab 100644 --- a/converters/ontology/src/osi/external/palantir/parser/__init__.py +++ b/converters/ontology/src/osi/external/palantir/parser/__init__.py @@ -32,7 +32,13 @@ def get_dict(d, key): def get_list(d, key): v = d.get(key) - return v if isinstance(v, list) else [] + if isinstance(v, list): + return v + # Normalize singletons: some exports emit a lone object/value where a list is + # expected. Treat a non-null singleton as a one-element list; missing/null -> []. + if v is None: + return [] + return [v] # DataSets in Palantir have their own JSON format that is separate from the Ontology JSON format. class PalantirDataSetParser: diff --git a/converters/ontology/src/osi/parser/__init__.py b/converters/ontology/src/osi/parser/__init__.py index 1e1f829a..a6d175c2 100644 --- a/converters/ontology/src/osi/parser/__init__.py +++ b/converters/ontology/src/osi/parser/__init__.py @@ -39,7 +39,8 @@ def parse(self, path: Path) -> OsiOntology: @staticmethod def load_data(path: Path): - content = path.read_text() + # Pin UTF-8 so parsing is reproducible regardless of the process locale. + content = path.read_text(encoding="utf-8") if path.suffix.lower() == ".json": return json.loads(content) return yaml.safe_load(content) From fa1eec049d55285dc0babaebeb6c5d1720c6b2dd Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 22 Jul 2026 12:29:10 +0200 Subject: [PATCH 24/29] Refactored after the Copilot code review --- .../converter/palantir_to_osi/converter.py | 9 +- .../src/osi/external/palantir/model.py | 3 + converters/ontology/src/osi/model.py | 2 +- .../ontology/tests/test_palantir_parser.py | 165 ++++++++++++++++++ 4 files changed, 173 insertions(+), 6 deletions(-) create mode 100644 converters/ontology/tests/test_palantir_parser.py diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index 19dba400..eb7a0ac8 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -225,8 +225,7 @@ def madlib_decl(c: Concept, p: PalantirProperty) -> str: if ontology.lookup_concept_relationship(concept, prop_name) is not None: return - relates: list[tuple[Concept, str | None]] = [] - relates = self._convert_property_type_roles(ontology, relates, prop.type()) + relates = self._convert_property_type_roles(ontology, [], prop.type()) ontology.add_relationship(Relationship( name=prop_name, @@ -250,7 +249,7 @@ def _convert_mappings( db_name: str, schema_name: str, ) -> None: - if not ot._syncs_from: + if not ot.has_syncs_from(): return parent_concept: Concept | None = None @@ -437,7 +436,7 @@ def _convert_many_to_one( ) ontology.add_relationship(relationship) - if mot._syncs_from: + if mot.has_syncs_from(): self._attach_link_to_concept_mappings( ontology, rel, relationship, mot, mot_concept, oot_concept, concept_mappings, semantic_model ) @@ -752,7 +751,7 @@ def _convert_property_type_roles( def _depth_role_name(depth: int) -> str: name = PalantirToOsiConverter.depths_role_names.get(depth) if not name: - raise Exception(f"Array types of depth {depth} are not supported") + raise ValueError(f"Array types of depth {depth} are not supported") return name @staticmethod diff --git a/converters/ontology/src/osi/external/palantir/model.py b/converters/ontology/src/osi/external/palantir/model.py index 7db5eae3..35a73a8b 100644 --- a/converters/ontology/src/osi/external/palantir/model.py +++ b/converters/ontology/src/osi/external/palantir/model.py @@ -346,6 +346,9 @@ def name(self): def type_groups(self): return self._type_groups + def has_syncs_from(self) -> bool: + return bool(self._syncs_from) + def syncs_from(self): if not self._syncs_from: raise RuntimeError(f"Mandatory constraint violation: ObjectType '{self.readable_id()}' must sync with some DataSet") diff --git a/converters/ontology/src/osi/model.py b/converters/ontology/src/osi/model.py index 98f9cbd1..3af67d10 100644 --- a/converters/ontology/src/osi/model.py +++ b/converters/ontology/src/osi/model.py @@ -1092,7 +1092,7 @@ def _parse_verbalization(relationship: Relationship, verbalization: str) -> Rela if len(tokens) != relationship.arity: raise ValueError( f"Number of roles in verbalization '{verbalization}' for relationship " - f"{relationship.full_name} don't match" + f"{relationship.full_name} doesn't match its arity ({relationship.arity})" ) segments: list[str] = [] roles: list[VerbalizationRole] = [] diff --git a/converters/ontology/tests/test_palantir_parser.py b/converters/ontology/tests/test_palantir_parser.py new file mode 100644 index 00000000..1879eabc --- /dev/null +++ b/converters/ontology/tests/test_palantir_parser.py @@ -0,0 +1,165 @@ +"""Tests for the Palantir parser's input-shape handling. + +A Palantir export can arrive in several layouts — a ZIP archive, an already +extracted folder, a folder that wraps a single ZIP, and any of those packaged +under a single root directory. These tests exercise each supported layout plus +the validation failure paths (missing/empty ``data_sets`` folder, ambiguous or +missing ontology JSON, unsupported inputs). +""" + +from __future__ import annotations + +import json +import zipfile +from pathlib import Path + +import pytest + +from osi.external.palantir.model import Ontology +from osi.external.palantir.parser import PalantirParser + +# A minimal-but-complete Palantir export: one object type backed by one dataset. +_ONTOLOGY_JSON = { + "objectTypes": [ + { + "rid": "ri.ot.1", + "id": "widget", + "displayName": "Widget", + "properties": [ + {"rid": "ri.p.1", "id": "widget_id", "baseType": {"type": "STRING"}} + ], + "primaryKeys": ["widget_id"], + } + ], + "relations": [], +} +_DATASET_JSON = [ + { + "mainDatasetId": "ri.ot.1", + "datasetName": "widget", + "datasetSchema": [{"name": "widget_id", "type": "STRING"}], + } +] + + +# ----- builders --------------------------------------------------------- + +def _write_dir_export(base: Path, *, root: str | None = None) -> Path: + """Create an extracted-folder export under *base*, optionally nested inside a + single wrapping *root* directory. Returns *base* (the path to hand to parse).""" + target = base / root if root else base + (target / "data_sets").mkdir(parents=True) + (target / "ontology.json").write_text(json.dumps(_ONTOLOGY_JSON)) + (target / "data_sets" / "ds.json").write_text(json.dumps(_DATASET_JSON)) + return base + + +def _write_zip_export(zip_path: Path, *, root: str | None = None) -> Path: + """Create a ZIP export at *zip_path*, optionally packaged under a single + *root* directory. Returns *zip_path*.""" + prefix = f"{root}/" if root else "" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr(f"{prefix}ontology.json", json.dumps(_ONTOLOGY_JSON)) + zf.writestr(f"{prefix}data_sets/ds.json", json.dumps(_DATASET_JSON)) + return zip_path + + +def _assert_widget_model(model: Ontology) -> None: + assert isinstance(model, Ontology) + object_types = model.object_types() + assert list(object_types.keys()) == ["ri.ot.1"] + widget = object_types["ri.ot.1"] + assert widget.readable_id() == "widget" + # The backing dataset should have been matched and synced onto the object type. + assert widget.has_syncs_from() + + +# ----- supported layouts ------------------------------------------------ + +def test_parse_top_level_zip(tmp_path: Path): + zip_path = _write_zip_export(tmp_path / "export.zip") + _assert_widget_model(PalantirParser().parse(zip_path)) + + +def test_parse_single_root_zip(tmp_path: Path): + zip_path = _write_zip_export(tmp_path / "export.zip", root="export") + _assert_widget_model(PalantirParser().parse(zip_path)) + + +def test_parse_extracted_directory(tmp_path: Path): + export = _write_dir_export(tmp_path / "export") + _assert_widget_model(PalantirParser().parse(export)) + + +def test_parse_single_root_directory(tmp_path: Path): + # base/ contains exactly one child dir which holds the export. + export = _write_dir_export(tmp_path / "export", root="inner") + _assert_widget_model(PalantirParser().parse(export)) + + +def test_parse_directory_wrapping_single_zip(tmp_path: Path): + wrapper = tmp_path / "wrapper" + wrapper.mkdir() + _write_zip_export(wrapper / "export.zip") + _assert_widget_model(PalantirParser().parse(wrapper)) + + +# ----- unsupported / missing inputs ------------------------------------- + +def test_parse_missing_path_raises(tmp_path: Path): + with pytest.raises(FileNotFoundError, match="does not exist"): + PalantirParser().parse(tmp_path / "nope") + + +def test_parse_non_zip_file_raises(tmp_path: Path): + bad = tmp_path / "notes.txt" + bad.write_text("not a zip") + with pytest.raises(ValueError, match="Expected a ZIP archive or a directory"): + PalantirParser().parse(bad) + + +# ----- invalid data_sets ----------------------------------------------- + +def test_zip_missing_data_sets_folder_raises(tmp_path: Path): + zip_path = tmp_path / "export.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("ontology.json", json.dumps(_ONTOLOGY_JSON)) + with pytest.raises(ValueError, match="does not contain required 'data_sets' folder"): + PalantirParser().parse(zip_path) + + +def test_directory_missing_data_sets_folder_raises(tmp_path: Path): + export = tmp_path / "export" + export.mkdir() + (export / "ontology.json").write_text(json.dumps(_ONTOLOGY_JSON)) + with pytest.raises(ValueError, match="does not contain required 'data_sets' folder"): + PalantirParser().parse(export) + + +def test_data_sets_folder_without_json_raises(tmp_path: Path): + zip_path = tmp_path / "export.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("ontology.json", json.dumps(_ONTOLOGY_JSON)) + # A data_sets/ entry exists but contains no JSON files. + zf.writestr("data_sets/README.txt", "no json here") + with pytest.raises(ValueError, match="'data_sets' folder contains no JSON files"): + PalantirParser().parse(zip_path) + + +# ----- ontology JSON resolution ----------------------------------------- + +def test_multiple_top_level_json_in_zip_raises(tmp_path: Path): + zip_path = tmp_path / "export.zip" + with zipfile.ZipFile(zip_path, "w") as zf: + zf.writestr("ontology.json", json.dumps(_ONTOLOGY_JSON)) + zf.writestr("other.json", json.dumps(_ONTOLOGY_JSON)) + zf.writestr("data_sets/ds.json", json.dumps(_DATASET_JSON)) + with pytest.raises(ValueError, match="exactly one top-level JSON file"): + PalantirParser().parse(zip_path) + + +def test_multiple_top_level_json_in_directory_raises(tmp_path: Path): + export = _write_dir_export(tmp_path / "export") + (export / "other.json").write_text(json.dumps(_ONTOLOGY_JSON)) + with pytest.raises(ValueError, match="exactly one top-level JSON file"): + PalantirParser().parse(export) \ No newline at end of file From 02ac02aef8e9c30cb90102f38894dbc65e3b20af Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 22 Jul 2026 13:02:16 +0200 Subject: [PATCH 25/29] Refactored after the Copilot code review --- .../ontology/src/osi/common/file_utils.py | 5 +- .../osi/external/palantir/parser/__init__.py | 11 +- converters/ontology/src/osi/spec.py | 11 +- converters/ontology/tests/conftest.py | 21 +- .../ontology/tests/fixtures/flights.yaml | 1155 +++++++++++++++++ .../ontology/tests/test_examples_in_sync.py | 34 + 6 files changed, 1216 insertions(+), 21 deletions(-) create mode 100644 converters/ontology/tests/fixtures/flights.yaml create mode 100644 converters/ontology/tests/test_examples_in_sync.py diff --git a/converters/ontology/src/osi/common/file_utils.py b/converters/ontology/src/osi/common/file_utils.py index d9f70896..725bae95 100644 --- a/converters/ontology/src/osi/common/file_utils.py +++ b/converters/ontology/src/osi/common/file_utils.py @@ -68,8 +68,11 @@ def iter_json_files_from_dir(base_dir: Path, dir_prefix: str) -> Iterable[tuple[ return for path in sorted(data_dir.rglob("*")): if path.is_file() and path.suffix.lower() == ".json": + # Yield a stable, root-relative name (posix separators) to mirror the + # ZIP variant and avoid leaking host-specific absolute paths. + name = path.relative_to(root).as_posix() with path.open("rb") as fp: - yield str(path), io.BytesIO(fp.read()) + yield name, io.BytesIO(fp.read()) def get_top_level_json_file_from_dir(base_dir: Path) -> Path: diff --git a/converters/ontology/src/osi/external/palantir/parser/__init__.py b/converters/ontology/src/osi/external/palantir/parser/__init__.py index c3bae0ab..071fbc62 100644 --- a/converters/ontology/src/osi/external/palantir/parser/__init__.py +++ b/converters/ontology/src/osi/external/palantir/parser/__init__.py @@ -68,14 +68,15 @@ def _dataset_from_dict(self, d: dict[str, Any], registry: dict[str, DataSet]) -> ds._readable_id = set_if_value(ds.readable_id(), norm(d.get("datasetName"))) ds._description = set_if_value(ds.description(), norm(d.get("description"))) - # Columns - ds_schema = d.get("datasetSchema") - if isinstance(ds_schema, list): + # Columns: normalize list-vs-singleton like the other helpers. Only touch + # columns when the field is actually present, so a missing schema in one + # JSON entry doesn't clobber columns populated from another. + if d.get("datasetSchema") is not None: cols: list[DataSetColumn] = [] - for item in ds_schema: + for item in get_list(d, "datasetSchema"): if isinstance(item, dict): cols.append(DataSetColumn(item.get("name"), item.get("type"), ds)) - ds._columns = cols # only when provided as a proper list + ds._columns = cols # Dependencies raw_inputs = d.get("inputDatasetIds") diff --git a/converters/ontology/src/osi/spec.py b/converters/ontology/src/osi/spec.py index 670496ad..7144da94 100644 --- a/converters/ontology/src/osi/spec.py +++ b/converters/ontology/src/osi/spec.py @@ -1,11 +1,14 @@ from __future__ import annotations -import sys from typing import Any, Literal import yaml from pydantic import BaseModel, ConfigDict, Field +# Default line width for YAML output: bounded so diffs stay reviewable, but wide +# enough that typical concept/relationship lines aren't wrapped mid-token. +DEFAULT_YAML_WIDTH = 1000 + class OsiObject(BaseModel): """Base for all OSI DTOs. Strict (`extra=forbid`) to surface spec drift early.""" @@ -236,8 +239,10 @@ def load_yaml(cls, text: str) -> OsiSpec: def dump_dict(self) -> dict: return self.model_dump(exclude_none=True, exclude_defaults=True, by_alias=True) - def dump_yaml(self) -> str: - return yaml.safe_dump(self.dump_dict(), sort_keys=False, width=sys.maxsize) + def dump_yaml(self, width: int = DEFAULT_YAML_WIDTH) -> str: + # A bounded width keeps output reasonably wrapped (reviewable diffs) while + # staying deterministic. Callers can pass a large width to disable wrapping. + return yaml.safe_dump(self.dump_dict(), sort_keys=False, width=width) # `ReferentMapping` and `LinkMapping` are self-referential (each can contain a diff --git a/converters/ontology/tests/conftest.py b/converters/ontology/tests/conftest.py index 1fbc983d..9e015f1b 100644 --- a/converters/ontology/tests/conftest.py +++ b/converters/ontology/tests/conftest.py @@ -9,24 +9,21 @@ from osi.model import OsiOntology from osi.parser import OsiParser -# Repo layout: /converters/ontology/tests/conftest.py -> /examples -_REPO_ROOT = Path(__file__).resolve().parents[3] -_EXAMPLES_DIR = _REPO_ROOT / "examples" +# Test inputs are vendored under tests/fixtures/ so the suite runs even when the +# repo-level examples/ directory isn't present (e.g. from an sdist/wheel or a +# subset checkout). tests/test_examples_in_sync.py guards against drift from the +# canonical examples/ copies. +_FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" @pytest.fixture(scope="session") -def examples_dir() -> Path: - if not _EXAMPLES_DIR.is_dir(): - pytest.skip(f"examples directory not found at {_EXAMPLES_DIR}") - return _EXAMPLES_DIR +def fixtures_dir() -> Path: + return _FIXTURES_DIR @pytest.fixture(scope="session") -def flights_path(examples_dir: Path) -> Path: - path = examples_dir / "flights.yaml" - if not path.is_file(): - pytest.skip(f"flights.yaml not found at {path}") - return path +def flights_path(fixtures_dir: Path) -> Path: + return fixtures_dir / "flights.yaml" @pytest.fixture diff --git a/converters/ontology/tests/fixtures/flights.yaml b/converters/ontology/tests/fixtures/flights.yaml new file mode 100644 index 00000000..f0f64d13 --- /dev/null +++ b/converters/ontology/tests/fixtures/flights.yaml @@ -0,0 +1,1155 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +version: 0.2.0.dev0 +name: Flights +description: Ontology of flights into and out of airports. +requires: + - COUNT[Airport] > 0 # there must be at least one Airport + - COUNT[Carrier] > 0 # there must be at least one Carrier +ontology: +- concept: + name: NrFeet + description: "Unit of measure for distance in feet" + type: ValueType + extends: [ Decimal ] +- concept: + name: NrPounds + description: "Unit of measure for weight in pounds" + type: ValueType + extends: [ Integer ] +- concept: + name: NrMiles + description: "Unit of measure for distance in miles" + type: ValueType + extends: [ Decimal ] +- concept: + name: NrMinutes + description: "Unit of measure for time in minutes" + type: ValueType + extends: [ Decimal ] +- concept: + name: CancelationCode + description: "The single character code that identifies the reason a flight is canceled" + type: ValueType + extends: [ String ] + requires: [ CancelationCode == 'A' OR CancelationCode == 'B' OR CancelationCode == 'C' OR CancelationCode == 'D' ] +- concept: + name: Capacity + description: "The capacity of an aircraft, measured in pounds." + type: ValueType + extends: [ NrPounds ] +- concept: + name: DegreesLatitude + type: ValueType + extends: [ Decimal ] + requires: [ DegreesLatitude <= 90, DegreesLatitude >= -90 ] +- concept: + name: DegreesLongitude + type: ValueType + extends: [ Decimal ] + requires: [ DegreesLongitude <= 180, DegreesLongitude >= -180 ] +- concept: + name: Polygon + description: "A polygon represented as a list of vertices, where each vertex is a pair of latitude and longitude coordinates." + type: ValueType + extends: [ String ] +- concept: + name: CityName + type: ValueType + extends: [ String ] +- concept: + name: Delay + type: ValueType + extends: [ NrMinutes ] +- concept: + name: Distance + type: ValueType + extends: [ NrMiles ] +- concept: + name: DistanceGroup + description: "A number used to group distances of different lengths, where 1 groups the shortest and 10 the longest." + type: ValueType + extends: [ Integer ] + requires: [ 1 <= DistanceGroup, DistanceGroup <= 10 ] +- concept: + name: RunwayGeometry + description: "A polygon that models the shape of a runway." + type: ValueType + extends: [ Polygon ] +- concept: + name: RunwayLength + description: "The unit for measuring the lengths of runways in American airports." + type: ValueType + extends: [ NrFeet ] +- concept: + name: RunwayDesignator + description: "Used to distinguish runways within a given airport." + type: ValueType + extends: [ String ] +- concept: + name: SerialNr + type: ValueType + extends: [ String ] +- concept: + name: StateCode + type: ValueType + extends: [ String ] +- concept: + name: StateName + type: ValueType + extends: [ String ] +- concept: + name: TailNr + type: ValueType + extends: [ String ] +- concept: + name: Year + type: ValueType + extends: [ String ] +- concept: + name: CancelationReason + description: "A curated set of reasons that explain why a flight is canceled" + type: EntityType + identify_by: [ code ] + relationships: + - name: code + roles: + - concept: CancelationCode + verbalizes: [ '{CancelationReason} is identified by {CancelationCode}'] + multiplicity: OneToOne + - name: description + roles: + - concept: String + verbalizes: [ '{CancelationReason} has description- {String}' ] # The hyphen after "description" is significant when verbalizing constraints + multiplicity: ManyToOne # Each CancelationReason has at most one description String +- concept: + name: State + type: EntityType + identify_by: [ code ] + relationships: + - name: code + roles: + - concept: StateCode + verbalizes: + - '{State} is identified by {StateCode}' + multiplicity: OneToOne + - name: name + roles: + - concept: StateName + verbalizes: + - '{State} has {StateName}' + multiplicity: ManyToOne +- concept: + name: City + type: EntityType + identify_by: [ name, state ] + relationships: + - name: name + roles: + - concept: CityName + verbalizes: + - '{City} has {CityName}' + multiplicity: ManyToOne + - name: state + roles: + - concept: State + verbalizes: + - '{City} is located in {State}' + multiplicity: ManyToOne +- concept: + name: Runway + type: EntityType + identify_by: [ designator, airport ] + relationships: + - name: airport + roles: + - concept: Airport + verbalizes: + - '{Runway} belongs to {Airport}' + multiplicity: ManyToOne + - name: designator + roles: + - concept: RunwayDesignator + verbalizes: + - '{Runway} uses {RunwayDesignator}' + multiplicity: ManyToOne + - name: length + roles: + - concept: RunwayLength + verbalizes: + - '{Runway} has {RunwayLength}' + multiplicity: ManyToOne + - name: geometry + roles: + - concept: RunwayGeometry + verbalizes: + - '{Runway} has {RunwayGeometry}' + multiplicity: ManyToOne +- concept: + name: ManufacturerName + type: ValueType + extends: [ String ] +- concept: + name: Manufacturer + type: EntityType + identify_by: [ name ] + relationships: + - name: name + roles: + - concept: ManufacturerName + verbalizes: + - '{Manufacturer} is identified by {ManufacturerName}' + multiplicity: OneToOne +- concept: + name: ModelName + type: ValueType + extends: [ String ] +- concept: + name: Model + type: EntityType + identify_by: [ name, manufacturer ] + relationships: + - name: name + roles: + - concept: ModelName + verbalizes: + - '{Model} has {ModelName}' + multiplicity: ManyToOne + - name: manufacturer + roles: + - concept: Manufacturer + verbalizes: + - '{Model} is manufactured by {Manufacturer}' + multiplicity: ManyToOne +- concept: + name: Aircraft + type: EntityType + identify_by: [ tailnum ] + relationships: + - name: serial_number + roles: + - concept: SerialNr + verbalizes: + - '{Aircraft} has {SerialNr}' + multiplicity: ManyToOne + - name: name + roles: + - concept: String + verbalizes: + - '{Aircraft} has name {String}' + multiplicity: ManyToOne + - name: number_of_seats + roles: + - concept: Integer + verbalizes: + - '{Aircraft} has {Integer} seats' + multiplicity: ManyToOne + - name: tailnum + roles: + - concept: TailNr + verbalizes: + - '{Aircraft} is identified by {TailNr}' + multiplicity: OneToOne + - name: model + roles: + - concept: Model + verbalizes: + - '{Aircraft} has {Model}' + multiplicity: ManyToOne + - name: year_manufactured + roles: + - concept: Year + verbalizes: + - '{Aircraft} was manufactured in {Year}' + multiplicity: ManyToOne + - name: capacity + roles: + - concept: Capacity + verbalizes: + - '{Aircraft} has {Capacity}' + multiplicity: ManyToOne + - name: carrier + roles: + - concept: Carrier + verbalizes: + - '{Aircraft} is operated by {Carrier}' + multiplicity: ManyToOne +- concept: + name: AirportName + type: ValueType + extends: [ String ] +- concept: + name: AirportCode + description: "The three-letter IATA code for the airport." + type: ValueType + extends: [ String ] +- concept: + name: AirportId + description: "Five digit number used as an alternate identifier for airports." + type: ValueType + extends: [ String ] +- concept: + name: Airport + type: EntityType + identify_by: [ code ] + requires: [ Airport.latitude, Airport.longitude ] + relationships: + - name: city + roles: + - concept: City + verbalizes: + - '{Airport} is located in {City}' + multiplicity: ManyToOne + - name: serves + roles: + - concept: Market + verbalizes: + - '{Airport} serves {Market}' + multiplicity: ManyToOne + - name: average_departure_delay + roles: + - concept: Delay + verbalizes: + - '{Airport} has average- departure {Delay}' # The hyphen after "average" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Airport has at most one average departure Delay + derived_by: [ 'Delay == AVG[Flight.departure_delay WHERE Airport == Flight.route.departure GROUP BY Airport]' ] + - name: longitude + roles: + - concept: DegreesLongitude + verbalizes: + - '{Airport} centers at {DegreesLongitude}' + multiplicity: ManyToOne + - name: average_arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Airport} has average- arrival {Delay}' # The hyphen after "average" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Airport has at most one average arrival Delay + derived_by: [ 'Delay == AVG[Flight.arrival_delay WHERE Airport == Flight.route.destination GROUP BY Airport]' ] + - name: name + roles: + - concept: AirportName + verbalizes: + - '{Airport} has {AirportName}' + multiplicity: ManyToOne + - name: code + roles: + - concept: AirportCode + verbalizes: + - '{Airport} has {AirportCode}' + multiplicity: OneToOne + - name: latitude + roles: + - concept: DegreesLatitude + verbalizes: + - '{Airport} centers at {DegreesLatitude}' + multiplicity: ManyToOne +- concept: + name: MarketName + type: ValueType + extends: [ String ] +- concept: + name: Market + type: EntityType + identify_by: [name] + relationships: + - name: name + roles: [ { concept: MarketName }] + verbalizes: [ '{Market} is identified by {MarketName}' ] + multiplicity: OneToOne +- concept: + name: FlightNr + description: "The IATA flight number, which is typically a combination of the airline's IATA code and a numeric code (e.g., 'AA1234')." + type: ValueType + extends: [ String ] +- concept: + name: FlightId + description: "A unique identifier for an instance of a flight" + type: ValueType + extends: [ String ] +- concept: + name: Flight + type: EntityType + identify_by: [ id ] + relationships: + - name: departure_delay + roles: + - concept: Delay + verbalizes: + - '{Flight} has departure- {Delay}' # The hyphen after "departure" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Flight has at most one departure Delay + - name: number + roles: + - concept: FlightNr + verbalizes: + - '{Flight} has {FlightNr}' + multiplicity: ManyToOne + - name: scheduled_departure + roles: [ { concept: DateTime } ] + verbalizes: + - '{Flight} is scheduled to depart at {DateTime}' + multiplicity: ManyToOne + requires: [ Flight.scheduled_departure < Flight.scheduled_arrival ] + - name: arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Flight} has arrival- {Delay}' # The hyphen after "arrival" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Flight has at most one arrival Delay + - name: canceled + verbalizes: + - '{Flight} was canceled' + - name: canceled_due_to + roles: + - concept: CancelationReason + verbalizes: + - '{Flight} was canceled due to {CancelationReason}' + multiplicity: ManyToOne + - name: distance + roles: + - concept: Distance + verbalizes: + - '{Flight} spans actual- {Distance}' # The hyphen after "actual" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Flight spans at most one actual Distance + - name: id + roles: + - concept: FlightId + verbalizes: + - '{Flight} is identified by {FlightId}' + multiplicity: OneToOne + - name: scheduled_arrival + roles: + - concept: DateTime + verbalizes: + - '{Flight} is scheduled to arrive at {DateTime}' + multiplicity: ManyToOne + - name: registers_longitude_series + roles: + - concept: DateTime + - concept: DegreesLongitude + verbalizes: + - '{Flight} at {DateTime} registers {DegreesLongitude}' + multiplicity: ManyToOne + - name: registers_latitude_series + roles: + - concept: DateTime + - concept: DegreesLatitude + verbalizes: + - '{Flight} at {DateTime} registers {DegreesLatitude}' + multiplicity: ManyToOne + - name: date + roles: + - concept: Date + verbalizes: + - '{Flight} is scheduled to depart on {Date}' + multiplicity: ManyToOne + - name: departs_at + roles: + - concept: DateTime + verbalizes: + - '{Flight} departs at {DateTime}' + multiplicity: ManyToOne + requires: [ Flight.departs_at < Flight.arrives_at ] + - name: diverted + verbalizes: + - '{Flight} was diverted' + - name: arrives_at + roles: + - concept: DateTime + verbalizes: + - '{Flight} arrives at {DateTime}' + multiplicity: ManyToOne + - name: route + roles: + - concept: Route + verbalizes: + - '{Flight} traverses {Route}' + multiplicity: ManyToOne + - name: aircraft + roles: + - concept: Aircraft + verbalizes: + - '{Flight} uses {Aircraft}' + multiplicity: ManyToOne + requires: [ Flight.operated_by(Aircraft.carrier) ] + - name: operated_by + roles: + - concept: Carrier + verbalizes: + - '{Flight} is operated by {Carrier}' + multiplicity: ManyToOne +- concept: + name: CarrierCode + description: "The two-letter IATA code for the airline carrier." + type: ValueType + extends: [ String ] +- concept: + name: CarrierName + type: ValueType + extends: [ String ] +- concept: + name: Carrier + type: EntityType + identify_by: [ code ] + relationships: + - name: code + roles: + - concept: CarrierCode + verbalizes: + - '{Carrier} uses {CarrierCode}' + multiplicity: OneToOne + - name: name + roles: + - concept: CarrierName + verbalizes: + - '{Carrier} has {CarrierName}' + multiplicity: ManyToOne +- concept: + name: RouteId + type: ValueType + description: "A unique identifier for a route between two airports. Constructed by concatenating the IATA codes of the departure and destination airports (e.g., 'ATL -> DCA')" + extends: [ String ] +- concept: + name: Route + type: EntityType + identify_by: [ id ] + relationships: + - name: average_departure_delay + roles: + - concept: Delay + verbalizes: + - '{Route} has average- departure {Delay}' # The hyphen after "average" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Route has at most one average departure Delay + derived_by: [ 'Delay == AVG[Flight.departure_delay WHERE Flight.route(Route) GROUP BY Route]' ] + - name: average_arrival_delay + roles: + - concept: Delay + verbalizes: + - '{Route} has average- arrival {Delay}' # The hyphen after "average" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Route has at most one average arrival Delay + derived_by: [ 'Delay == AVG[Flight.arrival_delay WHERE Flight.route(Route) GROUP BY Route]' ] + - name: distance + roles: + - concept: Distance + verbalizes: + - '{Route} spans {Distance}' + multiplicity: ManyToOne + - name: lies_in + roles: + - concept: DistanceGroup + verbalizes: + - '{Route} has {DistanceGroup}' + multiplicity: ManyToOne + - name: id + roles: + - concept: RouteId + verbalizes: + - '{Route} is identified by {RouteId}' + multiplicity: OneToOne + - name: route_name + roles: + - concept: String + verbalizes: + - '{Route} has name- {String}' # The hyphen after "name" has significance when verbalizing constraints + multiplicity: ManyToOne + - name: destination + roles: + - concept: Airport + verbalizes: + - '{Route} connects to destination- {Airport}' # The hyphen after "destination" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Route connects to at most one destination Airport + requires: [ 'NOT Route.departure(Airport)' ] + - name: departure + roles: + - concept: Airport + verbalizes: + - '{Route} connects to departure- {Airport}' # The hyphen after "departure" has significance when verbalizing constraints + multiplicity: ManyToOne # Each Route connects to at most one departure Airport + requires: [ 'NOT Route.destination(Airport)' ] +ontology_mappings: +- name: flights_mapping + semantic_model: + name: Flights semantic model + datasets: + - name: RUNWAY + source: DATABASE.SCHEMA.RUNWAYS + fields: + - name: airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: airport_code + - name: length + expression: + dialects: + - dialect: ANSI_SQL + expression: length + - name: shape + expression: + dialects: + - dialect: ANSI_SQL + expression: shape + - name: designator + expression: + dialects: + - dialect: ANSI_SQL + expression: designator + - name: AIRCRAFT + source: DATABASE.SCHEMA.AIRCRAFT + description: "An airplane, helicopter, or other machine capable of flight" + fields: + - name: serial_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: serial_nr + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: nr_seats + expression: + dialects: + - dialect: ANSI_SQL + expression: nr_seats + - name: tail_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: tail_nr + - name: carrier_code + expression: + dialects: + - dialect: ANSI_SQL + expression: carrier_code + - name: manufacturer + expression: + dialects: + - dialect: ANSI_SQL + expression: manufacturer + - name: model + expression: + dialects: + - dialect: ANSI_SQL + expression: model + - name: year + expression: + dialects: + - dialect: ANSI_SQL + expression: year + - name: capacity + expression: + dialects: + - dialect: ANSI_SQL + expression: capacity + - name: AIRPORT + source: DATABASE.SCHEMA.AIRPORTS + description: "An airport that is identified by an IATA code." + fields: + - name: state_code + expression: + dialects: + - dialect: ANSI_SQL + expression: state_code + - name: market_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: market_nm + - name: longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: longitude + - name: opened + expression: + dialects: + - dialect: ANSI_SQL + expression: opened + - name: state_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: state_nm + - name: city_nm + expression: + dialects: + - dialect: ANSI_SQL + expression: city_nm + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: code + expression: + dialects: + - dialect: ANSI_SQL + expression: code + - name: latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: latitude + - name: FLIGHT + source: DATABASE.SCHEMA.FLIGHTS + description: "A commercial passenger flight." + fields: + - name: dep_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: dep_delay + - name: air_time + expression: + dialects: + - dialect: ANSI_SQL + expression: air_time + - name: nr + expression: + dialects: + - dialect: ANSI_SQL + expression: nr + - name: carrier_code + expression: + dialects: + - dialect: ANSI_SQL + expression: carrier_code + - name: duration + expression: + dialects: + - dialect: ANSI_SQL + expression: duration + - name: scheduled_departure + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_departure + - name: arr_delay + expression: + dialects: + - dialect: ANSI_SQL + expression: arr_delay + - name: cancelled + expression: + dialects: + - dialect: ANSI_SQL + expression: cancelled + - name: cancel_code + expression: + dialects: + - dialect: ANSI_SQL + expression: cancel_code + - name: distance + expression: + dialects: + - dialect: ANSI_SQL + expression: distance + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + - name: scheduled_arrival + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_arrival + - name: wheels_on + expression: + dialects: + - dialect: ANSI_SQL + expression: wheels_on + - name: date + expression: + dialects: + - dialect: ANSI_SQL + expression: date + - name: wheels_off + expression: + dialects: + - dialect: ANSI_SQL + expression: wheels_off + - name: scheduled_duration + expression: + dialects: + - dialect: ANSI_SQL + expression: scheduled_duration + - name: tail_nr + expression: + dialects: + - dialect: ANSI_SQL + expression: tail_nr + - name: departure + expression: + dialects: + - dialect: ANSI_SQL + expression: departure + - name: route_id + expression: + dialects: + - dialect: ANSI_SQL + expression: route_id + - name: diverted + expression: + dialects: + - dialect: ANSI_SQL + expression: diverted + - name: arrival + expression: + dialects: + - dialect: ANSI_SQL + expression: arrival + - name: CARRIER + source: DATABASE.SCHEMA.CARRIERS + description: "An airline, such as Delta, United, or American." + fields: + - name: code + expression: + dialects: + - dialect: ANSI_SQL + expression: code + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + - name: ROUTE + source: DATABASE.SCHEMA.ROUTES + description: "Represents the existence of one or more flights between a pair of departure + arrival airports." + fields: + - name: orig_airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: orig_airport_code + - name: dest_airport_code + expression: + dialects: + - dialect: ANSI_SQL + expression: dest_airport_code + - name: distance + expression: + dialects: + - dialect: ANSI_SQL + expression: distance + - name: dist_grp + expression: + dialects: + - dialect: ANSI_SQL + expression: dist_grp + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + - name: name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + concept_mappings: + - concept: Runway + object_mappings: + - referent_mappings: + - relationship: designator + expression: RUNWAY.designator + - relationship: airport + referent_mappings: + - relationship: code + expression: RUNWAY.airport_code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: designator + expression: RUNWAY.designator + - relationship: airport + referent_mappings: + - relationship: code + expression: RUNWAY.airport_code + children: + - object_mapping: + concept: RunwayLength + expression: RUNWAY.length + relationship: length + - object_mapping: + concept: RunwayGeometry + expression: RUNWAY.shape + relationship: geometry + - concept: Manufacturer + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer + - concept: Model + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRCRAFT.model + - relationship: manufacturer + referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer + - concept: Aircraft + object_mappings: + - referent_mappings: + - relationship: tailnum + expression: AIRCRAFT.tail_nr + link_mappings: + - object_mapping: + referent_mappings: + - relationship: tailnum + expression: AIRCRAFT.tail_nr + children: + - object_mapping: + concept: SerialNr + expression: AIRCRAFT.serial_nr + relationship: serial_number + - object_mapping: + concept: String + expression: AIRCRAFT.name + relationship: name + - object_mapping: + concept: Integer + expression: AIRCRAFT.nr_seats + relationship: number_of_seats + - object_mapping: + concept: Model + referent_mappings: + - relationship: name + expression: AIRCRAFT.model + - relationship: manufacturer + referent_mappings: + - relationship: name + expression: AIRCRAFT.manufacturer + relationship: model + - object_mapping: + concept: Year + expression: AIRCRAFT.year + relationship: year_manufactured + - object_mapping: + concept: Capacity + expression: AIRCRAFT.capacity + relationship: capacity + - object_mapping: + concept: Carrier + referent_mappings: + - relationship: code + expression: AIRCRAFT.carrier_code + relationship: carrier + - concept: State + object_mappings: + - referent_mappings: + - relationship: code + expression: AIRPORT.state_code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + children: + - object_mapping: + concept: StateName + expression: AIRPORT.state_nm + relationship: name + - concept: City + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRPORT.city_nm + - relationship: state + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + - concept: Market + object_mappings: + - referent_mappings: + - relationship: name + expression: AIRPORT.market_nm + - concept: Airport + object_mappings: + - referent_mappings: + - relationship: code + expression: AIRPORT.code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: AIRPORT.code + children: + - object_mapping: + concept: City + referent_mappings: + - relationship: name + expression: AIRPORT.city_nm + - relationship: state + referent_mappings: + - relationship: code + expression: AIRPORT.state_code + relationship: city + - object_mapping: + concept: Market + referent_mappings: + - relationship: name + expression: AIRPORT.market_nm + relationship: serves + - object_mapping: + concept: DegreesLongitude + expression: AIRPORT.longitude + relationship: longitude + - object_mapping: + concept: AirportName + expression: AIRPORT.name + relationship: name + - object_mapping: + concept: DegreesLatitude + expression: AIRPORT.latitude + relationship: latitude + - concept: Flight + object_mappings: + - referent_mappings: + - relationship: id + expression: FLIGHT.id + link_mappings: + - object_mapping: + referent_mappings: + - relationship: id + expression: FLIGHT.id WHERE ( FLIGHT.diverted == TRUE ) + relationship: diverted + - object_mapping: + referent_mappings: + - relationship: id + expression: FLIGHT.id WHERE ( FLIGHT.cancelled == TRUE ) + relationship: canceled + - object_mapping: + referent_mappings: + - relationship: id + expression: FLIGHT.id + children: + - object_mapping: + concept: Delay + expression: FLIGHT.arr_delay + relationship: arrival_delay + - object_mapping: + concept: Delay + expression: FLIGHT.dep_delay + relationship: departure_delay + - object_mapping: + concept: FlightNr + expression: FLIGHT.nr + relationship: number + - object_mapping: + concept: DateTime + expression: FLIGHT.scheduled_departure + relationship: scheduled_departure + - object_mapping: + concept: CancelationReason + referent_mappings: + - relationship: code + expression: FLIGHT.cancel_code + relationship: canceled_due_to + - object_mapping: + concept: Distance + expression: FLIGHT.distance + relationship: distance + - object_mapping: + concept: DateTime + expression: FLIGHT.scheduled_arrival + relationship: scheduled_arrival + - object_mapping: + concept: Date + expression: FLIGHT.date + relationship: date + - object_mapping: + concept: DateTime + expression: FLIGHT.departure + relationship: departs_at + - object_mapping: + concept: DateTime + expression: FLIGHT.arrival + relationship: arrives_at + - object_mapping: + concept: Route + referent_mappings: + - relationship: id + expression: FLIGHT.route_id + relationship: route + - object_mapping: + concept: Aircraft + referent_mappings: + - relationship: tailnum + expression: FLIGHT.tail_nr + relationship: aircraft + - object_mapping: + concept: Carrier + referent_mappings: + - relationship: code + expression: FLIGHT.carrier_code + relationship: operated_by + - concept: Carrier + object_mappings: + - referent_mappings: + - relationship: code + expression: CARRIER.code + link_mappings: + - object_mapping: + referent_mappings: + - relationship: code + expression: CARRIER.code + children: + - object_mapping: + concept: CarrierName + expression: CARRIER.name + relationship: name + - concept: Route + object_mappings: + - referent_mappings: + - relationship: id + expression: ROUTE.id + link_mappings: + - object_mapping: + referent_mappings: + - relationship: id + expression: ROUTE.id + children: + - object_mapping: + concept: Distance + expression: ROUTE.distance + relationship: distance + - object_mapping: + concept: DistanceGroup + expression: ROUTE.dist_grp + relationship: lies_in + - object_mapping: + concept: String + expression: ROUTE.name + relationship: route_name + - object_mapping: + concept: Airport + referent_mappings: + - relationship: code + expression: ROUTE.dest_airport_code + relationship: destination + - object_mapping: + concept: Airport + referent_mappings: + - relationship: code + expression: ROUTE.orig_airport_code + relationship: departure diff --git a/converters/ontology/tests/test_examples_in_sync.py b/converters/ontology/tests/test_examples_in_sync.py new file mode 100644 index 00000000..b2694f19 --- /dev/null +++ b/converters/ontology/tests/test_examples_in_sync.py @@ -0,0 +1,34 @@ +"""Guard against drift between the vendored test inputs under tests/fixtures/ and +the canonical copies under the repo-level examples/ directory. + +The core test suite reads the vendored copies (so it runs from an sdist/wheel or +subset checkout). This test only runs when examples/ is available — it fails if a +vendored file falls out of sync, prompting a refresh. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +_FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" +# tests/ -> ontology -> converters -> +_EXAMPLES_DIR = Path(__file__).resolve().parents[3] / "examples" + +# (vendored filename, canonical filename in examples/) +_VENDORED = [("flights.yaml", "flights.yaml")] + + +@pytest.mark.parametrize("vendored_name, example_name", _VENDORED) +def test_vendored_input_matches_example(vendored_name: str, example_name: str): + example_path = _EXAMPLES_DIR / example_name + if not example_path.is_file(): + pytest.skip(f"canonical example not present at {example_path}") + + vendored_path = _FIXTURES_DIR / vendored_name + assert vendored_path.is_file(), f"vendored input missing: {vendored_path}" + assert vendored_path.read_text(encoding="utf-8") == example_path.read_text(encoding="utf-8"), ( + f"'{vendored_path}' is out of sync with '{example_path}'. " + f"Refresh it with: cp {example_path} {vendored_path}" + ) \ No newline at end of file From fd9166692036665f91a302c4c78ad84e487db775 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 22 Jul 2026 13:35:35 +0200 Subject: [PATCH 26/29] Refactored after the internal code review --- .../ontology/src/osi/common/file_utils.py | 2 +- .../converter/palantir_to_osi/converter.py | 33 +++++++++++-------- .../src/osi/external/palantir/model.py | 10 +++--- .../osi/external/palantir/parser/__init__.py | 16 ++++----- .../ontology/src/osi/parser/__init__.py | 3 +- 5 files changed, 34 insertions(+), 30 deletions(-) diff --git a/converters/ontology/src/osi/common/file_utils.py b/converters/ontology/src/osi/common/file_utils.py index 725bae95..3fbf2fe1 100644 --- a/converters/ontology/src/osi/common/file_utils.py +++ b/converters/ontology/src/osi/common/file_utils.py @@ -82,7 +82,7 @@ def get_top_level_json_file_from_dir(base_dir: Path) -> Path: if p.is_file() and p.suffix.lower() == ".json" ] if len(candidates) == 0: - raise FileNotFoundError("Directory must contain exactly one top-level JSON file (none found)") + raise ValueError("Directory must contain exactly one top-level JSON file (none found)") if len(candidates) > 1: raise ValueError("Directory must contain exactly one top-level JSON file (multiple found)") return candidates[0] diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index eb7a0ac8..88ed4117 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -27,7 +27,6 @@ DatasetField, DialectExpression, DialectExpressionSet, - Formula, FormulaFactory, LinkMapping, SemanticModel, @@ -574,7 +573,7 @@ def _convert_many_to_many(self, ontology: OntologyComponent, rel: ManyToManyRela if aot_concept is bot_concept: verbalize = f"{{{aot_concept}}} {rel_name} {{{bot_concept}:snd}}" - relates = [(bot_concept, "snd")] + relates: list[tuple[Concept, str | None]] = [(bot_concept, "snd")] else: verbalize = f"{{{aot_concept}}} {rel_name} {{{bot_concept}}}" relates = [(bot_concept, None)] @@ -624,18 +623,12 @@ def _convert_intermediary_relation( rel_b = palantir_ontology.relations()[rel.relation_b()] rel_b_name = PalantirToOsiConverter._attribute_name(rel_b) - fp_a = PalantirToOsiConverter._concept_name( - rel_a.many_object_type() if isinstance(rel_a, ManyToOneRelation) else rel_a.role_a_player() - ) - sp_a = PalantirToOsiConverter._concept_name( - rel_a.one_object_type() if isinstance(rel_a, ManyToOneRelation) else rel_a.role_b_player() - ) - fp_b = PalantirToOsiConverter._concept_name( - rel_b.many_object_type() if isinstance(rel_b, ManyToOneRelation) else rel_b.role_a_player() - ) - sp_b = PalantirToOsiConverter._concept_name( - rel_b.one_object_type() if isinstance(rel_b, ManyToOneRelation) else rel_b.role_b_player() - ) + fp_a_ot, sp_a_ot = PalantirToOsiConverter._relation_players(rel_a) + fp_a = PalantirToOsiConverter._concept_name(fp_a_ot) + sp_a = PalantirToOsiConverter._concept_name(sp_a_ot) + fp_b_ot, sp_b_ot = PalantirToOsiConverter._relation_players(rel_b) + fp_b = PalantirToOsiConverter._concept_name(fp_b_ot) + sp_b = PalantirToOsiConverter._concept_name(sp_b_ot) assert (aot_name == fp_a and bot_name == fp_b) or ( aot_name == sp_a and bot_name == sp_b @@ -747,6 +740,18 @@ def _convert_property_type_roles( roles.append((target, None)) return roles + @staticmethod + def _relation_players(rel: Relation) -> tuple[ObjectType, ObjectType]: + """Return the (first, second) role-player object types of a binary + relation, regardless of its concrete relation type.""" + if isinstance(rel, ManyToOneRelation): + return rel.many_object_type(), rel.one_object_type() + if isinstance(rel, (ManyToManyRelation, IntermediaryRelation)): + return rel.role_a_player(), rel.role_b_player() + raise ValueError( + f"Unsupported relation type '{type(rel).__name__}' for relation {rel.readable_id()}" + ) + @staticmethod def _depth_role_name(depth: int) -> str: name = PalantirToOsiConverter.depths_role_names.get(depth) diff --git a/converters/ontology/src/osi/external/palantir/model.py b/converters/ontology/src/osi/external/palantir/model.py index 35a73a8b..68687e19 100644 --- a/converters/ontology/src/osi/external/palantir/model.py +++ b/converters/ontology/src/osi/external/palantir/model.py @@ -39,7 +39,7 @@ def parse_datatype(name: str) -> DataType: try: return DataType[name.upper()] except KeyError: - raise ValueError(f"Unrecognized data type: {name}") + raise ValueError(f"Unrecognized data type: {name}") from None def to_type(self) -> str: if self in (DataType.STRING, DataType.GEOHASH, DataType.GEOSHAPE, DataType.GEOPOINT, DataType.TIMESERIES): @@ -509,7 +509,7 @@ def __init__(self, guid, rid, many_object_type: ObjectType, one_object_type: Obj def info(self) -> str: one_role = self._one_object_type many_role = self._many_object_type - return f'Relation "{self.readable_id()}" maps "{many_role._name}" to "{one_role._name}"' + return f'Relation "{self.readable_id()}" maps "{many_role.name()}" to "{one_role.name()}"' def many_object_type(self): return self._many_object_type @@ -597,7 +597,7 @@ def set_backing_datasource_id(self, id): self._backing_datasource_id = id def info(self): - result = [f'Relation "{self.readable_id()}" associates "{self.role_a_player()._name}" with "{self.role_b_player()._name}"'] + result = [f'Relation "{self.readable_id()}" associates "{self.role_a_player().name()}" with "{self.role_b_player().name()}"'] if self._data_set: result.append(f' DataSet "{self.data_set().readable_id()}"') return "\n".join(result) @@ -629,7 +629,7 @@ def relation_b(self): return self._relation_b def info(self): - return (f'Relation "{self.readable_id()}" associates "{self.role_a_player()._name}" with ' - f'"{self.role_b_player()._name}" via intermediary player "{self.intermediary_player()._name}" and ' + return (f'Relation "{self.readable_id()}" associates "{self.role_a_player().name()}" with ' + f'"{self.role_b_player().name()}" via intermediary player "{self.intermediary_player().name()}" and ' f'relations "{self.relation_a()}" and "{self.relation_b()}"') diff --git a/converters/ontology/src/osi/external/palantir/parser/__init__.py b/converters/ontology/src/osi/external/palantir/parser/__init__.py index 071fbc62..c0cb4c50 100644 --- a/converters/ontology/src/osi/external/palantir/parser/__init__.py +++ b/converters/ontology/src/osi/external/palantir/parser/__init__.py @@ -213,8 +213,9 @@ def _parse_object_types(self, data: dict) -> tuple[dict[str, ObjectType], dict[s properties = {} properties_by_readable_id = {} - # Support both formats: list (new) and dict (old) - raw_properties = raw_ot.get("properties", []) + # Support both formats: list (new) and dict (old). Use `or []` so a + # present-but-null `properties` is treated the same as missing. + raw_properties = raw_ot.get("properties") or [] if isinstance(raw_properties, dict): # Old format: properties is a dict keyed by property name raw_properties = list(raw_properties.values()) @@ -245,8 +246,8 @@ def _parse_object_types(self, data: dict) -> tuple[dict[str, ObjectType], dict[s primary_key_mapping = get_dict(raw_prop, "primaryKeyMapping") if primary_key_mapping: pk_mapping = {} - for k,v in primary_key_mapping.items(): - pk_column_name = norm(v.get("columnName")) + for k in primary_key_mapping: + pk_column_name = norm(get_dict(primary_key_mapping, k).get("columnName")) pk_mapping[k] = pk_column_name prop._pk_mapping = pk_mapping @@ -610,11 +611,8 @@ def _data_set_streams() -> Iterable[IOBase]: for _name, fh in iter_json_files_from_dir_in_zip(zf, "data_sets"): yield fh - try: - with open_top_level_file_from_zip(zf, self._get_ontology_json_file_path(zf)) as ontology_fh: - self._build_model(_data_set_streams(), ontology_fh) - except FileNotFoundError as e: - raise FileNotFoundError(str(e)) from e + with open_top_level_file_from_zip(zf, self._get_ontology_json_file_path(zf)) as ontology_fh: + self._build_model(_data_set_streams(), ontology_fh) def _parse_from_dir(self, base_dir: Path): validate_dir(base_dir) diff --git a/converters/ontology/src/osi/parser/__init__.py b/converters/ontology/src/osi/parser/__init__.py index a6d175c2..0b6e0d87 100644 --- a/converters/ontology/src/osi/parser/__init__.py +++ b/converters/ontology/src/osi/parser/__init__.py @@ -4,6 +4,7 @@ import json from pathlib import Path +from typing import Any import yaml @@ -38,7 +39,7 @@ def parse(self, path: Path) -> OsiOntology: return self._model @staticmethod - def load_data(path: Path): + def load_data(path: Path) -> Any: # Pin UTF-8 so parsing is reproducible regardless of the process locale. content = path.read_text(encoding="utf-8") if path.suffix.lower() == ".json": From bef38b5727c60749aaf809418bbce026663a5c51 Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Wed, 22 Jul 2026 14:38:53 +0200 Subject: [PATCH 27/29] Refactored after the Copilot code review --- .../converter/palantir_to_osi/converter.py | 5 +- .../osi/converter/spec_to_osi/converter.py | 17 +- converters/ontology/src/osi/model.py | 17 ++ .../tests/test_verbalization_parsing.py | 184 ++++++++++++++++++ 4 files changed, 220 insertions(+), 3 deletions(-) create mode 100644 converters/ontology/tests/test_verbalization_parsing.py diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py index 88ed4117..85bd8a80 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/palantir_to_osi/converter.py @@ -661,7 +661,10 @@ def _convert_dataset( fields: list[DatasetField] = [] for column in palantir_ds.columns(): - if column.type().upper() == "ARRAY": + # type() may be None for a partially-specified schema; only ARRAY + # columns are skipped, and a missing type falls back to String below. + col_type = column.type() + if col_type is not None and col_type.upper() == "ARRAY": continue field_name = PalantirToOsiConverter._normalize_field_name(column.name()) fields.append( diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index a60c6db0..0311ef52 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -356,8 +356,20 @@ def _build_rule(self, raw: str | None, parent: Container | None, ontology: Ontol def _resolve_mapping_expression(self, expression: str, semantic_model: SemanticModel, expected_type: Concept | None, ontology: OntologyComponent) -> DatasetField | Formula: - """Map a raw spec expression onto either a DatasetField (single - `DATASET.field` or bare `field` reference) or a Formula (anything else). + """Map a raw spec expression onto either a DatasetField or a Formula. + + A `DATASET.field` or unambiguous bare `field` reference that resolves + against *semantic_model* is returned as the corresponding DatasetField. + + Everything else — including a `DATASET.field`-shaped expression whose + dataset or field is not found here — is delegated to the + MappingFormulaFactory, which receives both the ontology and the semantic + model. Name resolution and validation of such expressions is deliberately + the factory's responsibility, not this method's: the default factory + wraps the raw text in a plain Formula, while downstream packages inject + enriched factories that parse, resolve, and validate references (e.g. + against constructs the base semantic-model lookup cannot see). An unknown + `DATASET.field` is therefore not treated as an error at this layer. """ qualified = _QUALIFIED_FIELD_RE.match(expression) if qualified: @@ -368,6 +380,7 @@ def _resolve_mapping_expression(self, expression: str, semantic_model: SemanticM if field is not None: _pin_field_type(field, expected_type) return field + # Deferred to the factory by design (see docstring), not an error here. return self._mapping_formula_factory(raw_expr=expression, ontology=ontology, semantic_model=semantic_model) bare = _BARE_FIELD_RE.match(expression) diff --git a/converters/ontology/src/osi/model.py b/converters/ontology/src/osi/model.py index 3af67d10..298dc6e3 100644 --- a/converters/ontology/src/osi/model.py +++ b/converters/ontology/src/osi/model.py @@ -1129,16 +1129,33 @@ def _parse_verbalization(relationship: Relationship, verbalization: str) -> Rela def _split_segment(segment: str) -> tuple[str | None, str | None, str | None]: + """Split a text segment into ``(postfix, middle, prefix)`` using the dash + conventions (dashes are stripped from the returned text): + + - **postfix** — text glued to the *preceding* concept. A leading-dash word + (e.g. ``-box``) marks the segment's opening run as a postfix: the first + word plus any following dash-prefixed words (so ``"big -box"`` -> ``"big box"``). + - **prefix** — text glued to the *following* concept. A trailing-dash word + (e.g. ``chain-``) marks the start of the closing run (so ``"chain- super"`` + -> ``"chain super"``). + - **middle** — the free text left between the postfix and prefix runs. + + See tests/test_verbalization_parsing.py for the full set of expected splits. + """ words = segment.split() if not words: return None, None, None + # Postfix run: first word plus any following dash-prefixed words, but only + # when the segment actually contains a leading-dash marker somewhere. postfix_end = 0 if any(w.startswith("-") for w in words): postfix_end = 1 while postfix_end < len(words) and words[postfix_end].startswith("-"): postfix_end += 1 + # Prefix run: begins at the first trailing-dash word (after the postfix run) + # and extends to the end of the segment. prefix_start = len(words) for i in range(postfix_end, len(words)): if words[i].endswith("-"): diff --git a/converters/ontology/tests/test_verbalization_parsing.py b/converters/ontology/tests/test_verbalization_parsing.py new file mode 100644 index 00000000..2b82846c --- /dev/null +++ b/converters/ontology/tests/test_verbalization_parsing.py @@ -0,0 +1,184 @@ +"""Tests for verbalization parsing — the dash-convention text splitting in +``_parse_verbalization`` / ``_split_segment``. + +These lock in the intended prefix/postfix/text semantics: + - a trailing dash (``chain-``) marks a *prefix* attached to the following concept; + - a segment containing a leading-dash word (``-box``) is a *postfix* group + attached to the preceding concept — the first word plus any following + dash-prefixed words (so ``big -box`` -> postfix ``"big box"``). +""" + +from __future__ import annotations + +import re + +import pytest + +from osi.model import Concept, Relationship, _parse_verbalization as parse_verbalization + + +def _rel(name: str, container: Concept, relates: list[tuple[Concept, str | None]], verbalization: str) -> Relationship: + return Relationship(name, container, relates, verbalizes=[verbalization]) + + +def test_all_possible_postfixes_prefixes(): + store_concept = Concept("Store") + item_concept = Concept("Item") + amount_concept = Concept("Amount") + verbalization = ( + "every chain- super {Store} reports returns of {Item}" + " big -box for average- {Amount:amt}" + ) + rel = _rel( + "chain_report", + store_concept, + [(item_concept, None), (amount_concept, "amt")], + verbalization, + ) + roles = parse_verbalization(rel, verbalization).roles + assert len(roles) == 3 + + store = roles[0] + assert store.concept == store_concept + assert store.name is None + assert store.preceding_text == "every" + assert store.prefix == "chain super" + assert store.following_text == "reports returns of" + assert store.postfix is None + + item = roles[1] + assert item.concept == item_concept + assert item.name is None + assert item.prefix is None + assert item.preceding_text is None + assert item.postfix == "big box" + + amount = roles[2] + assert amount.concept == amount_concept + assert amount.name == "amt" + assert amount.preceding_text is None + assert amount.prefix == "average" + assert amount.postfix is None + assert amount.following_text is None + + +def test_no_text_decoration(): + verbalization = "{Person} earns {Salary} annually" + rel = _rel("earns", Concept("Person"), [(Concept("Salary"), None)], verbalization) + roles = parse_verbalization(rel, verbalization).roles + assert roles[0].following_text == "earns" + assert roles[0].prefix is None + assert roles[0].postfix is None + assert roles[1].following_text == "annually" + + +def test_single_word_prefix(): + verbalization = "big- {Store} sells {Item}" + rel = _rel("sells", Concept("Store"), [(Concept("Item"), None)], verbalization) + roles = parse_verbalization(rel, verbalization).roles + assert roles[0].prefix == "big" + assert roles[0].preceding_text is None + + +def test_multi_word_prefix_before_first_concept(): + verbalization = "every chain- super {Store}" + rel = _rel("r", Concept("Store"), [], verbalization) + roles = parse_verbalization(rel, verbalization).roles + assert roles[0].preceding_text == "every" + assert roles[0].prefix == "chain super" + + +def test_multi_word_postfix_before_end(): + verbalization = "{Item} big -box end" + rel = _rel("r", Concept("Item"), [], verbalization) + roles = parse_verbalization(rel, verbalization).roles + assert roles[0].postfix == "big box" + assert roles[0].following_text == "end" + + +def test_postfix_directly_after_last_concept(): + verbalization = "{Person} big -city" + rel = _rel("r", Concept("Person"), [], verbalization) + roles = parse_verbalization(rel, verbalization).roles + assert roles[-1].postfix == "big city" + assert roles[-1].following_text is None + + +def test_only_following_text_after_last_concept(): + verbalization = "{Person} is active" + rel = _rel("r", Concept("Person"), [], verbalization) + roles = parse_verbalization(rel, verbalization).roles + assert roles[0].following_text == "is active" + assert roles[0].postfix is None + + +def test_ternary_relationship(): + """Three-role verbalization with inter-concept prefix markers. + A trailing-dashed word between two tokens is a prefix for the *following* concept.""" + verbalization = "{Supplier} delivers- {Item} to- {Warehouse} on {Date}" + rel = _rel( + "delivers", + Concept("Supplier"), + [(Concept("Item"), None), (Concept("Warehouse"), None), (Concept("Date"), None)], + verbalization, + ) + roles = parse_verbalization(rel, verbalization).roles + assert len(roles) == 4 + assert roles[0].prefix is None + assert roles[0].postfix is None + assert roles[1].prefix == "delivers" # 'delivers-' -> prefix of Item + assert roles[2].prefix == "to" # 'to-' -> prefix of Warehouse + assert roles[2].following_text == "on" + assert roles[3].prefix is None + + +def test_verbalization_role_concept_not_in_model_raises(): + """Verbalization says {Invalid} but the relationship role plays Ghost.""" + verbalization = "{Invalid} haunts {Person}" + with pytest.raises(ValueError, match=re.escape("Invalid")): + _rel("r", Concept("Ghost"), [(Concept("Person"), None)], verbalization) + + +def test_builtin_number_concept_resolves(): + """Number(p,s) is a valid concept name with parentheses.""" + number_concept = Concept("Number(12,4)") + verbalization = "{Order} has total {Number(12,4)}" + rel = _rel("has_total", Concept("Order"), [(number_concept, None)], verbalization) + roles = parse_verbalization(rel, verbalization).roles + assert len(roles) == 2 + assert roles[1].concept.name == "Number(12,4)" + + +def test_builtin_integer_concept_resolves(): + integer_concept = Concept("Integer") + verbalization = "{Product} has quantity {Integer}" + rel = _rel("has_qty", Concept("Product"), [(integer_concept, None)], verbalization) + roles = parse_verbalization(rel, verbalization).roles + assert roles[1].concept.name == "Integer" + + +def test_wrong_concept_name_in_token_raises(): + """Verbalization says {Item} but the relationship role plays Store.""" + verbalization = "{Item}" + with pytest.raises(ValueError, match="does not match verbalization role"): + _rel("r", Concept("Store"), [], verbalization) + + +def test_wrong_role_name_in_token_raises(): + """Verbalization says {Store:wrong} but the relationship role name is unset.""" + verbalization = "{Store:wrong}" + with pytest.raises(ValueError, match="does not match verbalization role"): + _rel("r", Concept("Store"), [], verbalization) + + +def test_same_concept_different_roles_valid(): + """Same concept (Person) playing two roles with distinct names is valid as + long as the verbalization tokens match the relationship roles.""" + person = Concept("Person") + verbalization = "{Person} has parent {Person:parent}" + rel = _rel("parenthood", person, [(person, "parent")], verbalization) + roles = parse_verbalization(rel, verbalization).roles + assert len(roles) == 2 + assert roles[0].name is None + assert roles[1].name == "parent" + assert roles[0].concept is roles[1].concept \ No newline at end of file From 09b1bd042176e24d033ec716cf974c361aff231c Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Fri, 24 Jul 2026 14:51:01 +0200 Subject: [PATCH 28/29] Migrated to a new flatten Ossie structure --- .../osi/converter/osi_to_spec/converter.py | 25 +- .../osi/converter/spec_to_osi/converter.py | 26 +- converters/ontology/src/osi/spec.py | 34 +- .../ontology/tests/fixtures/flights.yaml | 354 ++++++------- .../flights_roundtrip.yaml | 466 ++++++++---------- 5 files changed, 403 insertions(+), 502 deletions(-) diff --git a/converters/ontology/src/osi/converter/osi_to_spec/converter.py b/converters/ontology/src/osi/converter/osi_to_spec/converter.py index f73d6f3b..f6c5be5d 100644 --- a/converters/ontology/src/osi/converter/osi_to_spec/converter.py +++ b/converters/ontology/src/osi/converter/osi_to_spec/converter.py @@ -26,7 +26,6 @@ OsiOntology, ) from osi.spec import ( - Concept as SpecConcept, ConceptComponent, ConceptMapping as SpecConceptMapping, CustomExtension as SpecCustomExtension, @@ -76,33 +75,25 @@ def _convert_ontology_concepts(ont: OntologyComponent) -> list[ConceptComponent] if not concept.is_component: continue rels = [rel for rel in ont.relationships if rel.container is concept] - components.append( - ConceptComponent( - concept=_convert_concept(concept), - relationships=[_convert_relationship(rel) for rel in rels], - ) - ) + components.append(_convert_concept(concept, rels)) return components -def _convert_concept(concept: Concept) -> SpecConcept: +def _convert_concept(concept: Concept, rels: list[Relationship]) -> ConceptComponent: type_value: str | None = None if isinstance(concept.type, ConceptType): type_value = concept.type.value # type: ignore[union-attr] extends = [p.name for p in concept.extends] if concept.extends else None - identify_by: list[str] = [rel.name for rel in concept.identify_by.values()] - derived_by = [f.raw_expr for f in concept.derived_by] - requires = [f.raw_expr for f in concept.requires] - - return SpecConcept( - name=concept.name, + return ConceptComponent( + concept=concept.name, type=type_value, # type: ignore[arg-type] description=concept.description, extends=extends, - identify_by=identify_by, - derived_by=derived_by, - requires=requires, + identify_by=[rel.name for rel in concept.identify_by.values()], + derived_by=[f.raw_expr for f in concept.derived_by], + requires=[f.raw_expr for f in concept.requires], + relationships=[_convert_relationship(rel) for rel in rels], ) diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/osi/converter/spec_to_osi/converter.py index 0311ef52..e68d90a2 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/osi/converter/spec_to_osi/converter.py @@ -32,7 +32,7 @@ BUILTIN_CONCEPTS ) from osi.spec import ( - Concept as SpecConcept, + ConceptComponent as SpecConceptComponent, ConceptMapping as SpecConceptMapping, CustomExtension as SpecCustomExtension, Dataset as SpecDataset, @@ -95,7 +95,7 @@ def convert(self, spec: OsiSpec) -> OsiOntology: def _populate_ontology(self, ontology: OntologyComponent, spec: OsiSpec) -> None: - concept_specs = {concept_component.concept.name: concept_component.concept for concept_component in spec.ontology} + concept_specs = {cc.concept: cc for cc in spec.ontology} sorted_names = self._sort_spec_dependency_graph(list(concept_specs.values())) for name in sorted_names: concept_spec = concept_specs[name] @@ -110,7 +110,7 @@ def _populate_ontology(self, ontology: OntologyComponent, spec: OsiSpec) -> None extends.append(parent) ontology.add_concept( Concept( - name=concept_spec.name, + name=concept_spec.concept, type=ConceptType.from_value(concept_spec.type), description=concept_spec.description, extends=extends, @@ -118,20 +118,19 @@ def _populate_ontology(self, ontology: OntologyComponent, spec: OsiSpec) -> None ) for concept_component in spec.ontology: - container = ontology.lookup_concept(concept_component.concept.name) + container = ontology.lookup_concept(concept_component.concept) if container is None: - raise ValueError(f"Internal: container concept '{concept_component.concept.name}' not found") + raise ValueError(f"Internal: container concept '{concept_component.concept}' not found") for rel_spec in concept_component.relationships: self._convert_relationship(ontology, container, rel_spec) # Identifiers: now that all relationships exist, resolve identify_by. for concept_component in spec.ontology: - concept_spec = concept_component.concept - concept = ontology.lookup_concept(concept_spec.name) + concept = ontology.lookup_concept(concept_component.concept) if concept is None: continue identifiers: dict[str, Relationship] = {} - for ref_name in concept_spec.identify_by: + for ref_name in concept_component.identify_by: rel = ontology.lookup_concept_relationship(concept, ref_name) if rel is None: raise ValueError( @@ -143,15 +142,14 @@ def _populate_ontology(self, ontology: OntologyComponent, spec: OsiSpec) -> None # Formulas: derived_by + requires (after concepts/relationships exist). for concept_component in spec.ontology: - concept_spec = concept_component.concept - concept = ontology.lookup_concept(concept_spec.name) + concept = ontology.lookup_concept(concept_component.concept) if concept is None: continue - for raw in concept_spec.requires: + for raw in concept_component.requires: req = self._build_rule(raw, concept, ontology) if req: concept.add_require(req) - for raw in concept_spec.derived_by: + for raw in concept_component.derived_by: rule = self._build_rule(raw, concept, ontology) if rule: concept.add_derived_by(rule) @@ -408,11 +406,11 @@ def _resolve_mapping_expression(self, expression: str, semantic_model: SemanticM # ----- Structural helpers -------------------------- @staticmethod - def _sort_spec_dependency_graph(concepts: list[SpecConcept]) -> list[str]: + def _sort_spec_dependency_graph(concepts: list[SpecConceptComponent]) -> list[str]: nodes: list[str] = [] edges: list[tuple[str, str]] = [] for concept in concepts: - name = concept.name + name = concept.concept nodes.append(name) if concept.extends: for ext in concept.extends: diff --git a/converters/ontology/src/osi/spec.py b/converters/ontology/src/osi/spec.py index 7144da94..dea4ebd2 100644 --- a/converters/ontology/src/osi/spec.py +++ b/converters/ontology/src/osi/spec.py @@ -62,31 +62,31 @@ class Relationship(OsiObject): requires: list[str] = Field(default_factory=list) -class Concept(OsiObject): - """A type-like node in the ontology — either an `EntityType` (real-world - object referenced via other relationships) or a `ValueType` (a data type - with extra semantics, must transitively extend a built-in value type). +class ConceptComponent(OsiObject): + """A concept and the relationships grouped under it. + + A concept is a type-like node in the ontology — either an `EntityType` + (a real-world object referenced via other relationships) or a `ValueType` + (a data type with extra semantics, which must transitively extend a + built-in value type). + + Mirrors the flattened YAML shape: - `identify_by` lists the names of relationships (declared under this - concept) whose values uniquely reference its objects. + { concept: , type: ..., description: ..., extends: [...], + identify_by: [...], relationships: [...] } + + where `concept` names the concept and every relationship in + `relationships` takes it as the implicit first role. `identify_by` lists + the names of relationships (declared under this concept) whose values + uniquely reference its objects. """ - name: str + concept: str type: Literal["EntityType", "ValueType"] | None = None description: str | None = None extends: list[str] | None = None identify_by: list[str] = Field(default_factory=list) derived_by: list[str] = Field(default_factory=list) requires: list[str] = Field(default_factory=list) - - -class ConceptComponent(OsiObject): - """Envelope for a concept and the relationships nested under it. - - Mirrors the YAML shape `{ concept: {...}, relationships: [...] }` where - every relationship in the list takes the enclosing concept as its - implicit first role. - """ - concept: Concept relationships: list[Relationship] = Field(default_factory=list) diff --git a/converters/ontology/tests/fixtures/flights.yaml b/converters/ontology/tests/fixtures/flights.yaml index f0f64d13..803ff73d 100644 --- a/converters/ontology/tests/fixtures/flights.yaml +++ b/converters/ontology/tests/fixtures/flights.yaml @@ -22,110 +22,88 @@ requires: - COUNT[Airport] > 0 # there must be at least one Airport - COUNT[Carrier] > 0 # there must be at least one Carrier ontology: -- concept: - name: NrFeet - description: "Unit of measure for distance in feet" - type: ValueType - extends: [ Decimal ] -- concept: - name: NrPounds - description: "Unit of measure for weight in pounds" - type: ValueType - extends: [ Integer ] -- concept: - name: NrMiles - description: "Unit of measure for distance in miles" - type: ValueType - extends: [ Decimal ] -- concept: - name: NrMinutes - description: "Unit of measure for time in minutes" - type: ValueType - extends: [ Decimal ] -- concept: - name: CancelationCode - description: "The single character code that identifies the reason a flight is canceled" - type: ValueType - extends: [ String ] - requires: [ CancelationCode == 'A' OR CancelationCode == 'B' OR CancelationCode == 'C' OR CancelationCode == 'D' ] -- concept: - name: Capacity - description: "The capacity of an aircraft, measured in pounds." - type: ValueType - extends: [ NrPounds ] -- concept: - name: DegreesLatitude - type: ValueType - extends: [ Decimal ] - requires: [ DegreesLatitude <= 90, DegreesLatitude >= -90 ] -- concept: - name: DegreesLongitude - type: ValueType - extends: [ Decimal ] - requires: [ DegreesLongitude <= 180, DegreesLongitude >= -180 ] -- concept: - name: Polygon - description: "A polygon represented as a list of vertices, where each vertex is a pair of latitude and longitude coordinates." - type: ValueType - extends: [ String ] -- concept: - name: CityName - type: ValueType - extends: [ String ] -- concept: - name: Delay - type: ValueType - extends: [ NrMinutes ] -- concept: - name: Distance - type: ValueType - extends: [ NrMiles ] -- concept: - name: DistanceGroup - description: "A number used to group distances of different lengths, where 1 groups the shortest and 10 the longest." - type: ValueType - extends: [ Integer ] - requires: [ 1 <= DistanceGroup, DistanceGroup <= 10 ] -- concept: - name: RunwayGeometry - description: "A polygon that models the shape of a runway." - type: ValueType - extends: [ Polygon ] -- concept: - name: RunwayLength - description: "The unit for measuring the lengths of runways in American airports." - type: ValueType - extends: [ NrFeet ] -- concept: - name: RunwayDesignator - description: "Used to distinguish runways within a given airport." - type: ValueType - extends: [ String ] -- concept: - name: SerialNr - type: ValueType - extends: [ String ] -- concept: - name: StateCode - type: ValueType - extends: [ String ] -- concept: - name: StateName - type: ValueType - extends: [ String ] -- concept: - name: TailNr - type: ValueType - extends: [ String ] -- concept: - name: Year - type: ValueType - extends: [ String ] -- concept: - name: CancelationReason - description: "A curated set of reasons that explain why a flight is canceled" - type: EntityType - identify_by: [ code ] +- concept: NrFeet + description: "Unit of measure for distance in feet" + type: ValueType + extends: [ Decimal ] +- concept: NrPounds + description: "Unit of measure for weight in pounds" + type: ValueType + extends: [ Integer ] +- concept: NrMiles + description: "Unit of measure for distance in miles" + type: ValueType + extends: [ Decimal ] +- concept: NrMinutes + description: "Unit of measure for time in minutes" + type: ValueType + extends: [ Decimal ] +- concept: CancelationCode + description: "The single character code that identifies the reason a flight is canceled" + type: ValueType + extends: [ String ] + requires: [ CancelationCode == 'A' OR CancelationCode == 'B' OR CancelationCode == 'C' OR CancelationCode == 'D' ] +- concept: Capacity + description: "The capacity of an aircraft, measured in pounds." + type: ValueType + extends: [ NrPounds ] +- concept: DegreesLatitude + type: ValueType + extends: [ Decimal ] + requires: [ DegreesLatitude <= 90, DegreesLatitude >= -90 ] +- concept: DegreesLongitude + type: ValueType + extends: [ Decimal ] + requires: [ DegreesLongitude <= 180, DegreesLongitude >= -180 ] +- concept: Polygon + description: "A polygon represented as a list of vertices, where each vertex is a pair of latitude and longitude coordinates." + type: ValueType + extends: [ String ] +- concept: CityName + type: ValueType + extends: [ String ] +- concept: Delay + type: ValueType + extends: [ NrMinutes ] +- concept: Distance + type: ValueType + extends: [ NrMiles ] +- concept: DistanceGroup + description: "A number used to group distances of different lengths, where 1 groups the shortest and 10 the longest." + type: ValueType + extends: [ Integer ] + requires: [ 1 <= DistanceGroup, DistanceGroup <= 10 ] +- concept: RunwayGeometry + description: "A polygon that models the shape of a runway." + type: ValueType + extends: [ Polygon ] +- concept: RunwayLength + description: "The unit for measuring the lengths of runways in American airports." + type: ValueType + extends: [ NrFeet ] +- concept: RunwayDesignator + description: "Used to distinguish runways within a given airport." + type: ValueType + extends: [ String ] +- concept: SerialNr + type: ValueType + extends: [ String ] +- concept: StateCode + type: ValueType + extends: [ String ] +- concept: StateName + type: ValueType + extends: [ String ] +- concept: TailNr + type: ValueType + extends: [ String ] +- concept: Year + type: ValueType + extends: [ String ] +- concept: CancelationReason + description: "A curated set of reasons that explain why a flight is canceled" + type: EntityType + identify_by: [ code ] relationships: - name: code roles: @@ -137,10 +115,9 @@ ontology: - concept: String verbalizes: [ '{CancelationReason} has description- {String}' ] # The hyphen after "description" is significant when verbalizing constraints multiplicity: ManyToOne # Each CancelationReason has at most one description String -- concept: - name: State - type: EntityType - identify_by: [ code ] +- concept: State + type: EntityType + identify_by: [ code ] relationships: - name: code roles: @@ -154,10 +131,9 @@ ontology: verbalizes: - '{State} has {StateName}' multiplicity: ManyToOne -- concept: - name: City - type: EntityType - identify_by: [ name, state ] +- concept: City + type: EntityType + identify_by: [ name, state ] relationships: - name: name roles: @@ -171,10 +147,9 @@ ontology: verbalizes: - '{City} is located in {State}' multiplicity: ManyToOne -- concept: - name: Runway - type: EntityType - identify_by: [ designator, airport ] +- concept: Runway + type: EntityType + identify_by: [ designator, airport ] relationships: - name: airport roles: @@ -200,14 +175,12 @@ ontology: verbalizes: - '{Runway} has {RunwayGeometry}' multiplicity: ManyToOne -- concept: - name: ManufacturerName - type: ValueType - extends: [ String ] -- concept: - name: Manufacturer - type: EntityType - identify_by: [ name ] +- concept: ManufacturerName + type: ValueType + extends: [ String ] +- concept: Manufacturer + type: EntityType + identify_by: [ name ] relationships: - name: name roles: @@ -215,14 +188,12 @@ ontology: verbalizes: - '{Manufacturer} is identified by {ManufacturerName}' multiplicity: OneToOne -- concept: - name: ModelName - type: ValueType - extends: [ String ] -- concept: - name: Model - type: EntityType - identify_by: [ name, manufacturer ] +- concept: ModelName + type: ValueType + extends: [ String ] +- concept: Model + type: EntityType + identify_by: [ name, manufacturer ] relationships: - name: name roles: @@ -236,10 +207,9 @@ ontology: verbalizes: - '{Model} is manufactured by {Manufacturer}' multiplicity: ManyToOne -- concept: - name: Aircraft - type: EntityType - identify_by: [ tailnum ] +- concept: Aircraft + type: EntityType + identify_by: [ tailnum ] relationships: - name: serial_number roles: @@ -289,25 +259,21 @@ ontology: verbalizes: - '{Aircraft} is operated by {Carrier}' multiplicity: ManyToOne -- concept: - name: AirportName - type: ValueType - extends: [ String ] -- concept: - name: AirportCode - description: "The three-letter IATA code for the airport." - type: ValueType - extends: [ String ] -- concept: - name: AirportId - description: "Five digit number used as an alternate identifier for airports." - type: ValueType - extends: [ String ] -- concept: - name: Airport - type: EntityType - identify_by: [ code ] - requires: [ Airport.latitude, Airport.longitude ] +- concept: AirportName + type: ValueType + extends: [ String ] +- concept: AirportCode + description: "The three-letter IATA code for the airport." + type: ValueType + extends: [ String ] +- concept: AirportId + description: "Five digit number used as an alternate identifier for airports." + type: ValueType + extends: [ String ] +- concept: Airport + type: EntityType + identify_by: [ code ] + requires: [ Airport.latitude, Airport.longitude ] relationships: - name: city roles: @@ -359,33 +325,28 @@ ontology: verbalizes: - '{Airport} centers at {DegreesLatitude}' multiplicity: ManyToOne -- concept: - name: MarketName - type: ValueType - extends: [ String ] -- concept: - name: Market - type: EntityType - identify_by: [name] +- concept: MarketName + type: ValueType + extends: [ String ] +- concept: Market + type: EntityType + identify_by: [name] relationships: - name: name roles: [ { concept: MarketName }] verbalizes: [ '{Market} is identified by {MarketName}' ] multiplicity: OneToOne -- concept: - name: FlightNr - description: "The IATA flight number, which is typically a combination of the airline's IATA code and a numeric code (e.g., 'AA1234')." - type: ValueType - extends: [ String ] -- concept: - name: FlightId - description: "A unique identifier for an instance of a flight" - type: ValueType - extends: [ String ] -- concept: - name: Flight - type: EntityType - identify_by: [ id ] +- concept: FlightNr + description: "The IATA flight number, which is typically a combination of the airline's IATA code and a numeric code (e.g., 'AA1234')." + type: ValueType + extends: [ String ] +- concept: FlightId + description: "A unique identifier for an instance of a flight" + type: ValueType + extends: [ String ] +- concept: Flight + type: EntityType + identify_by: [ id ] relationships: - name: departure_delay roles: @@ -493,19 +454,16 @@ ontology: verbalizes: - '{Flight} is operated by {Carrier}' multiplicity: ManyToOne -- concept: - name: CarrierCode - description: "The two-letter IATA code for the airline carrier." - type: ValueType - extends: [ String ] -- concept: - name: CarrierName - type: ValueType - extends: [ String ] -- concept: - name: Carrier - type: EntityType - identify_by: [ code ] +- concept: CarrierCode + description: "The two-letter IATA code for the airline carrier." + type: ValueType + extends: [ String ] +- concept: CarrierName + type: ValueType + extends: [ String ] +- concept: Carrier + type: EntityType + identify_by: [ code ] relationships: - name: code roles: @@ -519,15 +477,13 @@ ontology: verbalizes: - '{Carrier} has {CarrierName}' multiplicity: ManyToOne -- concept: - name: RouteId - type: ValueType - description: "A unique identifier for a route between two airports. Constructed by concatenating the IATA codes of the departure and destination airports (e.g., 'ATL -> DCA')" - extends: [ String ] -- concept: - name: Route - type: EntityType - identify_by: [ id ] +- concept: RouteId + type: ValueType + description: "A unique identifier for a route between two airports. Constructed by concatenating the IATA codes of the departure and destination airports (e.g., 'ATL -> DCA')" + extends: [ String ] +- concept: Route + type: EntityType + identify_by: [ id ] relationships: - name: average_departure_delay roles: diff --git a/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml index df0a042b..7731a65f 100644 --- a/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml +++ b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml @@ -5,11 +5,10 @@ requires: - COUNT[Airport] > 0 - COUNT[Carrier] > 0 ontology: -- concept: - name: Route - type: EntityType - identify_by: - - id +- concept: Route + type: EntityType + identify_by: + - id relationships: - name: average_departure_delay roles: @@ -67,17 +66,15 @@ ontology: multiplicity: ManyToOne requires: - NOT Route.destination(Airport) -- concept: - name: RouteId - type: ValueType - description: A unique identifier for a route between two airports. Constructed by concatenating the IATA codes of the departure and destination airports (e.g., 'ATL -> DCA') - extends: - - String -- concept: - name: Carrier - type: EntityType - identify_by: - - code +- concept: RouteId + type: ValueType + description: A unique identifier for a route between two airports. Constructed by concatenating the IATA codes of the departure and destination airports (e.g., 'ATL -> DCA') + extends: + - String +- concept: Carrier + type: EntityType + identify_by: + - code relationships: - name: code roles: @@ -91,22 +88,19 @@ ontology: verbalizes: - '{Carrier} has {CarrierName}' multiplicity: ManyToOne -- concept: - name: CarrierName - type: ValueType - extends: - - String -- concept: - name: CarrierCode - type: ValueType - description: The two-letter IATA code for the airline carrier. - extends: - - String -- concept: - name: Flight - type: EntityType - identify_by: - - id +- concept: CarrierName + type: ValueType + extends: + - String +- concept: CarrierCode + type: ValueType + description: The two-letter IATA code for the airline carrier. + extends: + - String +- concept: Flight + type: EntityType + identify_by: + - id relationships: - name: departure_delay roles: @@ -218,23 +212,20 @@ ontology: verbalizes: - '{Flight} is operated by {Carrier}' multiplicity: ManyToOne -- concept: - name: FlightId - type: ValueType - description: A unique identifier for an instance of a flight - extends: - - String -- concept: - name: FlightNr - type: ValueType - description: The IATA flight number, which is typically a combination of the airline's IATA code and a numeric code (e.g., 'AA1234'). - extends: - - String -- concept: - name: Market - type: EntityType - identify_by: - - name +- concept: FlightId + type: ValueType + description: A unique identifier for an instance of a flight + extends: + - String +- concept: FlightNr + type: ValueType + description: The IATA flight number, which is typically a combination of the airline's IATA code and a numeric code (e.g., 'AA1234'). + extends: + - String +- concept: Market + type: EntityType + identify_by: + - name relationships: - name: name roles: @@ -242,19 +233,17 @@ ontology: verbalizes: - '{Market} is identified by {MarketName}' multiplicity: OneToOne -- concept: - name: MarketName - type: ValueType - extends: - - String -- concept: - name: Airport - type: EntityType - identify_by: - - code - requires: - - Airport.latitude - - Airport.longitude +- concept: MarketName + type: ValueType + extends: + - String +- concept: Airport + type: EntityType + identify_by: + - code + requires: + - Airport.latitude + - Airport.longitude relationships: - name: city roles: @@ -308,28 +297,24 @@ ontology: verbalizes: - '{Airport} centers at {DegreesLatitude}' multiplicity: ManyToOne -- concept: - name: AirportId - type: ValueType - description: Five digit number used as an alternate identifier for airports. - extends: - - String -- concept: - name: AirportCode - type: ValueType - description: The three-letter IATA code for the airport. - extends: - - String -- concept: - name: AirportName - type: ValueType - extends: - - String -- concept: - name: Aircraft - type: EntityType - identify_by: - - tailnum +- concept: AirportId + type: ValueType + description: Five digit number used as an alternate identifier for airports. + extends: + - String +- concept: AirportCode + type: ValueType + description: The three-letter IATA code for the airport. + extends: + - String +- concept: AirportName + type: ValueType + extends: + - String +- concept: Aircraft + type: EntityType + identify_by: + - tailnum relationships: - name: serial_number roles: @@ -379,12 +364,11 @@ ontology: verbalizes: - '{Aircraft} is operated by {Carrier}' multiplicity: ManyToOne -- concept: - name: Model - type: EntityType - identify_by: - - name - - manufacturer +- concept: Model + type: EntityType + identify_by: + - name + - manufacturer relationships: - name: name roles: @@ -398,16 +382,14 @@ ontology: verbalizes: - '{Model} is manufactured by {Manufacturer}' multiplicity: ManyToOne -- concept: - name: ModelName - type: ValueType - extends: - - String -- concept: - name: Manufacturer - type: EntityType - identify_by: - - name +- concept: ModelName + type: ValueType + extends: + - String +- concept: Manufacturer + type: EntityType + identify_by: + - name relationships: - name: name roles: @@ -415,17 +397,15 @@ ontology: verbalizes: - '{Manufacturer} is identified by {ManufacturerName}' multiplicity: OneToOne -- concept: - name: ManufacturerName - type: ValueType - extends: - - String -- concept: - name: Runway - type: EntityType - identify_by: - - designator - - airport +- concept: ManufacturerName + type: ValueType + extends: + - String +- concept: Runway + type: EntityType + identify_by: + - designator + - airport relationships: - name: airport roles: @@ -451,12 +431,11 @@ ontology: verbalizes: - '{Runway} has {RunwayGeometry}' multiplicity: ManyToOne -- concept: - name: City - type: EntityType - identify_by: - - name - - state +- concept: City + type: EntityType + identify_by: + - name + - state relationships: - name: name roles: @@ -470,11 +449,10 @@ ontology: verbalizes: - '{City} is located in {State}' multiplicity: ManyToOne -- concept: - name: State - type: EntityType - identify_by: - - code +- concept: State + type: EntityType + identify_by: + - code relationships: - name: code roles: @@ -488,12 +466,11 @@ ontology: verbalizes: - '{State} has {StateName}' multiplicity: ManyToOne -- concept: - name: CancelationReason - type: EntityType - description: A curated set of reasons that explain why a flight is canceled - identify_by: - - code +- concept: CancelationReason + type: EntityType + description: A curated set of reasons that explain why a flight is canceled + identify_by: + - code relationships: - name: code roles: @@ -507,133 +484,112 @@ ontology: verbalizes: - '{CancelationReason} has description- {String}' multiplicity: ManyToOne -- concept: - name: Year - type: ValueType - extends: - - String -- concept: - name: TailNr - type: ValueType - extends: - - String -- concept: - name: StateName - type: ValueType - extends: - - String -- concept: - name: StateCode - type: ValueType - extends: - - String -- concept: - name: SerialNr - type: ValueType - extends: - - String -- concept: - name: RunwayDesignator - type: ValueType - description: Used to distinguish runways within a given airport. - extends: - - String -- concept: - name: DistanceGroup - type: ValueType - description: A number used to group distances of different lengths, where 1 groups the shortest and 10 the longest. - extends: - - Integer - requires: - - 1 <= DistanceGroup - - DistanceGroup <= 10 -- concept: - name: CityName - type: ValueType - extends: - - String -- concept: - name: Polygon - type: ValueType - description: A polygon represented as a list of vertices, where each vertex is a pair of latitude and longitude coordinates. - extends: - - String -- concept: - name: RunwayGeometry - type: ValueType - description: A polygon that models the shape of a runway. - extends: - - Polygon -- concept: - name: DegreesLongitude - type: ValueType - extends: - - Decimal - requires: - - DegreesLongitude <= 180 - - DegreesLongitude >= -180 -- concept: - name: DegreesLatitude - type: ValueType - extends: - - Decimal - requires: - - DegreesLatitude <= 90 - - DegreesLatitude >= -90 -- concept: - name: CancelationCode - type: ValueType - description: The single character code that identifies the reason a flight is canceled - extends: - - String - requires: - - CancelationCode == 'A' OR CancelationCode == 'B' OR CancelationCode == 'C' OR CancelationCode == 'D' -- concept: - name: NrMinutes - type: ValueType - description: Unit of measure for time in minutes - extends: - - Decimal -- concept: - name: Delay - type: ValueType - extends: - - NrMinutes -- concept: - name: NrMiles - type: ValueType - description: Unit of measure for distance in miles - extends: - - Decimal -- concept: - name: Distance - type: ValueType - extends: - - NrMiles -- concept: - name: NrPounds - type: ValueType - description: Unit of measure for weight in pounds - extends: - - Integer -- concept: - name: Capacity - type: ValueType - description: The capacity of an aircraft, measured in pounds. - extends: - - NrPounds -- concept: - name: NrFeet - type: ValueType - description: Unit of measure for distance in feet - extends: - - Decimal -- concept: - name: RunwayLength - type: ValueType - description: The unit for measuring the lengths of runways in American airports. - extends: - - NrFeet +- concept: Year + type: ValueType + extends: + - String +- concept: TailNr + type: ValueType + extends: + - String +- concept: StateName + type: ValueType + extends: + - String +- concept: StateCode + type: ValueType + extends: + - String +- concept: SerialNr + type: ValueType + extends: + - String +- concept: RunwayDesignator + type: ValueType + description: Used to distinguish runways within a given airport. + extends: + - String +- concept: DistanceGroup + type: ValueType + description: A number used to group distances of different lengths, where 1 groups the shortest and 10 the longest. + extends: + - Integer + requires: + - 1 <= DistanceGroup + - DistanceGroup <= 10 +- concept: CityName + type: ValueType + extends: + - String +- concept: Polygon + type: ValueType + description: A polygon represented as a list of vertices, where each vertex is a pair of latitude and longitude coordinates. + extends: + - String +- concept: RunwayGeometry + type: ValueType + description: A polygon that models the shape of a runway. + extends: + - Polygon +- concept: DegreesLongitude + type: ValueType + extends: + - Decimal + requires: + - DegreesLongitude <= 180 + - DegreesLongitude >= -180 +- concept: DegreesLatitude + type: ValueType + extends: + - Decimal + requires: + - DegreesLatitude <= 90 + - DegreesLatitude >= -90 +- concept: CancelationCode + type: ValueType + description: The single character code that identifies the reason a flight is canceled + extends: + - String + requires: + - CancelationCode == 'A' OR CancelationCode == 'B' OR CancelationCode == 'C' OR CancelationCode == 'D' +- concept: NrMinutes + type: ValueType + description: Unit of measure for time in minutes + extends: + - Decimal +- concept: Delay + type: ValueType + extends: + - NrMinutes +- concept: NrMiles + type: ValueType + description: Unit of measure for distance in miles + extends: + - Decimal +- concept: Distance + type: ValueType + extends: + - NrMiles +- concept: NrPounds + type: ValueType + description: Unit of measure for weight in pounds + extends: + - Integer +- concept: Capacity + type: ValueType + description: The capacity of an aircraft, measured in pounds. + extends: + - NrPounds +- concept: NrFeet + type: ValueType + description: Unit of measure for distance in feet + extends: + - Decimal +- concept: RunwayLength + type: ValueType + description: The unit for measuring the lengths of runways in American airports. + extends: + - NrFeet ontology_mappings: - name: flights_mapping semantic_model: From a3639df5404598e7aadc561f649f0acb383b987b Mon Sep 17 00:00:00 2001 From: vmihalovski Date: Thu, 30 Jul 2026 15:24:08 +0200 Subject: [PATCH 29/29] Addressed review comments --- .github/workflows/converter-ontology-ci.yml | 60 ++++++++ converters/ontology/.gitignore | 17 +++ converters/ontology/README.md | 59 +++++--- converters/ontology/pyproject.toml | 21 ++- converters/ontology/requirements.lock | 4 +- .../ontology/scripts/palantir_to_osi.py | 52 ------- .../ontology/scripts/palantir_to_ossie.py | 69 +++++++++ converters/ontology/src/osi/__init__.py | 74 ---------- .../ontology/src/osi/common/__init__.py | 0 .../ontology/src/osi/converter/__init__.py | 0 .../src/osi/converter/osi_to_spec/__init__.py | 0 .../osi/converter/palantir_to_osi/__init__.py | 0 .../src/osi/converter/spec_to_osi/__init__.py | 0 .../ontology/src/osi/external/__init__.py | 0 .../src/osi/external/palantir/__init__.py | 0 .../ontology/src/osi/parser/__init__.py | 53 ------- .../ontology/src/ossie_ontology/__init__.py | 91 ++++++++++++ .../src/ossie_ontology/common/__init__.py | 16 +++ .../common/file_utils.py | 17 +++ .../{osi => ossie_ontology}/common/graph.py | 17 +++ .../{osi => ossie_ontology}/common/utils.py | 17 +++ .../src/ossie_ontology/converter/__init__.py | 16 +++ .../converter/ossie_to_spec/__init__.py | 16 +++ .../converter/ossie_to_spec}/converter.py | 39 ++++-- .../converter/palantir_to_ossie/__init__.py | 16 +++ .../converter/palantir_to_ossie}/converter.py | 131 ++++++++++-------- .../converter/spec_to_ossie/__init__.py | 16 +++ .../converter/spec_to_ossie}/converter.py | 55 +++++--- .../src/ossie_ontology/external/__init__.py | 16 +++ .../external/palantir/__init__.py | 16 +++ .../external/palantir/model.py | 17 +++ .../external/palantir/parser/__init__.py | 23 ++- .../src/{osi => ossie_ontology}/model.py | 27 +++- .../src/ossie_ontology/parser/__init__.py | 70 ++++++++++ .../src/{osi => ossie_ontology}/spec.py | 63 ++++++--- converters/ontology/tests/__init__.py | 16 +++ converters/ontology/tests/conftest.py | 78 ++++++++++- .../flights_roundtrip.yaml | 17 +++ .../flights_structure.txt | 17 +++ .../ontology/tests/test_examples_in_sync.py | 17 +++ .../ontology/tests/test_flights_snapshot.py | 25 +++- ...est_osi_parser.py => test_ossie_parser.py} | 51 ++++--- .../ontology/tests/test_palantir_parser.py | 21 ++- .../tests/test_verbalization_parsing.py | 19 ++- 44 files changed, 1000 insertions(+), 349 deletions(-) create mode 100644 .github/workflows/converter-ontology-ci.yml delete mode 100644 converters/ontology/scripts/palantir_to_osi.py create mode 100644 converters/ontology/scripts/palantir_to_ossie.py delete mode 100644 converters/ontology/src/osi/__init__.py delete mode 100644 converters/ontology/src/osi/common/__init__.py delete mode 100644 converters/ontology/src/osi/converter/__init__.py delete mode 100644 converters/ontology/src/osi/converter/osi_to_spec/__init__.py delete mode 100644 converters/ontology/src/osi/converter/palantir_to_osi/__init__.py delete mode 100644 converters/ontology/src/osi/converter/spec_to_osi/__init__.py delete mode 100644 converters/ontology/src/osi/external/__init__.py delete mode 100644 converters/ontology/src/osi/external/palantir/__init__.py delete mode 100644 converters/ontology/src/osi/parser/__init__.py create mode 100644 converters/ontology/src/ossie_ontology/__init__.py create mode 100644 converters/ontology/src/ossie_ontology/common/__init__.py rename converters/ontology/src/{osi => ossie_ontology}/common/file_utils.py (82%) rename converters/ontology/src/{osi => ossie_ontology}/common/graph.py (87%) rename converters/ontology/src/{osi => ossie_ontology}/common/utils.py (67%) create mode 100644 converters/ontology/src/ossie_ontology/converter/__init__.py create mode 100644 converters/ontology/src/ossie_ontology/converter/ossie_to_spec/__init__.py rename converters/ontology/src/{osi/converter/osi_to_spec => ossie_ontology/converter/ossie_to_spec}/converter.py (88%) create mode 100644 converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/__init__.py rename converters/ontology/src/{osi/converter/palantir_to_osi => ossie_ontology/converter/palantir_to_ossie}/converter.py (85%) create mode 100644 converters/ontology/src/ossie_ontology/converter/spec_to_ossie/__init__.py rename converters/ontology/src/{osi/converter/spec_to_osi => ossie_ontology/converter/spec_to_ossie}/converter.py (92%) create mode 100644 converters/ontology/src/ossie_ontology/external/__init__.py create mode 100644 converters/ontology/src/ossie_ontology/external/palantir/__init__.py rename converters/ontology/src/{osi => ossie_ontology}/external/palantir/model.py (96%) rename converters/ontology/src/{osi => ossie_ontology}/external/palantir/parser/__init__.py (96%) rename converters/ontology/src/{osi => ossie_ontology}/model.py (97%) create mode 100644 converters/ontology/src/ossie_ontology/parser/__init__.py rename converters/ontology/src/{osi => ossie_ontology}/spec.py (84%) rename converters/ontology/tests/{test_osi_parser.py => test_ossie_parser.py} (76%) diff --git a/.github/workflows/converter-ontology-ci.yml b/.github/workflows/converter-ontology-ci.yml new file mode 100644 index 00000000..ad9ec3da --- /dev/null +++ b/.github/workflows/converter-ontology-ci.yml @@ -0,0 +1,60 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Ontology CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/ontology/**' + - '.github/workflows/converter-ontology-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/ontology/**' + - '.github/workflows/converter-ontology-ci.yml' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + working-directory: converters/ontology + run: | + python -m pip install --upgrade pip + pip install -r requirements.lock + pip install -e ".[dev]" + + - name: Unit Tests + working-directory: converters/ontology + run: | + pytest \ No newline at end of file diff --git a/converters/ontology/.gitignore b/converters/ontology/.gitignore index 211fd5c5..553c47ac 100644 --- a/converters/ontology/.gitignore +++ b/converters/ontology/.gitignore @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + # Python __pycache__/ *.py[cod] diff --git a/converters/ontology/README.md b/converters/ontology/README.md index 94dfbf4c..1e7077ee 100644 --- a/converters/ontology/README.md +++ b/converters/ontology/README.md @@ -1,12 +1,31 @@ -# OSI Ontology Converters - -Converters between OSI, Palantir, and Spec ontology formats. - -| Converter | Direction | -|-----------|-----------| -| `palantir_to_osi` | Palantir ontology → OSI model | -| `osi_to_spec` | OSI model → Spec YAML | -| `spec_to_osi` | Spec YAML → OSI model | + + +# Ossie Ontology Converters + +Converters between Ossie, Palantir, and Spec ontology formats. + +| Converter | Direction | +|---------------------|-----------| +| `palantir_to_ossie` | Palantir ontology → Ossie model | +| `ossie_to_spec` | Ossie model → Spec YAML | +| `spec_to_ossie` | Spec YAML → Ossie model | ## Prerequisites @@ -50,29 +69,29 @@ pip-compile --output-file requirements.lock pyproject.toml ## Usage -The package is importable as `osi` after installation: +The package is importable as `ossie_ontology` after installation: ```python -from osi.converter.palantir_to_osi.converter import PalantirToOsiConverter -from osi.converter.osi_to_spec.converter import OsiToSpecConverter -from osi.converter.spec_to_osi.converter import SpecToOsiConverter +from ossie_ontology.converter.palantir_to_ossie.converter import PalantirToOssieConverter +from ossie_ontology.converter.ossie_to_spec.converter import OssieToSpecConverter +from ossie_ontology.converter.spec_to_ossie.converter import SpecToOssieConverter ``` ## Scripts -### `scripts/palantir_to_osi.py` +### `scripts/palantir_to_ossie.py` -Converts a Palantir ontology export (a `.zip` archive or an already extracted folder containing a Palantir ontology JSON and a `data_sets` folder of one or more dataset spec JSON files) into an OSI-compliant YAML representation, printed to stdout. +Converts a Palantir ontology export (a `.zip` archive or an already extracted folder containing a Palantir ontology JSON and a `data_sets` folder of one or more dataset spec JSON files) into an Ossie-compliant YAML representation, printed to stdout. **Usage:** ```bash -python scripts/palantir_to_osi.py path/to/palantir_export.zip +python scripts/palantir_to_ossie.py path/to/palantir_export.zip # or an extracted folder: -python scripts/palantir_to_osi.py path/to/palantir_export/ +python scripts/palantir_to_ossie.py path/to/palantir_export/ ``` -Warnings are written to stderr; the OSI YAML is written to stdout. +Warnings are written to stderr; the Ossie YAML is written to stdout. **Environment variables (optional):** @@ -85,11 +104,11 @@ If already set in your environment they will be picked up automatically. To over ```bash SNOWFLAKE_DATABASE_NAME=MY_DB SNOWFLAKE_SCHEMA_NAME=MY_SCHEMA \ - python scripts/palantir_to_osi.py path/to/palantir_export.zip + python scripts/palantir_to_ossie.py path/to/palantir_export.zip ``` ## Deactivating the environment ```bash deactivate -``` \ No newline at end of file +``` diff --git a/converters/ontology/pyproject.toml b/converters/ontology/pyproject.toml index 458ed8d8..201705d2 100644 --- a/converters/ontology/pyproject.toml +++ b/converters/ontology/pyproject.toml @@ -1,7 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + [project] -name = "ontology" +name = "apache-ossie-ontology" version = "0.1.0" -description = "OSI ontology converters — Palantir → OSI, OSI → Spec, Spec → OSI" +description = "Ossie ontology converters — Palantir → Ossie, Ossie → Spec, Spec → Ossie" readme = "README.md" authors = [ { name = "RelationalAI", email = "support@relational.ai" }, diff --git a/converters/ontology/requirements.lock b/converters/ontology/requirements.lock index 2b69d8bf..bc25bab0 100644 --- a/converters/ontology/requirements.lock +++ b/converters/ontology/requirements.lock @@ -7,11 +7,11 @@ annotated-types==0.7.0 # via pydantic pydantic==2.13.4 - # via ontology (pyproject.toml) + # via apache-ossie-ontology (pyproject.toml) pydantic-core==2.46.4 # via pydantic pyyaml==6.0.3 - # via ontology (pyproject.toml) + # via apache-ossie-ontology (pyproject.toml) typing-extensions==4.15.0 # via # pydantic diff --git a/converters/ontology/scripts/palantir_to_osi.py b/converters/ontology/scripts/palantir_to_osi.py deleted file mode 100644 index 1dcea201..00000000 --- a/converters/ontology/scripts/palantir_to_osi.py +++ /dev/null @@ -1,52 +0,0 @@ -# Description: -# -# This script converts a Palantir ontology export into an OSI compliant YAML -# representation of that ontology, using environment variables to configure the -# Snowflake database and schema names. The export may be supplied either as a -# zip archive or as an already extracted folder, and must contain: -# 1. A Palantir ontology (JSON file) and -# 2. A 'data_sets' folder containing one or more Palantir dataset specs (JSON files) -# -# Usage: -# -# $ python palantir_to_osi.py -# -# Environment variables used: -# -# - SNOWFLAKE_DATABASE_NAME -# - SNOWFLAKE_SCHEMA_NAME -# -# The tables that populate the ontology are named -# "{SNOWFLAKE_DATABASE_NAME}.{SNOWFLAKE_SCHEMA_NAME}.{TABLE_NAME}" -# where TABLE_NAME is the name of a data set that is referenced in -# the Palantir ontology. -# -# Outputs: -# -# - stderr: Warnings -# -import os -import sys -from pathlib import Path - -from osi.converter.palantir_to_osi.converter import PalantirToOsiConverter -from osi.converter.osi_to_spec.converter import OsiToSpecConverter - -from osi.external.palantir.parser import PalantirParser - -if __name__ == "__main__": - db_name = os.environ.get("SNOWFLAKE_DATABASE_NAME", "PALANTIR") - schema_name = os.environ.get("SNOWFLAKE_SCHEMA_NAME", "PALANTIR") - - if len(sys.argv) != 2: - sys.exit(f"Usage: {sys.argv[0]} ") - - path = Path(sys.argv[1]) - - parser = PalantirParser() - palantir_model = parser.parse(path) - - ontology_model = PalantirToOsiConverter().convert(palantir_model, db_name, schema_name) - - osi_spec = OsiToSpecConverter.convert(ontology_model) - print(osi_spec.dump_yaml()) diff --git a/converters/ontology/scripts/palantir_to_ossie.py b/converters/ontology/scripts/palantir_to_ossie.py new file mode 100644 index 00000000..e1166c66 --- /dev/null +++ b/converters/ontology/scripts/palantir_to_ossie.py @@ -0,0 +1,69 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Description: +# +# This script converts a Palantir ontology export into an Ossie compliant YAML +# representation of that ontology, using environment variables to configure the +# Snowflake database and schema names. The export may be supplied either as a +# zip archive or as an already extracted folder, and must contain: +# 1. A Palantir ontology (JSON file) and +# 2. A 'data_sets' folder containing one or more Palantir dataset specs (JSON files) +# +# Usage: +# +# $ python palantir_to_ossie.py +# +# Environment variables used: +# +# - SNOWFLAKE_DATABASE_NAME +# - SNOWFLAKE_SCHEMA_NAME +# +# The tables that populate the ontology are named +# "{SNOWFLAKE_DATABASE_NAME}.{SNOWFLAKE_SCHEMA_NAME}.{TABLE_NAME}" +# where TABLE_NAME is the name of a data set that is referenced in +# the Palantir ontology. +# +# Outputs: +# +# - stderr: Warnings +# +import os +import sys +from pathlib import Path + +from ossie_ontology.converter.palantir_to_ossie.converter import PalantirToOssieConverter +from ossie_ontology.converter.ossie_to_spec.converter import OssieToSpecConverter + +from ossie_ontology.external.palantir.parser import PalantirParser + +if __name__ == "__main__": + db_name = os.environ.get("SNOWFLAKE_DATABASE_NAME", "PALANTIR") + schema_name = os.environ.get("SNOWFLAKE_SCHEMA_NAME", "PALANTIR") + + if len(sys.argv) != 2: + sys.exit(f"Usage: {sys.argv[0]} ") + + path = Path(sys.argv[1]) + + parser = PalantirParser() + palantir_model = parser.parse(path) + + ontology_model = PalantirToOssieConverter().convert(palantir_model, db_name, schema_name) + + ossie_spec = OssieToSpecConverter.convert(ontology_model) + print(ossie_spec.dump_yaml()) diff --git a/converters/ontology/src/osi/__init__.py b/converters/ontology/src/osi/__init__.py deleted file mode 100644 index e81bd319..00000000 --- a/converters/ontology/src/osi/__init__.py +++ /dev/null @@ -1,74 +0,0 @@ -""" -Public API surface for osi. - -Consumers should import from here rather than from deep sub-paths. -""" - -from osi.model import ( - Concept, - ConceptMapping, - ConceptType, - CustomExtension, - Dataset, - DatasetField, - DialectExpression, - DialectExpressionSet, - Formula, - FormulaFactory, - JoinPath, - LinkMapping, - Metric, - ObjectMapping, - OntologyComponent, - OntologyMapping, - OsiOntology, - ReferentMapping, - Relationship, - RelationshipMultiplicity, - Role, - SemanticModel, -) -from osi.spec import OsiSpec -from osi.parser import OsiParser -from osi.external.palantir.parser import PalantirParser -from osi.converter.spec_to_osi.converter import SpecToOsiConverter -from osi.converter.osi_to_spec.converter import OsiToSpecConverter -from osi.converter.palantir_to_osi.converter import PalantirToOsiConverter - -__all__ = [ - # Model — ontology layer - "Concept", - "ConceptType", - "Relationship", - "RelationshipMultiplicity", - "Role", - "Formula", - # Model — semantic layer - "Dataset", - "DatasetField", - "DialectExpression", - "DialectExpressionSet", - "JoinPath", - "Metric", - "SemanticModel", - # Model — mapping layer - "ObjectMapping", - "ReferentMapping", - "LinkMapping", - "ConceptMapping", - "OntologyMapping", - "OntologyComponent", - "OsiOntology", - # Supporting types - "CustomExtension", - "FormulaFactory", - # Spec DTO - "OsiSpec", - # Parsers - "OsiParser", - "PalantirParser", - # Converters - "SpecToOsiConverter", - "OsiToSpecConverter", - "PalantirToOsiConverter", -] \ No newline at end of file diff --git a/converters/ontology/src/osi/common/__init__.py b/converters/ontology/src/osi/common/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/converters/ontology/src/osi/converter/__init__.py b/converters/ontology/src/osi/converter/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/converters/ontology/src/osi/converter/osi_to_spec/__init__.py b/converters/ontology/src/osi/converter/osi_to_spec/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/__init__.py b/converters/ontology/src/osi/converter/palantir_to_osi/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/converters/ontology/src/osi/converter/spec_to_osi/__init__.py b/converters/ontology/src/osi/converter/spec_to_osi/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/converters/ontology/src/osi/external/__init__.py b/converters/ontology/src/osi/external/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/converters/ontology/src/osi/external/palantir/__init__.py b/converters/ontology/src/osi/external/palantir/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/converters/ontology/src/osi/parser/__init__.py b/converters/ontology/src/osi/parser/__init__.py deleted file mode 100644 index 0b6e0d87..00000000 --- a/converters/ontology/src/osi/parser/__init__.py +++ /dev/null @@ -1,53 +0,0 @@ -"""Entrypoint: read a YAML/JSON OSI spec and produce an OsiOntology.""" - -from __future__ import annotations - -import json -from pathlib import Path -from typing import Any - -import yaml - -from osi.converter.spec_to_osi.converter import SpecToOsiConverter -from osi.model import OsiOntology, FormulaFactory, MappingFormulaFactory -from osi.spec import OsiSpec - - -class OsiParser: - _model: OsiOntology | None - _spec: OsiSpec | None - _debug: bool - - def __init__(self, debug: bool = False, - formula_factory: FormulaFactory | None = None, - mapping_formula_factory: MappingFormulaFactory | None = None): - self._debug = debug - self._model = None - self._spec = None - self._formula_factory = formula_factory or FormulaFactory() - self._mapping_formula_factory = mapping_formula_factory or MappingFormulaFactory() - - def parse(self, path: Path) -> OsiOntology: - # OSI always expects a single spec file. - if not path.is_file(): - raise ValueError(f"Expected a single OSI spec file, but '{path}' is not a file") - raw = OsiParser.load_data(path) - self._spec = OsiSpec.model_validate(raw) - self._model = SpecToOsiConverter( - formula_factory=self._formula_factory, mapping_formula_factory=self._mapping_formula_factory - ).convert(self._spec) - return self._model - - @staticmethod - def load_data(path: Path) -> Any: - # Pin UTF-8 so parsing is reproducible regardless of the process locale. - content = path.read_text(encoding="utf-8") - if path.suffix.lower() == ".json": - return json.loads(content) - return yaml.safe_load(content) - - def spec(self) -> OsiSpec: - spec = self._spec - if spec is None: - raise RuntimeError("You must call 'parse()' before accessing 'spec()'") - return spec diff --git a/converters/ontology/src/ossie_ontology/__init__.py b/converters/ontology/src/ossie_ontology/__init__.py new file mode 100644 index 00000000..9579ef3b --- /dev/null +++ b/converters/ontology/src/ossie_ontology/__init__.py @@ -0,0 +1,91 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" +Public API surface for ossie. + +Consumers should import from here rather than from deep sub-paths. +""" + +from ossie_ontology.model import ( + Concept, + ConceptMapping, + ConceptType, + CustomExtension, + Dataset, + DatasetField, + DialectExpression, + DialectExpressionSet, + Formula, + FormulaFactory, + JoinPath, + LinkMapping, + Metric, + ObjectMapping, + OntologyComponent, + OntologyMapping, + OssieOntology, + ReferentMapping, + Relationship, + RelationshipMultiplicity, + Role, + SemanticModel, +) +from ossie_ontology.spec import OssieSpec +from ossie_ontology.parser import OssieParser +from ossie_ontology.external.palantir.parser import PalantirParser +from ossie_ontology.converter.spec_to_ossie.converter import SpecToOssieConverter +from ossie_ontology.converter.ossie_to_spec.converter import OssieToSpecConverter +from ossie_ontology.converter.palantir_to_ossie.converter import PalantirToOssieConverter + +__all__ = [ + # Model — ontology layer + "Concept", + "ConceptType", + "Relationship", + "RelationshipMultiplicity", + "Role", + "Formula", + # Model — semantic layer + "Dataset", + "DatasetField", + "DialectExpression", + "DialectExpressionSet", + "JoinPath", + "Metric", + "SemanticModel", + # Model — mapping layer + "ObjectMapping", + "ReferentMapping", + "LinkMapping", + "ConceptMapping", + "OntologyMapping", + "OntologyComponent", + "OssieOntology", + # Supporting types + "CustomExtension", + "FormulaFactory", + # Spec DTO + "OssieSpec", + # Parsers + "OssieParser", + "PalantirParser", + # Converters + "SpecToOssieConverter", + "OssieToSpecConverter", + "PalantirToOssieConverter", +] diff --git a/converters/ontology/src/ossie_ontology/common/__init__.py b/converters/ontology/src/ossie_ontology/common/__init__.py new file mode 100644 index 00000000..13a83393 --- /dev/null +++ b/converters/ontology/src/ossie_ontology/common/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/ontology/src/osi/common/file_utils.py b/converters/ontology/src/ossie_ontology/common/file_utils.py similarity index 82% rename from converters/ontology/src/osi/common/file_utils.py rename to converters/ontology/src/ossie_ontology/common/file_utils.py index 3fbf2fe1..89674119 100644 --- a/converters/ontology/src/osi/common/file_utils.py +++ b/converters/ontology/src/ossie_ontology/common/file_utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import io import zipfile from pathlib import Path diff --git a/converters/ontology/src/osi/common/graph.py b/converters/ontology/src/ossie_ontology/common/graph.py similarity index 87% rename from converters/ontology/src/osi/common/graph.py rename to converters/ontology/src/ossie_ontology/common/graph.py index aa09f61a..d08e0fd3 100644 --- a/converters/ontology/src/osi/common/graph.py +++ b/converters/ontology/src/ossie_ontology/common/graph.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import annotations import warnings diff --git a/converters/ontology/src/osi/common/utils.py b/converters/ontology/src/ossie_ontology/common/utils.py similarity index 67% rename from converters/ontology/src/osi/common/utils.py rename to converters/ontology/src/ossie_ontology/common/utils.py index e2fa8bdd..d1dc55bb 100644 --- a/converters/ontology/src/osi/common/utils.py +++ b/converters/ontology/src/ossie_ontology/common/utils.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import logging import re from keyword import iskeyword diff --git a/converters/ontology/src/ossie_ontology/converter/__init__.py b/converters/ontology/src/ossie_ontology/converter/__init__.py new file mode 100644 index 00000000..13a83393 --- /dev/null +++ b/converters/ontology/src/ossie_ontology/converter/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/__init__.py b/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/__init__.py new file mode 100644 index 00000000..13a83393 --- /dev/null +++ b/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/ontology/src/osi/converter/osi_to_spec/converter.py b/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py similarity index 88% rename from converters/ontology/src/osi/converter/osi_to_spec/converter.py rename to converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py index f6c5be5d..d0b65437 100644 --- a/converters/ontology/src/osi/converter/osi_to_spec/converter.py +++ b/converters/ontology/src/ossie_ontology/converter/ossie_to_spec/converter.py @@ -1,11 +1,28 @@ -"""Reverse converter: OsiOntology (runtime) -> OsiSpec (Pydantic DTO). - -Pairs with spec_to_osi.SpecToOsiConverter so a full round-trip +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Reverse converter: OssieOntology (runtime) -> OssieSpec (Pydantic DTO). + +Pairs with spec_to_ossie.SpecToOssieConverter so a full round-trip yaml -> spec -> model -> spec -> yaml is structurally stable.""" from __future__ import annotations -from osi.model import ( +from ossie_ontology.model import ( Concept, ConceptMapping, ConceptType, @@ -23,9 +40,9 @@ OntologyMapping, ReferentMapping, Relationship, - OsiOntology, + OssieOntology, ) -from osi.spec import ( +from ossie_ontology.spec import ( ConceptComponent, ConceptMapping as SpecConceptMapping, CustomExtension as SpecCustomExtension, @@ -40,21 +57,21 @@ Metric as SpecMetric, ObjectMapping as SpecObjectMapping, OntologyMapping as SpecOntologyMapping, - OsiSpec, + OssieSpec, ReferentMapping as SpecReferentMapping, Relationship as SpecRelationship, Role as SpecRole, ) -class OsiToSpecConverter: +class OssieToSpecConverter: """Top-level reverse converter.""" @staticmethod - def convert(model: OsiOntology) -> OsiSpec: + def convert(model: OssieOntology) -> OssieSpec: ont = model.ontology ontology_mappings = [_convert_ontology_mapping(ontology_mapping) for ontology_mapping in model.ontology_mappings] - return OsiSpec( + return OssieSpec( version=model.version, name=model.name, description=model.description, @@ -244,7 +261,7 @@ def _render_mapping_expression(expr) -> str | None: string the forward converter saw in the spec.""" if expr is None: return None - from osi.model import DatasetField as _DF, Formula as _F + from ossie_ontology.model import DatasetField as _DF, Formula as _F if isinstance(expr, _DF): ds = expr.dataset return f"{ds.name}.{expr.name}" if ds is not None else expr.name diff --git a/converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/__init__.py b/converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/__init__.py new file mode 100644 index 00000000..13a83393 --- /dev/null +++ b/converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py b/converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/converter.py similarity index 85% rename from converters/ontology/src/osi/converter/palantir_to_osi/converter.py rename to converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/converter.py index 85bd8a80..1c504433 100644 --- a/converters/ontology/src/osi/converter/palantir_to_osi/converter.py +++ b/converters/ontology/src/ossie_ontology/converter/palantir_to_ossie/converter.py @@ -1,12 +1,29 @@ -"""Palantir `Ontology` -> `OsiOntology`.""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Palantir `Ontology` -> `OssieOntology`.""" from __future__ import annotations import warnings -from osi.common.graph import topological_sort_break_cycles -from osi.common.utils import to_pascal_case, to_verbalization_string -from osi.external.palantir.model import ( +from ossie_ontology.common.graph import topological_sort_break_cycles +from ossie_ontology.common.utils import to_pascal_case, to_verbalization_string +from ossie_ontology.external.palantir.model import ( ArrayDataType, DataSet as PalantirDataSet, DataSetColumn, @@ -19,7 +36,7 @@ Property as PalantirProperty, Relation, ) -from osi.model import ( +from ossie_ontology.model import ( Concept, ConceptMapping, ConceptType, @@ -36,22 +53,22 @@ ReferentMapping, Relationship, RelationshipMultiplicity, - OsiOntology + OssieOntology ) _DEFAULT_DIALECT = "ANSI_SQL" -class PalantirToOsiConverter: - """Converts a Palantir Ontology to OsiOntology. +class PalantirToOssieConverter: + """Converts a Palantir Ontology to OssieOntology. Pass a *formula_factory* to control how Formula objects are created. The default produces plain ``Formula`` instances; downstream packages can inject a factory that returns enriched subclasses (e.g. with an AST). - model = PalantirToOsiConverter().convert(palantir_ontology) - model = PalantirToOsiConverter(formula_factory=my_parser).convert(palantir_ontology) + model = PalantirToOssieConverter().convert(palantir_ontology) + model = PalantirToOssieConverter(formula_factory=my_parser).convert(palantir_ontology) """ depths_role_names = {1: "fst", 2: "snd", 3: "thd", 4: "frt"} @@ -68,9 +85,9 @@ def convert( palantir_ontology: PalantirOntology, db_name: str = "palantir", schema_name: str = "palantir", - ) -> OsiOntology: + ) -> OssieOntology: ontology = OntologyComponent() - model = OsiOntology(name="Palantir model", ontology=ontology, version="0.1.0") + model = OssieOntology(name="Palantir model", ontology=ontology, version="0.1.0") semantic_model = SemanticModel(name="Palantir semantic model") @@ -145,7 +162,7 @@ def _convert_object_type( db_name: str, schema_name: str, ) -> None: - concept_name = PalantirToOsiConverter._concept_name(ot) + concept_name = PalantirToOssieConverter._concept_name(ot) relevant_props = [ p for p in ot.properties().values() if p.active() or p.experimental() or p.intermediary() ] @@ -160,7 +177,7 @@ def _convert_object_type( if is_subtype and not ignore_subtype: parent_ot = subtype_relation.many_object_type() # type: ignore[union-attr] - parent_name = PalantirToOsiConverter._concept_name(parent_ot) + parent_name = PalantirToOssieConverter._concept_name(parent_ot) parent = ontology.lookup_concept(parent_name) assert parent is not None, f"Parent concept '{parent_name}' not found (expected from topological order)" concept = Concept(name=concept_name, type=ConceptType.ENTITY_TYPE, extends=[parent]) @@ -176,7 +193,7 @@ def _convert_object_type( # primary_keys() is a set; sort by readable_id so identify_by # ordering (and the resulting YAML) is stable across runs. for prop in sorted(ot.primary_keys(), key=lambda p: p.readable_id()): - prop_name = PalantirToOsiConverter._attribute_name(prop) + prop_name = PalantirToOssieConverter._attribute_name(prop) rel = ontology.lookup_concept_relationship(concept, prop_name) if rel is None: raise ValueError( @@ -189,7 +206,7 @@ def _convert_object_type( # A non-composite identifier is OneToOne; all others stay ManyToOne. sole = next(iter(identifiers.values())) if len(identifiers) == 1 else None for prop in relevant_props: - prop_name = PalantirToOsiConverter._attribute_name(prop) + prop_name = PalantirToOssieConverter._attribute_name(prop) prop_rel = ontology.lookup_concept_relationship(concept, prop_name) if prop_rel is not None: mult = RelationshipMultiplicity.ONE_TO_ONE if prop_rel is sole else RelationshipMultiplicity.MANY_TO_ONE @@ -202,7 +219,7 @@ def _convert_object_type( # otherwise the second dataset is contributing fields the first # didn't declare, which produces an asymmetric model. for prop in relevant_props: - prop_name = PalantirToOsiConverter._attribute_name(prop) + prop_name = PalantirToOssieConverter._attribute_name(prop) if ontology.lookup_concept_relationship(concept, prop_name) is None: raise ValueError( f"Concept '{concept_name}' refers to multiple datasets but not all " @@ -217,10 +234,10 @@ def _convert_property(self, ontology: OntologyComponent, concept: Concept, prop: def madlib_decl(c: Concept, p: PalantirProperty) -> str: return ( f"{{{c}}} {p.readable_id()} " - f"{PalantirToOsiConverter._type_to_madlib_suffix(p.type())}" + f"{PalantirToOssieConverter._type_to_madlib_suffix(p.type())}" ) - prop_name = PalantirToOsiConverter._attribute_name(prop) + prop_name = PalantirToOssieConverter._attribute_name(prop) if ontology.lookup_concept_relationship(concept, prop_name) is not None: return @@ -257,7 +274,7 @@ def _convert_mappings( if subtype_relation is not None: parent_ot = subtype_relation.many_object_type() parent_concept = ontology.lookup_concept( - PalantirToOsiConverter._concept_name(parent_ot) + PalantirToOssieConverter._concept_name(parent_ot) ) property_map = subtype_relation.property_map() identifier_props = list(parent_ot.primary_keys()) @@ -278,13 +295,13 @@ def resolve(p: PalantirProperty) -> PalantirProperty: # dataset's columns. id_referents: list[ReferentMapping] = [] for prop in identifier_props: - prop_name = PalantirToOsiConverter._attribute_name(prop) + prop_name = PalantirToOssieConverter._attribute_name(prop) # For subtypes, identifying relationships live on the parent # concept; the child reaches them via `lookup_concept_relationship`. rel = ontology.lookup_concept_relationship(concept, prop_name) if rel is None: continue - field = PalantirToOsiConverter._get_dataset_field_by_palantir_property( + field = PalantirToOssieConverter._get_dataset_field_by_palantir_property( resolve(prop), palantir_ds, dataset ) if field is None: @@ -327,11 +344,11 @@ def resolve(p: PalantirProperty) -> PalantirProperty: ) continue - prop_name = PalantirToOsiConverter._attribute_name(prop) + prop_name = PalantirToOssieConverter._attribute_name(prop) relationship = ontology.lookup_concept_relationship(concept, prop_name) if relationship is None: continue - field = PalantirToOsiConverter._get_dataset_field_by_palantir_property( + field = PalantirToOssieConverter._get_dataset_field_by_palantir_property( prop, palantir_ds, dataset ) if field is None: @@ -410,14 +427,14 @@ def _convert_many_to_one( semantic_model: SemanticModel, ) -> None: mot = rel.many_object_type() - mot_name = PalantirToOsiConverter._concept_name(mot) + mot_name = PalantirToOssieConverter._concept_name(mot) mot_concept = ontology.lookup_concept(mot_name) oot = rel.one_object_type() - oot_name = PalantirToOsiConverter._concept_name(oot) + oot_name = PalantirToOssieConverter._concept_name(oot) oot_concept = ontology.lookup_concept(oot_name) if mot_concept is None or oot_concept is None: return - prop_name = PalantirToOsiConverter._attribute_name(rel) + prop_name = PalantirToOssieConverter._attribute_name(rel) if mot_concept is oot_concept: verbalize = f"{{{mot_concept}}} {prop_name} {{{oot_concept}:snd}}" @@ -443,8 +460,8 @@ def _convert_many_to_one( # No many-side datasets: fall back to a derived_by formula that # equates FK columns. frags = [ - f"{relationship.first_role.name}.{PalantirToOsiConverter._attribute_name(mprop)}" - f" == {relationship.last_role.name}.{PalantirToOsiConverter._attribute_name(oprop)}" + f"{relationship.first_role.name}.{PalantirToOssieConverter._attribute_name(mprop)}" + f" == {relationship.last_role.name}.{PalantirToOssieConverter._attribute_name(oprop)}" for mprop, oprop in rel.property_map().items() ] if frags: @@ -472,7 +489,7 @@ def _attach_link_to_concept_mappings( # Resolve target (oot) identifying relationships once. target_id_rels: list[tuple[Relationship, PalantirProperty]] = [] for mprop, oprop in property_map.items(): - oot_attr = PalantirToOsiConverter._attribute_name(oprop) + oot_attr = PalantirToOssieConverter._attribute_name(oprop) id_rel = ontology.lookup_concept_relationship(oot_concept, oot_attr) if id_rel is None: return @@ -480,13 +497,13 @@ def _attach_link_to_concept_mappings( for palantir_ds in mot.syncs_from(): ds_name = ( - f"{PalantirToOsiConverter._concept_name(mot)}_{palantir_ds.readable_id()}" + f"{PalantirToOssieConverter._concept_name(mot)}_{palantir_ds.readable_id()}" ) dataset = semantic_model.lookup_dataset(ds_name) if dataset is None: continue - cm = PalantirToOsiConverter._find_concept_mapping(concept_mappings, mot_concept, dataset) + cm = PalantirToOssieConverter._find_concept_mapping(concept_mappings, mot_concept, dataset) if cm is None: warnings.warn( f"No ConceptMapping for entity '{mot_concept.name}' and dataset " @@ -498,7 +515,7 @@ def _attach_link_to_concept_mappings( referents: list[ReferentMapping] = [] resolved = True for id_rel, mprop in target_id_rels: - fk_field = PalantirToOsiConverter._get_dataset_field_by_palantir_property( + fk_field = PalantirToOssieConverter._get_dataset_field_by_palantir_property( mprop, palantir_ds, dataset ) if fk_field is None: @@ -549,7 +566,7 @@ def _find_concept_mapping( if len(candidates) <= 1: return candidates[0] if candidates else None return next( - (cm for cm in candidates if PalantirToOsiConverter._references_dataset(cm, dataset)), + (cm for cm in candidates if PalantirToOssieConverter._references_dataset(cm, dataset)), candidates[0], ) @@ -564,12 +581,12 @@ def _references_dataset(cm: ConceptMapping, dataset: Dataset) -> bool: def _convert_many_to_many(self, ontology: OntologyComponent, rel: ManyToManyRelation) -> None: aot = rel.role_a_player() - aot_concept = ontology.lookup_concept(PalantirToOsiConverter._concept_name(aot)) + aot_concept = ontology.lookup_concept(PalantirToOssieConverter._concept_name(aot)) bot = rel.role_b_player() - bot_concept = ontology.lookup_concept(PalantirToOsiConverter._concept_name(bot)) + bot_concept = ontology.lookup_concept(PalantirToOssieConverter._concept_name(bot)) if aot_concept is None or bot_concept is None: return - rel_name = PalantirToOsiConverter._attribute_name(rel) + rel_name = PalantirToOssieConverter._attribute_name(rel) if aot_concept is bot_concept: verbalize = f"{{{aot_concept}}} {rel_name} {{{bot_concept}:snd}}" @@ -594,14 +611,14 @@ def _convert_intermediary_relation( rel: IntermediaryRelation, ) -> None: aot = rel.role_a_player() - aot_name = PalantirToOsiConverter._concept_name(aot) + aot_name = PalantirToOssieConverter._concept_name(aot) aot_concept = ontology.lookup_concept(aot_name) bot = rel.role_b_player() - bot_name = PalantirToOsiConverter._concept_name(bot) + bot_name = PalantirToOssieConverter._concept_name(bot) bot_concept = ontology.lookup_concept(bot_name) if aot_concept is None or bot_concept is None: return - rel_name = PalantirToOsiConverter._attribute_name(rel) + rel_name = PalantirToOssieConverter._attribute_name(rel) if aot_concept is bot_concept: verbalize = f"{{{aot_concept}}} {rel_name} {{{bot_concept}:snd}}" @@ -619,16 +636,16 @@ def _convert_intermediary_relation( ontology.add_relationship(relationship) rel_a = palantir_ontology.relations()[rel.relation_a()] - rel_a_name = PalantirToOsiConverter._attribute_name(rel_a) + rel_a_name = PalantirToOssieConverter._attribute_name(rel_a) rel_b = palantir_ontology.relations()[rel.relation_b()] - rel_b_name = PalantirToOsiConverter._attribute_name(rel_b) + rel_b_name = PalantirToOssieConverter._attribute_name(rel_b) - fp_a_ot, sp_a_ot = PalantirToOsiConverter._relation_players(rel_a) - fp_a = PalantirToOsiConverter._concept_name(fp_a_ot) - sp_a = PalantirToOsiConverter._concept_name(sp_a_ot) - fp_b_ot, sp_b_ot = PalantirToOsiConverter._relation_players(rel_b) - fp_b = PalantirToOsiConverter._concept_name(fp_b_ot) - sp_b = PalantirToOsiConverter._concept_name(sp_b_ot) + fp_a_ot, sp_a_ot = PalantirToOssieConverter._relation_players(rel_a) + fp_a = PalantirToOssieConverter._concept_name(fp_a_ot) + sp_a = PalantirToOssieConverter._concept_name(sp_a_ot) + fp_b_ot, sp_b_ot = PalantirToOssieConverter._relation_players(rel_b) + fp_b = PalantirToOssieConverter._concept_name(fp_b_ot) + sp_b = PalantirToOssieConverter._concept_name(sp_b_ot) assert (aot_name == fp_a and bot_name == fp_b) or ( aot_name == sp_a and bot_name == sp_b @@ -654,7 +671,7 @@ def _convert_dataset( db_name: str, schema_name: str, ) -> Dataset: - ds_name = f"{PalantirToOsiConverter._concept_name(ot)}_{palantir_ds.readable_id()}" + ds_name = f"{PalantirToOssieConverter._concept_name(ot)}_{palantir_ds.readable_id()}" existing = semantic_model.lookup_dataset(ds_name) if existing is not None: return existing @@ -666,7 +683,7 @@ def _convert_dataset( col_type = column.type() if col_type is not None and col_type.upper() == "ARRAY": continue - field_name = PalantirToOsiConverter._normalize_field_name(column.name()) + field_name = PalantirToOssieConverter._normalize_field_name(column.name()) fields.append( DatasetField( name=field_name, @@ -675,7 +692,7 @@ def _convert_dataset( DialectExpression(dialect=_DEFAULT_DIALECT, expression=field_name) ] ), - type=PalantirToOsiConverter._resolve_field_type(ontology, palantir_ds, column), + type=PalantirToOssieConverter._resolve_field_type(ontology, palantir_ds, column), ) ) @@ -720,8 +737,8 @@ def _type_to_madlib_suffix(type_, arr_depth: int = 1) -> str: if isinstance(type_, ArrayDataType): depth = arr_depth return ( - f"{{Integer:{PalantirToOsiConverter._depth_role_name(depth)}}} maps to " - f"{PalantirToOsiConverter._type_to_madlib_suffix(type_.base_type(), depth + 1)}" + f"{{Integer:{PalantirToOssieConverter._depth_role_name(depth)}}} maps to " + f"{PalantirToOssieConverter._type_to_madlib_suffix(type_.base_type(), depth + 1)}" ) return f"{{{type_.to_type()}}}" @@ -732,7 +749,7 @@ def _convert_property_type_roles( integer = ontology.lookup_concept("Integer") if integer is None: raise ValueError("Builtin 'Integer' could not be resolved for array role.") - roles.append((integer, PalantirToOsiConverter._depth_role_name(arr_depth))) + roles.append((integer, PalantirToOssieConverter._depth_role_name(arr_depth))) self._convert_property_type_roles(ontology, roles, type_.base_type(), arr_depth + 1) else: target = ontology.lookup_concept(type_.to_type()) @@ -757,7 +774,7 @@ def _relation_players(rel: Relation) -> tuple[ObjectType, ObjectType]: @staticmethod def _depth_role_name(depth: int) -> str: - name = PalantirToOsiConverter.depths_role_names.get(depth) + name = PalantirToOssieConverter.depths_role_names.get(depth) if not name: raise ValueError(f"Array types of depth {depth} are not supported") return name @@ -773,12 +790,12 @@ def _get_dataset_field_by_palantir_property( if ds_guid not in pk_mapping: raise ValueError( f"Primary key mapping for Palantir DataSet '{palantir_ds.readable_id()}' " - f"is missing property '{PalantirToOsiConverter._attribute_name(prop)}'" + f"is missing property '{PalantirToOssieConverter._attribute_name(prop)}'" ) column_name = pk_mapping[ds_guid] if not column_name: return None - field = dataset.field(PalantirToOsiConverter._normalize_field_name(column_name)) + field = dataset.field(PalantirToOssieConverter._normalize_field_name(column_name)) if not field: warnings.warn(f"Dataset '{dataset.name}' does not contain a field named '{column_name}'") return field diff --git a/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/__init__.py b/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/__init__.py new file mode 100644 index 00000000..13a83393 --- /dev/null +++ b/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/ontology/src/osi/converter/spec_to_osi/converter.py b/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py similarity index 92% rename from converters/ontology/src/osi/converter/spec_to_osi/converter.py rename to converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py index e68d90a2..c2bf812e 100644 --- a/converters/ontology/src/osi/converter/spec_to_osi/converter.py +++ b/converters/ontology/src/ossie_ontology/converter/spec_to_ossie/converter.py @@ -1,11 +1,28 @@ -"""Converter from OsiSpec (Pydantic DTOs) to OsiOntology (runtime semantic model).""" +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Converter from OssieSpec (Pydantic DTOs) to OssieOntology (runtime semantic model).""" from __future__ import annotations import re -from osi.common.graph import topological_sort -from osi.model import ( +from ossie_ontology.common.graph import topological_sort +from ossie_ontology.model import ( Concept, ConceptMapping, ConceptType, @@ -28,10 +45,10 @@ ReferentMapping, Relationship, RelationshipMultiplicity, - OsiOntology, + OssieOntology, BUILTIN_CONCEPTS ) -from osi.spec import ( +from ossie_ontology.spec import ( ConceptComponent as SpecConceptComponent, ConceptMapping as SpecConceptMapping, CustomExtension as SpecCustomExtension, @@ -46,7 +63,7 @@ Metric as SpecMetric, ObjectMapping as SpecObjectMapping, OntologyMapping as SpecOntologyMapping, - OsiSpec, + OssieSpec, ReferentMapping as SpecReferentMapping, Relationship as SpecRelationship, ) @@ -58,15 +75,15 @@ _BARE_FIELD_RE = re.compile(r"^\s*([A-Za-z_][A-Za-z0-9_]*)\s*$") -class SpecToOsiConverter: - """Converts OsiSpec (Pydantic DTOs) to OsiOntology (runtime model). +class SpecToOssieConverter: + """Converts OssieSpec (Pydantic DTOs) to OssieOntology (runtime model). Pass a *formula_factory* to control how Formula objects are created. The default produces plain ``Formula`` instances; downstream packages can inject a factory that returns enriched subclasses (e.g. with an AST). - model = SpecToOsiConverter().convert(spec) - model = SpecToOsiConverter(formula_factory=my_parser).convert(spec) + model = SpecToOssieConverter().convert(spec) + model = SpecToOssieConverter(formula_factory=my_parser).convert(spec) """ def __init__(self, formula_factory: FormulaFactory | None = None, @@ -74,9 +91,9 @@ def __init__(self, formula_factory: FormulaFactory | None = None, self._formula_factory = formula_factory or FormulaFactory() self._mapping_formula_factory = mapping_formula_factory or MappingFormulaFactory() - def convert(self, spec: OsiSpec) -> OsiOntology: + def convert(self, spec: OssieSpec) -> OssieOntology: ontology = OntologyComponent() - model = OsiOntology( + model = OssieOntology( name=spec.name, ontology=ontology, description=spec.description, @@ -93,7 +110,7 @@ def convert(self, spec: OsiSpec) -> OsiOntology: # ----- Ontology ------------------------------------------------------ - def _populate_ontology(self, ontology: OntologyComponent, spec: OsiSpec) -> None: + def _populate_ontology(self, ontology: OntologyComponent, spec: OssieSpec) -> None: concept_specs = {cc.concept: cc for cc in spec.ontology} sorted_names = self._sort_spec_dependency_graph(list(concept_specs.values())) @@ -226,7 +243,7 @@ def _convert_semantic_model(self, lm_spec: SpecSemanticModel) -> SemanticModel: # ----- Ontology mapping --------------------------------------------- - def _convert_ontology_mapping(self, model: OsiOntology, om_spec: SpecOntologyMapping) -> None: + def _convert_ontology_mapping(self, model: OssieOntology, om_spec: SpecOntologyMapping) -> None: ontology = model.ontology semantic_model = self._convert_semantic_model(om_spec.semantic_model) @@ -244,7 +261,7 @@ def _convert_ontology_mapping(self, model: OsiOntology, om_spec: SpecOntologyMap def _convert_concept_mapping( self, - model: OsiOntology, + model: OssieOntology, ontology: OntologyComponent, semantic_model: SemanticModel, cm_spec: SpecConceptMapping, @@ -267,7 +284,7 @@ def _convert_concept_mapping( def _convert_object_mapping( self, - model: OsiOntology, + model: OssieOntology, ontology: OntologyComponent, semantic_model: SemanticModel, container: Concept, @@ -295,7 +312,7 @@ def _convert_object_mapping( def _convert_referent_mapping( self, - model: OsiOntology, + model: OssieOntology, ontology: OntologyComponent, semantic_model: SemanticModel, container: Concept, @@ -321,7 +338,7 @@ def _convert_referent_mapping( def _convert_link_mapping( self, - model: OsiOntology, + model: OssieOntology, ontology: OntologyComponent, semantic_model: SemanticModel, container: Concept, @@ -521,4 +538,4 @@ def _convert_metric(m: SpecMetric) -> Metric: description=m.description, ai_context=m.ai_context, custom_extensions=[_convert_custom_extension(ce) for ce in m.custom_extensions], - ) \ No newline at end of file + ) diff --git a/converters/ontology/src/ossie_ontology/external/__init__.py b/converters/ontology/src/ossie_ontology/external/__init__.py new file mode 100644 index 00000000..13a83393 --- /dev/null +++ b/converters/ontology/src/ossie_ontology/external/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/ontology/src/ossie_ontology/external/palantir/__init__.py b/converters/ontology/src/ossie_ontology/external/palantir/__init__.py new file mode 100644 index 00000000..13a83393 --- /dev/null +++ b/converters/ontology/src/ossie_ontology/external/palantir/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/ontology/src/osi/external/palantir/model.py b/converters/ontology/src/ossie_ontology/external/palantir/model.py similarity index 96% rename from converters/ontology/src/osi/external/palantir/model.py rename to converters/ontology/src/ossie_ontology/external/palantir/model.py index 68687e19..50a2ffde 100644 --- a/converters/ontology/src/osi/external/palantir/model.py +++ b/converters/ontology/src/ossie_ontology/external/palantir/model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import annotations from enum import Enum diff --git a/converters/ontology/src/osi/external/palantir/parser/__init__.py b/converters/ontology/src/ossie_ontology/external/palantir/parser/__init__.py similarity index 96% rename from converters/ontology/src/osi/external/palantir/parser/__init__.py rename to converters/ontology/src/ossie_ontology/external/palantir/parser/__init__.py index c0cb4c50..ea74bf29 100644 --- a/converters/ontology/src/osi/external/palantir/parser/__init__.py +++ b/converters/ontology/src/ossie_ontology/external/palantir/parser/__init__.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import json import warnings import zipfile @@ -5,10 +22,10 @@ from pathlib import Path from typing import Any, Iterable -from osi.common.utils import camel_to_snake -from osi.external.palantir.model import DataSet, DataSetColumn, DataSetModel, ObjectType, Ontology, DataType, \ +from ossie_ontology.common.utils import camel_to_snake +from ossie_ontology.external.palantir.model import DataSet, DataSetColumn, DataSetModel, ObjectType, Ontology, DataType, \ ArrayDataType, Property, Status, ManyToOneRelation, Relation, ManyToManyRelation, IntermediaryRelation, DataSource -from osi.common.file_utils import iter_json_files_from_dir_in_zip, open_top_level_file_from_zip, \ +from ossie_ontology.common.file_utils import iter_json_files_from_dir_in_zip, open_top_level_file_from_zip, \ iter_json_files_from_dir, get_top_level_json_file_from_dir, validate_dir diff --git a/converters/ontology/src/osi/model.py b/converters/ontology/src/ossie_ontology/model.py similarity index 97% rename from converters/ontology/src/osi/model.py rename to converters/ontology/src/ossie_ontology/model.py index 298dc6e3..80f01ee1 100644 --- a/converters/ontology/src/osi/model.py +++ b/converters/ontology/src/ossie_ontology/model.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import annotations import re @@ -132,7 +149,7 @@ def description(self) -> str | None: def is_builtin(self) -> bool: return self._builtin - # True if this concept identifies a concept component in an OSI ontology + # True if this concept identifies a concept component in an Ossie ontology @property def is_component(self) -> bool: return self._is_component @@ -179,7 +196,7 @@ def __str__(self) -> str: class Relationship: """A relationship grouped under its first-role concept (the container). - In this model class we choose to store all the roles explicitly, including the first implicit role from the OSI spec. + In this model class we choose to store all the roles explicitly, including the first implicit role from the Ossie spec. """ _name: str _container: Concept @@ -878,7 +895,7 @@ def on_require_added(self, require: Formula) -> None: ... class OntologyComponent: """Structural container for concepts, relationships, constraints, and rules. - Document-level metadata (name, description, ai_context) lives on OsiOntology.""" + Document-level metadata (name, description, ai_context) lives on OssieOntology.""" _concepts: list[Concept] _relationships: list[Relationship] _requires: list[Formula] @@ -957,10 +974,10 @@ def lookup_concept_relationship(self, concept: Concept, name: str) -> Relationsh # --------------------------------------------------------------------------- -# Root semantic model (per OsiSpec) +# Root semantic model (per OssieSpec) # --------------------------------------------------------------------------- -class OsiOntology: +class OssieOntology: _name: str _description: str | None _ai_context: AiContext | None diff --git a/converters/ontology/src/ossie_ontology/parser/__init__.py b/converters/ontology/src/ossie_ontology/parser/__init__.py new file mode 100644 index 00000000..049d7a35 --- /dev/null +++ b/converters/ontology/src/ossie_ontology/parser/__init__.py @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Entrypoint: read a YAML/JSON Ossie spec and produce an OssieOntology.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import yaml + +from ossie_ontology.converter.spec_to_ossie.converter import SpecToOssieConverter +from ossie_ontology.model import OssieOntology, FormulaFactory, MappingFormulaFactory +from ossie_ontology.spec import OssieSpec + + +class OssieParser: + _model: OssieOntology | None + _spec: OssieSpec | None + _debug: bool + + def __init__(self, debug: bool = False, + formula_factory: FormulaFactory | None = None, + mapping_formula_factory: MappingFormulaFactory | None = None): + self._debug = debug + self._model = None + self._spec = None + self._formula_factory = formula_factory or FormulaFactory() + self._mapping_formula_factory = mapping_formula_factory or MappingFormulaFactory() + + def parse(self, path: Path) -> OssieOntology: + # Ossie always expects a single spec file. + if not path.is_file(): + raise ValueError(f"Expected a single Ossie spec file, but '{path}' is not a file") + raw = OssieParser.load_data(path) + self._spec = OssieSpec.model_validate(raw) + self._model = SpecToOssieConverter( + formula_factory=self._formula_factory, mapping_formula_factory=self._mapping_formula_factory + ).convert(self._spec) + return self._model + + @staticmethod + def load_data(path: Path) -> Any: + # Pin UTF-8 so parsing is reproducible regardless of the process locale. + content = path.read_text(encoding="utf-8") + if path.suffix.lower() == ".json": + return json.loads(content) + return yaml.safe_load(content) + + def spec(self) -> OssieSpec: + spec = self._spec + if spec is None: + raise RuntimeError("You must call 'parse()' before accessing 'spec()'") + return spec diff --git a/converters/ontology/src/osi/spec.py b/converters/ontology/src/ossie_ontology/spec.py similarity index 84% rename from converters/ontology/src/osi/spec.py rename to converters/ontology/src/ossie_ontology/spec.py index dea4ebd2..377c2821 100644 --- a/converters/ontology/src/osi/spec.py +++ b/converters/ontology/src/ossie_ontology/spec.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from __future__ import annotations from typing import Any, Literal @@ -10,8 +27,8 @@ DEFAULT_YAML_WIDTH = 1000 -class OsiObject(BaseModel): - """Base for all OSI DTOs. Strict (`extra=forbid`) to surface spec drift early.""" +class OssieObject(BaseModel): + """Base for all Ossie DTOs. Strict (`extra=forbid`) to surface spec drift early.""" model_config = ConfigDict( populate_by_name=True, arbitrary_types_allowed=True, @@ -24,7 +41,7 @@ class OsiObject(BaseModel): AiContext = str | dict[str, Any] -class CustomExtension(OsiObject): +class CustomExtension(OssieObject): """Vendor-specific metadata attached to any logical-model element (core.md). `data` is a JSON-encoded string so vendors can carry arbitrary payloads @@ -36,7 +53,7 @@ class CustomExtension(OsiObject): # ---------- Ontology ---------- -class Role(OsiObject): +class Role(OssieObject): """An additional role in a Relationship (the first role is implicit — the container concept). `name` is only required to disambiguate when the same concept plays multiple roles in the same relationship.""" @@ -44,7 +61,7 @@ class Role(OsiObject): name: str | None = None -class Relationship(OsiObject): +class Relationship(OssieObject): """A relationship grouped under its first-role concept. `roles` enumerates the *additional* roles (the first is the container @@ -62,7 +79,7 @@ class Relationship(OsiObject): requires: list[str] = Field(default_factory=list) -class ConceptComponent(OsiObject): +class ConceptComponent(OssieObject): """A concept and the relationships grouped under it. A concept is a type-like node in the ontology — either an `EntityType` @@ -90,26 +107,26 @@ class ConceptComponent(OsiObject): relationships: list[Relationship] = Field(default_factory=list) -# ---------- Logical model (per osi/core.md) ---------- +# ---------- Logical model (per ossie/core.md) ---------- -class DialectExpression(OsiObject): +class DialectExpression(OssieObject): """A scalar (non-aggregating) SQL/expression in a specific dialect.""" dialect: str expression: str -class Expression(OsiObject): +class Expression(OssieObject): """Multi-dialect expression carrier — same logical expression rendered in one or more dialects (e.g. ANSI_SQL + SNOWFLAKE).""" dialects: list[DialectExpression] = Field(default_factory=list) -class Dimension(OsiObject): +class Dimension(OssieObject): """Dimensional metadata on a DatasetField.""" is_time: bool | None = None -class DatasetField(OsiObject): +class DatasetField(OssieObject): """A row-level attribute of a Dataset. `expression` is scalar (no aggregations); use Metric for aggregates.""" name: str @@ -121,7 +138,7 @@ class DatasetField(OsiObject): custom_extensions: list[CustomExtension] = Field(default_factory=list) -class Dataset(OsiObject): +class Dataset(OssieObject): """A logical dataset (fact or dimension table) backed by `source` — a physical table/view reference or a query.""" name: str @@ -134,7 +151,7 @@ class Dataset(OsiObject): custom_extensions: list[CustomExtension] = Field(default_factory=list) -class JoinPath(OsiObject): +class JoinPath(OssieObject): """A foreign-key style join between two Datasets: rows in `from` reference rows in `to` by matching `from_columns` against `to_columns` in order. Same arity required on both sides.""" @@ -147,7 +164,7 @@ class JoinPath(OsiObject): custom_extensions: list[CustomExtension] = Field(default_factory=list) -class Metric(OsiObject): +class Metric(OssieObject): """A model-level quantitative measure defined as an aggregate expression. Can reference fields across multiple Datasets.""" name: str @@ -157,7 +174,7 @@ class Metric(OsiObject): custom_extensions: list[CustomExtension] = Field(default_factory=list) -class SemanticModel(OsiObject): +class SemanticModel(OssieObject): """A complete logical/semantic model (the body that the core spec calls `semantic_model`): datasets plus the join paths and metrics defined over them. One or more SemanticModels can feed a single OntologyMapping.""" @@ -172,7 +189,7 @@ class SemanticModel(OsiObject): # ---------- Ontology mapping ---------- -class ReferentMapping(OsiObject): +class ReferentMapping(OssieObject): """Locates an entity object by walking one of its identifying relationships. Carries either a leaf `expression` (SQL over dataset fields) or a nested `referent_mappings` list when the referenced concept @@ -182,7 +199,7 @@ class ReferentMapping(OsiObject): referent_mappings: list[ReferentMapping] | None = None -class ObjectMapping(OsiObject): +class ObjectMapping(OssieObject): """Maps to objects of some concept. Either a direct scalar `expression` (for value types or simple-id entities) or `referent_mappings` (for entities with compound identifiers). XOR — never both.""" @@ -191,7 +208,7 @@ class ObjectMapping(OsiObject): referent_mappings: list[ReferentMapping] | None = None -class LinkMapping(OsiObject): +class LinkMapping(OssieObject): """A node in the link-mapping tree. The arity of `relationship` equals the node's depth (top-level = unary, depth 2 = binary, etc.). `children` extend the mapped tuple by one role each, sharing this node's @@ -201,7 +218,7 @@ class LinkMapping(OsiObject): children: list[LinkMapping] | None = None -class ConceptMapping(OsiObject): +class ConceptMapping(OssieObject): """Mappings that populate one concept and the relationships grouped under it. `object_mappings` populate the concept's objects; `link_mappings` is a forest of trees populating its relationships.""" @@ -210,7 +227,7 @@ class ConceptMapping(OsiObject): link_mappings: list[LinkMapping] = Field(default_factory=list) -class OntologyMapping(OsiObject): +class OntologyMapping(OssieObject): """Binds a semantic model to the document ontology, then declares how its fields populate the ontology's concepts and relationships.""" name: str @@ -221,8 +238,8 @@ class OntologyMapping(OsiObject): # ---------- Root ---------- -class OsiSpec(OsiObject): - """Root OSI document: a single ontology definition and the ontology +class OssieSpec(OssieObject): + """Root Ossie document: a single ontology definition and the ontology mappings that wire semantic models into it.""" version: str | None = None name: str @@ -233,7 +250,7 @@ class OsiSpec(OsiObject): ontology_mappings: list[OntologyMapping] = Field(default_factory=list) @classmethod - def load_yaml(cls, text: str) -> OsiSpec: + def load_yaml(cls, text: str) -> OssieSpec: return cls.model_validate(yaml.safe_load(text)) def dump_dict(self) -> dict: diff --git a/converters/ontology/tests/__init__.py b/converters/ontology/tests/__init__.py index e69de29b..13a83393 100644 --- a/converters/ontology/tests/__init__.py +++ b/converters/ontology/tests/__init__.py @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/converters/ontology/tests/conftest.py b/converters/ontology/tests/conftest.py index 9e015f1b..947c4eb0 100644 --- a/converters/ontology/tests/conftest.py +++ b/converters/ontology/tests/conftest.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Shared fixtures for the ontology converter test suite.""" from __future__ import annotations @@ -6,8 +23,8 @@ import pytest -from osi.model import OsiOntology -from osi.parser import OsiParser +from ossie_ontology.model import OssieOntology +from ossie_ontology.parser import OssieParser # Test inputs are vendored under tests/fixtures/ so the suite runs even when the # repo-level examples/ directory isn't present (e.g. from an sdist/wheel or a @@ -15,6 +32,59 @@ # canonical examples/ copies. _FIXTURES_DIR = Path(__file__).resolve().parent / "fixtures" +# The snapshot files under tests/snapshots/ carry the ASF license header inline, +# so that a source-release audit (Apache RAT) finds a header on every file in the +# tree without needing an exclude list. pytest-snapshot compares the whole file +# byte-for-byte, so the header has to be part of the asserted value; the snapshot +# fixture below applies it, which also means `pytest --snapshot-update` reproduces +# it and it cannot drift. +# +# Both snapshot formats tolerate it: '#' starts a comment in YAML, and the +# structure snapshot is free-form text. +LICENSE_HEADER = """\ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +""" + + +@pytest.fixture +def snapshot(snapshot): + """pytest-snapshot's fixture, with the ASF license header applied for free. + + Overrides the plugin fixture so tests assert only their own payload and the + header stays in one place. Applies on both paths: comparison prepends it to + the expected value, and --snapshot-update writes it into the file. + + Note: only assert_match is wrapped, since that is all this suite uses. A test + reaching for assert_match_dir would need the same treatment. + """ + assert_match = snapshot.assert_match + + def assert_match_with_license(value: str | bytes, snapshot_name: str | Path) -> None: + __tracebackhide__ = True + if isinstance(value, bytes): + assert_match(LICENSE_HEADER.encode() + value, snapshot_name) + else: + assert_match(LICENSE_HEADER + value, snapshot_name) + + snapshot.assert_match = assert_match_with_license + return snapshot + @pytest.fixture(scope="session") def fixtures_dir() -> Path: @@ -27,5 +97,5 @@ def flights_path(fixtures_dir: Path) -> Path: @pytest.fixture -def flights_model(flights_path: Path) -> OsiOntology: - return OsiParser().parse(flights_path) \ No newline at end of file +def flights_model(flights_path: Path) -> OssieOntology: + return OssieParser().parse(flights_path) \ No newline at end of file diff --git a/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml index 7731a65f..805fadef 100644 --- a/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml +++ b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_roundtrip_yaml_snapshot/flights_roundtrip.yaml @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + version: 0.2.0.dev0 name: Flights description: Ontology of flights into and out of airports. diff --git a/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_structure_snapshot/flights_structure.txt b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_structure_snapshot/flights_structure.txt index 65cf234b..0c66acf4 100644 --- a/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_structure_snapshot/flights_structure.txt +++ b/converters/ontology/tests/snapshots/test_flights_snapshot/test_flights_structure_snapshot/flights_structure.txt @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + name: Flights version: 0.2.0.dev0 description: Ontology of flights into and out of airports. diff --git a/converters/ontology/tests/test_examples_in_sync.py b/converters/ontology/tests/test_examples_in_sync.py index b2694f19..724092b1 100644 --- a/converters/ontology/tests/test_examples_in_sync.py +++ b/converters/ontology/tests/test_examples_in_sync.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Guard against drift between the vendored test inputs under tests/fixtures/ and the canonical copies under the repo-level examples/ directory. diff --git a/converters/ontology/tests/test_flights_snapshot.py b/converters/ontology/tests/test_flights_snapshot.py index 92ca1692..6f31430c 100644 --- a/converters/ontology/tests/test_flights_snapshot.py +++ b/converters/ontology/tests/test_flights_snapshot.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Snapshot tests for the flights ontology. These lock in the converted structure and the round-tripped YAML so that any @@ -10,11 +27,11 @@ from __future__ import annotations -from osi.converter.osi_to_spec.converter import OsiToSpecConverter -from osi.model import OntologyComponent, OsiOntology +from ossie_ontology.converter.ossie_to_spec.converter import OssieToSpecConverter +from ossie_ontology.model import OntologyComponent, OssieOntology -def _render_structure(model: OsiOntology) -> str: +def _render_structure(model: OssieOntology) -> str: """Render a compact, deterministic text summary of the ontology structure.""" ontology: OntologyComponent = model.ontology lines: list[str] = [ @@ -54,5 +71,5 @@ def test_flights_structure_snapshot(flights_model, snapshot): def test_flights_roundtrip_yaml_snapshot(flights_model, snapshot): - spec = OsiToSpecConverter.convert(flights_model) + spec = OssieToSpecConverter.convert(flights_model) snapshot.assert_match(spec.dump_yaml(), "flights_roundtrip.yaml") \ No newline at end of file diff --git a/converters/ontology/tests/test_osi_parser.py b/converters/ontology/tests/test_ossie_parser.py similarity index 76% rename from converters/ontology/tests/test_osi_parser.py rename to converters/ontology/tests/test_ossie_parser.py index ab682222..8d1a0cf6 100644 --- a/converters/ontology/tests/test_osi_parser.py +++ b/converters/ontology/tests/test_ossie_parser.py @@ -1,4 +1,21 @@ -"""Unit tests for OsiParser and the spec -> OsiOntology conversion, driven by +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Unit tests for OssieParser and the spec -> OssieOntology conversion, driven by the `examples/flights.yaml` ontology.""" from __future__ import annotations @@ -9,9 +26,9 @@ import pytest import yaml -from osi.converter.osi_to_spec.converter import OsiToSpecConverter -from osi.model import ConceptType, OsiOntology, RelationshipMultiplicity -from osi.parser import OsiParser +from ossie_ontology.converter.ossie_to_spec.converter import OssieToSpecConverter +from ossie_ontology.model import ConceptType, OssieOntology, RelationshipMultiplicity +from ossie_ontology.parser import OssieParser # ----- Document-level metadata ------------------------------------------ @@ -111,13 +128,13 @@ def test_ontology_mapping(flights_model): def test_load_data_reads_yaml(tmp_path: Path): path = tmp_path / "spec.yaml" path.write_text("a: 1\nb:\n - x\n - y\n") - assert OsiParser.load_data(path) == {"a": 1, "b": ["x", "y"]} + assert OssieParser.load_data(path) == {"a": 1, "b": ["x", "y"]} def test_load_data_reads_json(tmp_path: Path): path = tmp_path / "spec.json" path.write_text(json.dumps({"a": 1, "b": ["x", "y"]})) - assert OsiParser.load_data(path) == {"a": 1, "b": ["x", "y"]} + assert OssieParser.load_data(path) == {"a": 1, "b": ["x", "y"]} def test_parse_of_flights_as_json(flights_path: Path, tmp_path: Path): @@ -125,7 +142,7 @@ def test_parse_of_flights_as_json(flights_path: Path, tmp_path: Path): # the same spec must produce an equivalent model. json_path = tmp_path / "flights.json" json_path.write_text(json.dumps(yaml.safe_load(flights_path.read_text()))) - model = OsiParser().parse(json_path) + model = OssieParser().parse(json_path) assert model.name == "Flights" assert len(model.ontology.concepts(exclude_builtin=True)) == 44 @@ -134,29 +151,29 @@ def test_parse_of_flights_as_json(flights_path: Path, tmp_path: Path): def test_parse_rejects_directory(tmp_path: Path): with pytest.raises(ValueError, match="is not a file"): - OsiParser().parse(tmp_path) + OssieParser().parse(tmp_path) def test_parse_rejects_missing_file(tmp_path: Path): with pytest.raises(ValueError, match="is not a file"): - OsiParser().parse(tmp_path / "does_not_exist.yaml") + OssieParser().parse(tmp_path / "does_not_exist.yaml") def test_spec_requires_parse_first(): - parser = OsiParser() + parser = OssieParser() with pytest.raises(RuntimeError): parser.spec() def test_parsers_do_not_share_formula_factories(): - a, b = OsiParser(), OsiParser() + a, b = OssieParser(), OssieParser() assert a._formula_factory is not b._formula_factory assert a._mapping_formula_factory is not b._mapping_formula_factory # ----- Round-trip invariants -------------------------------------------- -def _structure_sets(model: OsiOntology): +def _structure_sets(model: OssieOntology): ontology = model.ontology return ( {c.name for c in ontology.concepts(exclude_builtin=True)}, @@ -173,12 +190,12 @@ def test_roundtrip_preserves_structure(flights_path, tmp_path: Path): order), so we compare the sets of concepts, relationships, and requires rather than the raw YAML. """ - model1 = OsiParser().parse(flights_path) - yaml1 = OsiToSpecConverter.convert(model1).dump_yaml() + model1 = OssieParser().parse(flights_path) + yaml1 = OssieToSpecConverter.convert(model1).dump_yaml() roundtrip_path = tmp_path / "roundtrip.yaml" roundtrip_path.write_text(yaml1) - model2 = OsiParser().parse(roundtrip_path) + model2 = OssieParser().parse(roundtrip_path) assert _structure_sets(model1) == _structure_sets(model2) @@ -186,6 +203,6 @@ def test_roundtrip_preserves_structure(flights_path, tmp_path: Path): def test_dump_yaml_is_deterministic_for_fixed_input(flights_path): """The same input file always dumps to identical YAML (so the snapshot is stable across runs).""" - yaml_a = OsiToSpecConverter.convert(OsiParser().parse(flights_path)).dump_yaml() - yaml_b = OsiToSpecConverter.convert(OsiParser().parse(flights_path)).dump_yaml() + yaml_a = OssieToSpecConverter.convert(OssieParser().parse(flights_path)).dump_yaml() + yaml_b = OssieToSpecConverter.convert(OssieParser().parse(flights_path)).dump_yaml() assert yaml_a == yaml_b \ No newline at end of file diff --git a/converters/ontology/tests/test_palantir_parser.py b/converters/ontology/tests/test_palantir_parser.py index 1879eabc..dfdeab5f 100644 --- a/converters/ontology/tests/test_palantir_parser.py +++ b/converters/ontology/tests/test_palantir_parser.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Tests for the Palantir parser's input-shape handling. A Palantir export can arrive in several layouts — a ZIP archive, an already @@ -15,8 +32,8 @@ import pytest -from osi.external.palantir.model import Ontology -from osi.external.palantir.parser import PalantirParser +from ossie_ontology.external.palantir.model import Ontology +from ossie_ontology.external.palantir.parser import PalantirParser # A minimal-but-complete Palantir export: one object type backed by one dataset. _ONTOLOGY_JSON = { diff --git a/converters/ontology/tests/test_verbalization_parsing.py b/converters/ontology/tests/test_verbalization_parsing.py index 2b82846c..4f9b405f 100644 --- a/converters/ontology/tests/test_verbalization_parsing.py +++ b/converters/ontology/tests/test_verbalization_parsing.py @@ -1,3 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + """Tests for verbalization parsing — the dash-convention text splitting in ``_parse_verbalization`` / ``_split_segment``. @@ -14,7 +31,7 @@ import pytest -from osi.model import Concept, Relationship, _parse_verbalization as parse_verbalization +from ossie_ontology.model import Concept, Relationship, _parse_verbalization as parse_verbalization def _rel(name: str, container: Concept, relates: list[tuple[Concept, str | None]], verbalization: str) -> Relationship: