From b1bf4155db53647bb840e1e704dbe1936c32707b Mon Sep 17 00:00:00 2001 From: Ralf Becher Date: Tue, 16 Jun 2026 09:31:34 +0200 Subject: [PATCH 1/7] Add OrionBelt (OBML) converter Bidirectional OSI <-> OrionBelt OBML converter, plus a ROADMAP entry. OSI -> OBML and OBML -> OSI with roundtrip fidelity via vendor custom_extensions. Highlights from review on this PR: - Reads ANSI_SQL, SNOWFLAKE, and DATABRICKS metric expressions (their aggregations are ANSI-compatible); MDX/TABLEAU/MAQL are not parsed as SQL. - Resolves dataset.column references against the OSI dataset/field map case-insensitively and with SQL quoting stripped; decimal literals are kept literal. Unmappable references preserve the metric verbatim. - Metrics with no OBML representation are preserved verbatim under an OSI-vendor customExtension and re-emitted on OBML -> OSI, with a loud LOSSY: warning, so the OSI -> OBML -> OSI roundtrip stays lossless. - Emits schema-conformant OSI: no root-level dialects/vendors (those live per-expression and per-entity), matching the published core schema. - Name-collision safety and idempotent convert() on both directions. --- ROADMAP.md | 1 + converters/orionbelt/README.md | 125 + .../orionbelt/osi_obml_mapping_analysis.md | 269 ++ .../osi_obml_ontology_mapping_analysis.md | 82 + converters/orionbelt/pyproject.toml | 58 + .../orionbelt/src/osi_orionbelt/__init__.py | 41 + converters/orionbelt/src/osi_orionbelt/cli.py | 155 + .../orionbelt/src/osi_orionbelt/converter.py | 2967 +++++++++++++++++ .../osi_orionbelt/schemas/obml-schema.json | 1319 ++++++++ .../schemas/osi-ontology-schema.json | 299 ++ .../src/osi_orionbelt/schemas/osi-schema.json | 330 ++ .../orionbelt/tests/fixtures/obml_as_osi.yaml | 787 +++++ .../tests/fixtures/tpcds_as_obml.yaml | 381 +++ .../orionbelt/tests/fixtures/tpcds_osi.yaml | 578 ++++ .../tests/fixtures/tpcds_semantic_model.yaml | 578 ++++ .../tests/test_osi_converter_cumulative.py | 323 ++ .../tests/test_osi_converter_filters.py | 124 + .../test_osi_converter_measure_overrides.py | 297 ++ .../tests/test_osi_converter_ontology.py | 178 + .../orionbelt/tests/test_osi_converter_pop.py | 424 +++ .../tests/test_osi_converter_properties.py | 314 ++ .../tests/test_osi_converter_trend_v26.py | 212 ++ .../tests/test_osi_converter_vendors.py | 237 ++ .../tests/test_osi_metric_no_silent_loss.py | 365 ++ .../tests/test_osi_tpcds_baseline.py | 72 + .../orionbelt/tests/test_osi_v02_compat.py | 417 +++ converters/orionbelt/uv.lock | 1115 +++++++ 27 files changed, 12048 insertions(+) create mode 100644 converters/orionbelt/README.md create mode 100644 converters/orionbelt/osi_obml_mapping_analysis.md create mode 100644 converters/orionbelt/osi_obml_ontology_mapping_analysis.md create mode 100644 converters/orionbelt/pyproject.toml create mode 100644 converters/orionbelt/src/osi_orionbelt/__init__.py create mode 100644 converters/orionbelt/src/osi_orionbelt/cli.py create mode 100644 converters/orionbelt/src/osi_orionbelt/converter.py create mode 100644 converters/orionbelt/src/osi_orionbelt/schemas/obml-schema.json create mode 100644 converters/orionbelt/src/osi_orionbelt/schemas/osi-ontology-schema.json create mode 100644 converters/orionbelt/src/osi_orionbelt/schemas/osi-schema.json create mode 100644 converters/orionbelt/tests/fixtures/obml_as_osi.yaml create mode 100644 converters/orionbelt/tests/fixtures/tpcds_as_obml.yaml create mode 100644 converters/orionbelt/tests/fixtures/tpcds_osi.yaml create mode 100644 converters/orionbelt/tests/fixtures/tpcds_semantic_model.yaml create mode 100644 converters/orionbelt/tests/test_osi_converter_cumulative.py create mode 100644 converters/orionbelt/tests/test_osi_converter_filters.py create mode 100644 converters/orionbelt/tests/test_osi_converter_measure_overrides.py create mode 100644 converters/orionbelt/tests/test_osi_converter_ontology.py create mode 100644 converters/orionbelt/tests/test_osi_converter_pop.py create mode 100644 converters/orionbelt/tests/test_osi_converter_properties.py create mode 100644 converters/orionbelt/tests/test_osi_converter_trend_v26.py create mode 100644 converters/orionbelt/tests/test_osi_converter_vendors.py create mode 100644 converters/orionbelt/tests/test_osi_metric_no_silent_loss.py create mode 100644 converters/orionbelt/tests/test_osi_tpcds_baseline.py create mode 100644 converters/orionbelt/tests/test_osi_v02_compat.py create mode 100644 converters/orionbelt/uv.lock diff --git a/ROADMAP.md b/ROADMAP.md index 04a8ae94..992efe00 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -415,6 +415,7 @@ Broad ecosystem adoption depends on practical tools that let teams validate thei - [GoodData Converter](converters/gooddata/) — bidirectional OSI ↔ GoodData LDM converter - [Salesforce Converter](converters/salesforce/) — OSI ↔ Salesforce converter - [Apache Polaris Converter](converters/polaris/) — OSI → Apache Polaris converter +- [OrionBelt Converter](converters/orionbelt/) — bidirectional OSI ↔ OrionBelt OBML converter **Related Issues:** diff --git a/converters/orionbelt/README.md b/converters/orionbelt/README.md new file mode 100644 index 00000000..94b3ae04 --- /dev/null +++ b/converters/orionbelt/README.md @@ -0,0 +1,125 @@ +# osi-orionbelt + +Bidirectional converter between **OBML** (OrionBelt Markup Language) semantic +models and **OSI** ([Open Semantic Interchange](https://open-semantic-interchange.org/)), +the open standard for portable semantic models (metrics, dimensions, +relationships). + +This package is licensed under **Apache-2.0** and may be used freely. It is the +OrionBelt converter in the OSI converter ecosystem. The canonical source is +developed in the +[orionbelt-semantic-layer](https://github.com/ralfbecher/orionbelt-semantic-layer) +repository (under `packages/osi-orionbelt`) and published to PyPI from there; +file issues and contributions upstream. + +## Requirements + +- Python 3.12+ +- [uv](https://docs.astral.sh/uv/) (recommended) or pip + +## Install + +```bash +pip install osi-orionbelt +``` + +Optional deep OBML semantic validation (cycles, duplicate names, invalid refs) +via the full OrionBelt engine: + +```bash +pip install "osi-orionbelt[obml-validation]" +``` + +Without that extra, OBML validation runs JSON-schema checks only and emits a +warning for the deeper semantic pass. + +## CLI + +A single `osi-orionbelt` command with two subcommands (mirroring `osi-dbt`): + +| Subcommand | Direction | In | Out | +|---------|-----------|----|----| +| `obml-to-osi` | OBML -> OSI core-spec | OBML YAML | OSI YAML | +| `obml-to-osi --ontology` | OBML -> OSI ontology | OBML YAML | OSI ontology YAML | +| `osi-to-obml` | OSI core-spec -> OBML | OSI YAML | OBML YAML | + +```bash +osi-orionbelt obml-to-osi -i model.obml.yaml -o model.osi.yaml +osi-orionbelt obml-to-osi --ontology -i model.obml.yaml -o model.ontology.yaml +osi-orionbelt osi-to-obml -i model.osi.yaml -o model.obml.yaml +``` + +`-i/--input` and `-o/--output` are required. Each subcommand prints conversion +warnings and a validation summary to stderr, and exits non-zero when the +produced document fails schema validation (unless `--no-validate`). Run +`osi-orionbelt --help` or `osi-orionbelt obml-to-osi --help` for the full +option list. + +## Python API + +```python +import yaml +from osi_orionbelt import OBMLtoOSI, OSItoOBML, validate_osi + +obml = yaml.safe_load(open("model.obml.yaml")) +osi = OBMLtoOSI(obml, "sales", "Sales model").convert() +result = validate_osi(osi) +assert result.valid + +obml_again = OSItoOBML(osi).convert() +``` + +## Vendor extensions + +OSI `custom_extensions` carry vendor-tagged payloads. This converter: + +- emits OrionBelt/OBML-proprietary data under the **`ORIONBELT`** vendor on OBML + to OSI (OBML-only filters, settings, owner, refresh, type info, etc.); +- stashes OSI-native fields that OBML can't represent (unique keys, field + labels, leftover `ai_context`) under the **`OSI`** vendor when going OSI to + OBML, restoring them to first-class OSI fields on the way back; +- **preserves third-party vendor extensions verbatim** (e.g. `SNOWFLAKE`, + `DBT`, `SALESFORCE`, `GOODDATA`) at the model, dataset, field, and + measure/metric levels, so a full OSI to OBML to OSI roundtrip keeps the + original vendor and data. OSI has no separate dimension entity, so an OBML + dimension's foreign extensions surface on its OSI field. + +Legacy `COMMON` / `OBSL` tags from earlier converter versions are still accepted +on read. + +## Limitations / unsupported constructs + +Some OBML constructs have no native OSI equivalent and are carried in vendor +`custom_extensions` (`obml_*` payloads) so they round-trip without loss back to +OBML, but are not interpreted by other OSI consumers: + +- **Many-to-many joins** - represented in OBML join cardinality; flagged on + export. +- **Named secondary join paths** - OBML's multiple join paths between the same + pair of objects are an OBML-specific topology feature. +- **Measures / metrics and column-level value concepts in the ontology layer** - + not represented in the OSI ontology export. +- **OSI metrics with no OBML representation** - a metric whose only expression is + in a non-SQL dialect (`MDX`, `TABLEAU`, `MAQL`), or whose SQL expression cannot + be decomposed into OBML measures/metrics, is **not** dropped: the original OSI + metric is preserved verbatim in a model-level `OSI`-vendor `custom_extension` + (`obml_unconverted_metrics`) and re-emitted on OBML to OSI, so the OSI to OBML + to OSI roundtrip stays lossless. A `LOSSY:` warning is raised for each such + metric because it is **not queryable through OBML**. SQL expressions in the + `ANSI_SQL`, `SNOWFLAKE`, and `DATABRICKS` dialects are all read on import. + +OSI v0.1.x inputs are accepted on read via a legacy normalization shim; output +targets OSI **v0.2.0.dev0**. + +See [`osi_obml_mapping_analysis.md`](./osi_obml_mapping_analysis.md) for the +full OBML <-> OSI core-spec mapping and +[`osi_obml_ontology_mapping_analysis.md`](./osi_obml_ontology_mapping_analysis.md) +for the ontology-layer mapping and its documented gaps. + +## Development + +```bash +uv sync # install +uv run pytest # run the test suite (includes a TPC-DS baseline) +uv run ruff check && uv run mypy src/osi_orionbelt +``` diff --git a/converters/orionbelt/osi_obml_mapping_analysis.md b/converters/orionbelt/osi_obml_mapping_analysis.md new file mode 100644 index 00000000..9c3f6ee3 --- /dev/null +++ b/converters/orionbelt/osi_obml_mapping_analysis.md @@ -0,0 +1,269 @@ +# OSI ↔ OBML Mapping Analysis + +> Bidirectional conversion between [Open Semantic Interchange (OSI)](https://github.com/open-semantic-interchange/OSI) v0.2.0.dev0 and [OrionBelt ML (OBML)](https://github.com/ralfbecher/orionbelt-semantic-layer) v1.0 semantic model formats. OSI v0.1.x inputs are still accepted on read via a legacy normalization shim; output targets v0.2.0.dev0. + +## 1. Structural Comparison + +| Aspect | OSI v0.2.0.dev0 | OBML v1.0 | +|---|---|---| +| **Top-level** | `semantic_model[]` (array of models) | Single model with `dataObjects`, `dimensions`, `measures`, `metrics` sections | +| **Tables / Entities** | `datasets[]` (flat array) | `dataObjects{}` (named dictionary) | +| **Column identifiers** | `fields[].name` (snake_case code) | `columns{}.code` (with display name as dict key) | +| **Expressions** | `expression.dialects[]` per field (multi-dialect) | Single SQL expression via `code` (single dialect) | +| **Joins / Relationships** | `relationships[]` (global, separate section) | `joins[]` (inline on each data object) | +| **Dimensions** | `field.dimension.is_time` (inline flag on fields) | `dimensions{}` (separate top-level section) | +| **Measures** | N/A (merged into metrics) | `measures{}` (explicit aggregation definitions) | +| **Metrics** | `metrics[]` with full SQL expressions | `metrics{}` with `{[Measure]}` references | +| **AI Context** | `ai_context` on every entity | `customExtensions` with `vendor: "OSI"` | +| **Extensibility** | `custom_extensions[]` per entity | `customExtensions[]` per entity | +| **Keys** | `primary_key`, `unique_keys` on datasets | N/A | +| **Secondary joins** | N/A | `secondary: true`, `pathName` | +| **Fan-out protection** | N/A | `allowFanOut`, `reduceToRelationDimensionality` | + +## 2. Key Differences + +### 2.1 Naming Convention + +- **OSI** uses snake_case codes everywhere (`name: "store_sales"`) +- **OBML** supports dual naming — a display name as the dictionary key and a `code` for the physical SQL reference + +During OSI → OBML conversion, field names are used directly as both the display name and code. During OBML → OSI conversion, the `code` value becomes the OSI field `name`. + +### 2.2 Relationship Placement + +- **OSI** defines relationships globally, referencing dataset names by string +- **OBML** defines joins inline on the "from" side data object + +The converter restructures between these two representations automatically, preserving column mappings and generating descriptive relationship names. + +### 2.3 Measures vs. Metrics + +This is the most fundamental structural difference between the two formats. + +- **OSI** has a single "metrics" concept with full SQL expressions (e.g., `SUM(store_sales.ss_ext_sales_price)`) +- **OBML** explicitly separates: + - **Measures**: Simple aggregations on columns (e.g., `SUM` of `ss_ext_sales_price`) + - **Metrics**: Derived calculations referencing measures via `{[Name]}` syntax (e.g., `{[total_sales]} / {[customer_count]}`) + +The converter handles this decomposition automatically: + +| OSI metric type | OBML mapping | +|---|---| +| `AGG(dataset.column)` | Direct measure | +| `AGG(DISTINCT dataset.column)` | Measure with `distinct: true` | +| `AGG(expr)` (e.g., `SUM(a.x * a.y)`) | Expression-based measure | +| Multi-aggregation expression | Auto-generated measures + metric formula | + +**Example** — OSI metric `customer_lifetime_value`: +```yaml +# OSI +expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) +``` + +is decomposed into OBML: +```yaml +# OBML measures (auto-generated) +measures: + total_sales: + columns: + - dataObject: store_sales + column: ss_ext_sales_price + resultType: float + aggregation: sum + + _customer_c_customer_sk_count_distinct: + columns: + - dataObject: customer + column: c_customer_sk + resultType: float + aggregation: count + distinct: true + +# OBML metric (references the measures) +metrics: + customer_lifetime_value: + expression: "{[total_sales]} / {[_customer_c_customer_sk_count_distinct]}" +``` + +When a simple OSI metric (e.g., `SUM(store_sales.ss_ext_sales_price)`) is equivalent to an existing named measure, the converter deduplicates and reuses the named measure rather than creating a redundant auto-measure. + +### 2.4 AI Context Preservation + +OSI's `ai_context` (instructions, synonyms, examples) is preserved losslessly during conversion via OBML's `customExtensions` mechanism: + +```yaml +# OSI input +ai_context: + synonyms: + - "sales transactions" + - "store purchases" + +# OBML output (via customExtensions) +customExtensions: + - vendor: OSI + data: '{"synonyms": ["sales transactions", "store purchases"]}' +``` + +This applies at all levels: datasets → data objects, fields → columns, and model-level `ai_context`. + +During OBML → OSI conversion, the `customExtensions` with `vendor: "OSI"` are read back and restored as native `ai_context` on the OSI side. + +### 2.5 OBML-Specific Features (Not Representable in OSI) + +These OBML features have no direct OSI equivalent. Where possible, metadata is preserved in OSI `ai_context` or `custom_extensions` (with `vendor_name: "COMMON"` and `obml_`-prefixed keys) for lossless roundtrip: + +- Secondary joins with `pathName` (preserved in relationship `ai_context`) +- `allowFanOut` — preserved in metric `custom_extensions` (`obml_allow_fan_out`) +- Dynamic date filters (`dynamicDate`, `dynamicDateRange`) — not yet preserved +- `timeGrain` on dimensions — preserved in field `custom_extensions` (`obml_time_grain`) +- Dimension `format` — preserved in field `custom_extensions` (`obml_dimension_format`) +- Measure filters — preserved in metric `custom_extensions` (`obml_filters`) +- Measure `total` — preserved in metric `custom_extensions` (`obml_total`) +- Measure `format` — preserved in metric `custom_extensions` (`obml_format`) +- Measure `delimiter` — preserved in metric `custom_extensions` (`obml_delimiter`) +- Measure `withinGroup` — preserved in metric `custom_extensions` (`obml_within_group`) +- Metric `format` — preserved in metric `custom_extensions` (`obml_format`) +- Locale settings — not yet preserved +- `abstractType` (OBML type system) — preserved in field `custom_extensions` (`obml_abstract_type`) + +### 2.6 OSI-Specific Features and How They Map to OBML + +- **`primary_key`** — natively represented: OSI's dataset-level `primary_key` array maps to per-column `primaryKey: true` on OBML columns (`DataObjectColumn.primaryKey`), and back to the dataset array on export. +- **`unique_keys`** — no native OBML equivalent; round-trips via an `OSI`-vendor `customExtension` (`obml_unique_keys`). +- **Multi-dialect expressions** — on import the converter reads the first available SQL dialect in the order `ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`; non-SQL dialects (`MDX`, `TABLEAU`, `MAQL`) are not parsed. A metric with no SQL-parseable dialect, or an expression OBML cannot decompose, is preserved verbatim (`obml_unconverted_metrics`) with a `LOSSY:` warning rather than dropped. On export, OBML measures/metrics emit `ANSI_SQL`. +- **`ai_context`** — preserved losslessly via `customExtensions` (see Section 2.4). +- **`custom_extensions`** — mapped to OBML `customExtensions`. + +## 3. Conversion Strategies + +### 3.1 OSI → OBML + +1. Parse `source` string to extract `database`, `schema`, and `table` +2. Convert fields to columns with type inference (heuristic-based `abstractType`) +3. Restructure global relationships into inline joins on data objects +4. Decompose metric SQL expressions into OBML measures + metrics +5. Extract dimension-flagged fields into the top-level `dimensions` section (excluding FK/PK join keys) +6. Preserve `ai_context` losslessly via `customExtensions` (vendor: `"OSI"`) + +### 3.2 OBML → OSI + +1. Combine `database.schema.code` into the OSI `source` string +2. Convert columns to fields with `ANSI_SQL` dialect expressions +3. Extract inline joins into global relationships with generated names +4. Convert measures to OSI metrics with SQL expressions +5. Expand metric templates by substituting measure SQL into `{[Name]}` references +6. Map OBML dimension metadata into `field.dimension.is_time` flags +7. Preserve secondary join info in relationship `ai_context` +8. Store OBML-specific type info in `custom_extensions` with `vendor_name: "COMMON"` + +## 4. Validation + +The converter includes dual-layer validation for both formats, ensuring that converted output is structurally and semantically correct. + +### 4.1 OBML Validation + +1. **JSON Schema** — validates against `schema/obml-schema.json` (Draft 7) +2. **Semantic** — runs OrionBelt's `ReferenceResolver` + `SemanticValidator` (reference integrity, cycle detection, multipath detection, duplicate identifiers) + +### 4.2 OSI Validation + +1. **JSON Schema** — validates against `osi-schema.json` (Draft 2020-12) +2. **Unique names** — checks uniqueness of dataset, field, metric, and relationship names +3. **References** — verifies that relationship `from`/`to` reference existing datasets + +Validation runs automatically after each conversion. Use `--no-validate` to skip. + +## 5. Converter Usage + +### CLI + +A single `osi-orionbelt` command with two subcommands is installed with the package: + +```bash +# OSI → OBML +osi-orionbelt osi-to-obml -i tpcds_osi.yaml -o tpcds_as_obml.yaml + +# OBML → OSI +osi-orionbelt obml-to-osi -i tpcds_as_obml.yaml -o tpcds_obml_as_osi.yaml \ + --model-name tpcds_retail_model \ + --description "TPC-DS retail semantic model" + +# OBML → OSI ontology document +osi-orionbelt obml-to-osi --ontology -i tpcds_as_obml.yaml -o tpcds_ontology.yaml + +# Skip validation +osi-orionbelt osi-to-obml -i input.yaml -o output.yaml --no-validate +``` + +### CLI Options + +| Subcommand / Option | Description | +|---|---| +| `osi-to-obml` | Convert OSI → OBML | +| `obml-to-osi` | Convert OBML → OSI | +| `--ontology` | (`obml-to-osi`) emit an OSI ontology document instead of core-spec | +| `-i`, `--input` | Input file (required) | +| `-o`, `--output` | Output file (required) | +| `--model-name` | Model name for OBML → OSI | +| `--description` | Model description for OBML → OSI | +| `--ai-instructions` | AI instructions for OBML → OSI | +| `--database` | Default database for OSI → OBML (default: `ANALYTICS`) | +| `--schema` | Default schema for OSI → OBML (default: `PUBLIC`) | +| `--no-validate` | Skip post-conversion validation | + +### Python API + +```python +from osi_orionbelt import OSItoOBML, OBMLtoOSI, validate_obml, validate_osi + +# OSI → OBML +converter = OSItoOBML(osi_dict) +obml = converter.convert() +result = validate_obml(obml) +assert result.valid + +# OBML → OSI +converter = OBMLtoOSI(obml_dict, model_name="my_model") +osi = converter.convert() +result = validate_osi(osi) +assert result.valid +``` + +## 6. Example: TPC-DS Roundtrip + +The converter is validated against the official [TPC-DS example](https://github.com/open-semantic-interchange/OSI/blob/main/examples/tpcds_semantic_model.yaml) from the OSI repository. That file is vendored at `tests/fixtures/tpcds_semantic_model.yaml` and exercised by `tests/test_osi_tpcds_baseline.py`, which runs the OSI converters guide's [conceptual conversion flow](https://github.com/open-semantic-interchange/OSI/blob/main/converters/index.md#example-conceptual-conversion-flow) end to end: OSI to OBML to OSI, asserting validity at each step and that the example's `SALESFORCE` and `DBT` custom extensions survive the round-trip (step 7). + +### OSI → OBML + +The TPC-DS OSI model with 5 datasets, 4 relationships, and 5 metrics converts cleanly to OBML: + +- 5 data objects with inline joins +- 16 dimensions (FK/PK join keys excluded) +- 5 measures (3 direct + 2 auto-generated for metric decomposition) +- 2 metrics (composite expressions referencing measures) +- All `ai_context` synonyms preserved via `customExtensions` + +### OBML → OSI (Roundtrip) + +Converting the OBML output back to OSI produces a valid OSI model where: + +- All `ai_context` synonyms are restored from `customExtensions` +- Measures are re-expanded into SQL metric expressions +- Inline joins are extracted back into global relationships +- OBML type information is preserved in `custom_extensions` + +### Files + +| File | Description | +|---|---| +| `tests/fixtures/tpcds_osi.yaml` | Official TPC-DS OSI example (from OSI repo) | +| `tests/fixtures/tpcds_as_obml.yaml` | Converted OBML output | +| `src/osi_orionbelt/schemas/osi-schema.json` | OSI JSON Schema (Draft 2020-12, from OSI repo) | +| `src/osi_orionbelt/converter.py` | Bidirectional converter with validation | + +## 7. Future Considerations + +- **MCP/API integration** — Expose OSI import/export as OrionBelt MCP tools or REST API endpoints +- **Multi-dialect support** — Preserve non-ANSI dialect expressions during roundtrip +- **Primary keys in OBML** — Add optional `primary_key` to data objects for richer metadata +- **Vendor enum** — Register `"ORIONBELT"` as an OSI vendor for OBML-specific extensions diff --git a/converters/orionbelt/osi_obml_ontology_mapping_analysis.md b/converters/orionbelt/osi_obml_ontology_mapping_analysis.md new file mode 100644 index 00000000..1887b850 --- /dev/null +++ b/converters/orionbelt/osi_obml_ontology_mapping_analysis.md @@ -0,0 +1,82 @@ +# OBML → OSI Ontology Mapping Analysis + +This pins the rules used by `OBMLtoOSIOntology` to derive an **OSI ontology +document** (validated against `src/osi_orionbelt/schemas/osi-ontology-schema.json`, OSI version +`0.2.0.dev0`) from an OBML semantic model. + +The OSI ontology is a **separate document** from the OSI core-spec semantic +model (different `$id`, different required root). It is produced alongside the +core export, never merged into it — OSI's own `validation/validate.py` validates +one document against one schema and reads only the first YAML document, so a +combined or multi-doc file is not portable. See the `include_ontology` flag on +the export endpoints. + +## Document shape produced + +```yaml +version: 0.2.0.dev0 +name: # required +description: # optional +ai_context: { instructions: ... } # optional, from ai_instructions / model +ontology: # required, minItems 1 + - concept: + name: # = OBML dataObject display name + type: EntityType + description: ... + relationships: # outgoing joins keyed by this entity + - name: _to_ + roles: [{ concept: }] # declaring concept (A) is the implicit first role + multiplicity: ManyToOne | OneToOne + verbalizes: ["{} relates to {}"] +ontology_mappings: + - name: _map + semantic_model: { ...full OSI core-spec model... } # reused from OBMLtoOSI + concept_mappings: + - concept: + object_mappings: [{ expression: "." }] + link_mappings: + - relationship: _to_ + object_mapping: { concept: , expression: "." } +``` + +## Mapping rules + +| OBML construct | OSI ontology target | Notes | +|----------------|---------------------|-------| +| `dataObject` | `EntityType` `Concept` (one per object) | name = display name (matches OSI dataset name for ref consistency) | +| `dataObject.description` / `.comment` | `concept.description` | first non-empty wins | +| `join` (A → B) | `Relationship` under A's component | declaring concept A is the implicit first role; B is an explicit `role` | +| `join.joinType` | `Relationship.multiplicity` | `many-to-one`→`ManyToOne`, `one-to-one`→`OneToOne` | +| `column.primaryKey` | entity `object_mappings[].expression` | `
.`; identifies the entity | +| `join.columnsFrom` (FK) | `link_mappings[].object_mapping.expression` | `.`; binds the relationship to its far role | +| whole core model | `ontology_mappings[].semantic_model` | embedded verbatim from `OBMLtoOSI.convert()` | + +`
` is the final identifier of the dataset `source` (e.g. `db.schema.t` → `t`), +falling back to the dataset name when `source` has no dotted physical table. + +## Gaps and warnings (emitted to `warnings`) + +| OBML construct | Handling | Reason | +|----------------|----------|--------| +| `joinType: many-to-many` | relationship **skipped** + warning | OSI `Multiplicity` enum is only `ManyToOne`/`OneToOne` | +| missing/unknown `joinType` | defaults to `ManyToOne` + warning | matches core-converter default | +| secondary / `pathName` joins | emitted as ordinary relationships + warning | OSI ontology has no named-alternate-path concept | +| composite primary/foreign keys | first column used + warning | `object_mapping.expression` is a single scalar SQL expression | +| measures / metrics | **not** in the ontology layer | live only in the embedded core `semantic_model`; ontology models entities/relationships | +| columns (non-key) as value concepts | not modeled (entities only) | keeps v1 valid and focused; ORM-style `ValueType` modeling deferred | +| `verbalizes` / `derived_by` / `requires` / `identify_by` | `verbalizes` emitted as a generated stub; others omitted | OBML has no native source for fact-based verbalization or derivation | + +## Validation + +`validate_osi_ontology()` runs: +1. JSON Schema (Draft 2020-12) against `osi-ontology-schema.json`. +2. Semantic checks: unique concept names; relationship `roles` reference defined + concepts; `concept_mappings` reference defined concepts. + +## Stability note + +OSI ontology is `0.2.0.dev0` (pre-1.0). This exporter is the supported, +schema-validated direction. An ontology **importer** (OSI ontology → OBML) is +intentionally deferred until OSI drops the `dev` pre-release suffix, because the +gaps above make the reverse direction lossy. Import the OSI **core spec** +instead (already supported). diff --git a/converters/orionbelt/pyproject.toml b/converters/orionbelt/pyproject.toml new file mode 100644 index 00000000..7aa309ca --- /dev/null +++ b/converters/orionbelt/pyproject.toml @@ -0,0 +1,58 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "osi-orionbelt" +version = "0.1.0" +description = "Bidirectional OBML <-> OSI (Open Semantic Interchange) converter for OrionBelt semantic models" +license = { text = "Apache-2.0" } +readme = "README.md" +authors = [{ name = "Ralf Becher", email = "info@orionbelt.ai" }] +keywords = ["osi", "open-semantic-interchange", "obml", "semantic-layer", "orionbelt", "converter"] +requires-python = ">=3.12" +dependencies = [ + "pyyaml>=6.0", + "jsonschema>=4.18", + "referencing>=0.30", +] + +[project.optional-dependencies] +# Deep OBML semantic validation (cycles, duplicate names, invalid refs) via the +# full OrionBelt engine. Optional: validate_obml degrades to JSON-schema-only +# checks with a warning when this is not installed. +obml-validation = ["orionbelt-semantic-layer"] +dev = [ + "pytest>=8.0", + "mypy>=1.10", + "ruff>=0.4", + "types-jsonschema", + "types-PyYAML", +] + +[project.scripts] +osi-orionbelt = "osi_orionbelt.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/osi_orionbelt"] + +# All three schemas (the two OSI artefacts plus a vendored snapshot of the +# canonical OBML schema) live in src/osi_orionbelt/schemas/ as tracked files, +# so the sdist and wheel are self-contained and build in isolation. The OBML +# snapshot is kept in sync with the repo-root schema/obml-schema.json by a +# drift-guard test (tests/unit/test_osi_orionbelt_schema_sync.py). + +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "N", "UP", "B", "A", "SIM"] + +[tool.mypy] +python_version = "3.12" +files = ["src/osi_orionbelt"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] diff --git a/converters/orionbelt/src/osi_orionbelt/__init__.py b/converters/orionbelt/src/osi_orionbelt/__init__.py new file mode 100644 index 00000000..9bccf42c --- /dev/null +++ b/converters/orionbelt/src/osi_orionbelt/__init__.py @@ -0,0 +1,41 @@ +"""osi-orionbelt: bidirectional OBML <-> OSI converter. + +Converts between OrionBelt Markup Language (OBML) semantic models and Open +Semantic Interchange (OSI) models, in both directions, plus an OSI ontology +emitter. Validation helpers check OBML and OSI documents against their JSON +schemas. + +Public API: + OSItoOBML - convert an OSI model dict to OBML + OBMLtoOSI - convert an OBML model dict to OSI core-spec + OBMLtoOSIOntology - emit an OSI ontology document from an OBML model + validate_obml - validate an OBML model dict + validate_osi - validate an OSI model dict + validate_osi_ontology - validate an OSI ontology document dict + ValidationResult - structured validation result +""" + +from __future__ import annotations + +from osi_orionbelt.converter import ( + OBMLtoOSI, + OBMLtoOSIOntology, + OSItoOBML, + ValidationResult, + validate_obml, + validate_osi, + validate_osi_ontology, +) + +__version__ = "0.1.0" + +__all__ = [ + "OBMLtoOSI", + "OBMLtoOSIOntology", + "OSItoOBML", + "ValidationResult", + "validate_obml", + "validate_osi", + "validate_osi_ontology", + "__version__", +] diff --git a/converters/orionbelt/src/osi_orionbelt/cli.py b/converters/orionbelt/src/osi_orionbelt/cli.py new file mode 100644 index 00000000..351d0649 --- /dev/null +++ b/converters/orionbelt/src/osi_orionbelt/cli.py @@ -0,0 +1,155 @@ +"""Command-line entry point for the OBML <-> OSI converter. + +A single ``osi-orionbelt`` command with two format-named subcommands, mirroring +the OSI converter convention (e.g. ``osi-dbt msi-to-osi``): + + osi-orionbelt obml-to-osi -i model.obml.yaml -o model.osi.yaml + osi-orionbelt obml-to-osi --ontology -i model.obml.yaml -o model.ontology.yaml + osi-orionbelt osi-to-obml -i model.osi.yaml -o model.obml.yaml + +Both subcommands print conversion warnings and a validation summary to stderr, +and exit non-zero when the produced document fails schema validation (unless +``--no-validate``). +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path +from typing import Any + +import yaml + +from osi_orionbelt.converter import ( + OBMLtoOSI, + OBMLtoOSIOntology, + OSItoOBML, + validate_obml, + validate_osi, + validate_osi_ontology, +) + + +def _load(input_path: str) -> dict[str, Any]: + data = yaml.safe_load(Path(input_path).read_text()) + if not isinstance(data, dict): + print(f"Error: {input_path} is not a YAML mapping", file=sys.stderr) + raise SystemExit(2) + return data + + +def _emit(result: dict[str, Any], output: str) -> None: + output_yaml = yaml.dump( + result, default_flow_style=False, allow_unicode=True, sort_keys=False, width=120 + ) + Path(output).write_text(output_yaml) + print(f"Written to {output}", file=sys.stderr) + + +def _print_warnings(warnings: list[str]) -> None: + if warnings: + print("\nConversion warnings:", file=sys.stderr) + for w in warnings: + print(f" - {w}", file=sys.stderr) + + +def _report_validation(label: str, result: dict[str, Any], validate_fn: Any) -> bool: + """Validate ``result`` and print a summary. Return True if there are errors.""" + print(f"\nValidating {label}...", file=sys.stderr) + vr = validate_fn(result) + for line in vr.summary_lines(): + print(line, file=sys.stderr) + if vr.valid: + print(f"{label} is valid", file=sys.stderr) + return False + print(f"{label} has validation errors", file=sys.stderr) + return True + + +def _cmd_obml_to_osi(args: argparse.Namespace) -> int: + """OBML -> OSI core-spec (or, with --ontology, OSI ontology).""" + data = _load(args.input) + + validate_fn: Any + if args.ontology: + converter: Any = OBMLtoOSIOntology( + data, args.model_name, args.description, args.ai_instructions + ) + result = converter.convert() + validate_fn, label = validate_osi_ontology, "OSI ontology output" + else: + converter = OBMLtoOSI(data, args.model_name, args.description, args.ai_instructions) + result = converter.convert() + validate_fn, label = validate_osi, "OSI output" + + _emit(result, args.output) + _print_warnings(converter.warnings) + + if args.no_validate: + return 0 + return 1 if _report_validation(label, result, validate_fn) else 0 + + +def _cmd_osi_to_obml(args: argparse.Namespace) -> int: + """OSI core-spec -> OBML.""" + data = _load(args.input) + + converter = OSItoOBML(data, args.database, args.schema) + result = converter.convert() + + _emit(result, args.output) + _print_warnings(converter.warnings) + + if args.no_validate: + return 0 + return 1 if _report_validation("OBML output", result, validate_obml) else 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="osi-orionbelt", + description="Convert between OrionBelt OBML and OSI YAML.", + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + o2s = subparsers.add_parser("obml-to-osi", help="Convert OBML YAML → OSI YAML") + o2s.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to OBML YAML") + o2s.add_argument( + "-o", "--output", required=True, metavar="FILE", help="Path for output OSI YAML" + ) + o2s.add_argument( + "--ontology", action="store_true", help="Emit an OSI ontology document instead of core-spec" + ) + o2s.add_argument( + "--model-name", default="semantic_model", metavar="NAME", help="OSI semantic model name" + ) + o2s.add_argument("--description", default="", metavar="TEXT", help="OSI model description") + o2s.add_argument( + "--ai-instructions", default="", metavar="TEXT", help="OSI ai_context instructions" + ) + o2s.add_argument("--no-validate", action="store_true", help="Skip output validation") + + s2o = subparsers.add_parser("osi-to-obml", help="Convert OSI YAML → OBML YAML") + s2o.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to OSI YAML") + s2o.add_argument( + "-o", "--output", required=True, metavar="FILE", help="Path for output OBML YAML" + ) + s2o.add_argument( + "--database", default="ANALYTICS", metavar="NAME", help="Default database for OBML output" + ) + s2o.add_argument( + "--schema", default="PUBLIC", metavar="NAME", help="Default schema for OBML output" + ) + s2o.add_argument("--no-validate", action="store_true", help="Skip output validation") + + args = parser.parse_args(argv) + if args.command == "obml-to-osi": + return _cmd_obml_to_osi(args) + if args.command == "osi-to-obml": + return _cmd_osi_to_obml(args) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/converters/orionbelt/src/osi_orionbelt/converter.py b/converters/orionbelt/src/osi_orionbelt/converter.py new file mode 100644 index 00000000..6ecf64d2 --- /dev/null +++ b/converters/orionbelt/src/osi_orionbelt/converter.py @@ -0,0 +1,2967 @@ +#!/usr/bin/env python3 +""" +OSI ↔ OBML Bidirectional Converter +=================================== +Converts between Open Semantic Interchange (OSI v0.2.0.dev0) YAML models +and OrionBelt Markup Language (OBML v1.0) YAML models. + +OSI v0.1.1 inputs are still accepted on read — the legacy shim +``_normalize_legacy_v01`` promotes pre-v0.2 custom_extensions into the +v0.2 first-class fields before regular parsing runs. + +Author: OrionBelt / RALFORION +""" + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Any + +import yaml + +# ─── Spec version pin ─────────────────────────────────────────────────────── +# Single source of truth for the OSI spec we emit. Bump when upstream cuts +# a stable v0.2.0 (drop the ``.dev0`` suffix). All read paths accept both +# 0.1.x (via the legacy shim) and 0.2.x. +_OSI_VERSION = "0.2.0.dev0" + +# Dialect / vendor enum extras new in v0.2.0.dev0 +_OSI_KNOWN_DIALECTS = ("ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL") +# SQL dialects (of the OSI enum) whose aggregation expressions our regex-based +# metric parser can read, in preference order. ANSI_SQL first; SNOWFLAKE and +# DATABRICKS are SQL engines OrionBelt also targets, and their simple/expression +# aggregations (``SUM(t.c)``, ``SUM(t.a * t.b)``) are syntactically identical to +# ANSI. MDX / TABLEAU / MAQL are non-SQL languages and are never parsed as SQL. +_SQL_PARSEABLE_DIALECTS = ("ANSI_SQL", "SNOWFLAKE", "DATABRICKS") + +# Matches a ``dataset.column`` reference inside a SQL expression, where each +# side is a bare identifier or a quoted identifier (double quotes, backticks, or +# brackets). The leading lookbehind prevents matching the tail of a longer path +# (``a.b.c``) or a mid-token boundary; the bare form must start with a letter or +# underscore so numeric literals (``1.5``) are never treated as references. +_COLUMN_REF_RE = re.compile( + r'(?[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' + r"\s*\.\s*" + r'(?P[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' +) +_OSI_KNOWN_VENDORS = ( + "COMMON", + "ORIONBELT", + "SNOWFLAKE", + "SALESFORCE", + "DBT", + "DATABRICKS", + "GOODDATA", +) + +# Vendor identities for custom_extensions. +# ORIONBELT - OrionBelt/OBML-proprietary payloads we author on OBML -> OSI. +# OSI - OSI-native fields OBML can't hold (unique_keys, field label, +# ai_context leftovers), stashed into OBML on OSI -> OBML. +# Read paths also accept the legacy tags we emitted before this scheme so older +# documents still round-trip; foreign vendors (SNOWFLAKE, DBT, ...) are +# preserved verbatim, never relabelled. +_VENDOR_OBML = "ORIONBELT" +_VENDOR_OSI = "OSI" +_OBML_VENDOR_READ = ("ORIONBELT", "COMMON") +_OSI_VENDOR_READ = ("OSI", "OBSL") +# Vendors the converter handles internally (its own payloads + native-field +# stashes). Any custom_extension from a vendor outside this set is third-party +# and is carried through verbatim in both directions, never relabelled. +_INTERNAL_VENDORS = frozenset({"ORIONBELT", "COMMON", "OSI", "OBSL"}) + +# ─── Type mapping ─────────────────────────────────────────────────────────── + +OBML_TO_OSI_TYPE = { + "string": "string", + "json": "string", + "int": "integer", + "float": "number", + "date": "date", + "time": "time", + "time_tz": "time", + "timestamp": "timestamp", + "timestamp_tz": "timestamp", + "boolean": "boolean", +} + +OSI_TO_OBML_TYPE = { + "string": "string", + "integer": "int", + "number": "float", + "date": "date", + "time": "time", + "timestamp": "timestamp", + "boolean": "boolean", +} + + +# ═══════════════════════════════════════════════════════════════════════════ +# OSI → OBML Converter +# ═══════════════════════════════════════════════════════════════════════════ + + +class OSItoOBML: + """Convert an OSI semantic model YAML to OBML format.""" + + def __init__( + self, osi: dict, default_database: str = "ANALYTICS", default_schema: str = "PUBLIC" + ): + self.osi = osi + self.default_database = default_database + self.default_schema = default_schema + self.warnings: list[str] = [] + # OSI metrics that have no OBML representation (non-SQL dialect only, + # or an expression our parser cannot decompose). Preserved verbatim + # rather than dropped — see ``_preserve_unconverted_metric``. + self._unconverted_metrics: list[dict] = [] + + def _normalize_legacy_v01(self) -> None: + """Promote OSI v0.1.x payloads to the v0.2 shape, in place. + + The v0.2 spec promotes ``primary_key`` and ``unique_keys`` to + first-class dataset fields. v0.1.x serializers (including ours + pre-bump) stash both under ``custom_extensions`` with vendor + ``OBSL`` and keys ``obml_primary_key`` / ``obml_unique_keys``. + This shim runs before parsing so the rest of the converter can + assume v0.2 shape regardless of input version. + + No-op for documents that already declare ``version`` >= 0.2 or + that have nothing to migrate. + """ + version = str(self.osi.get("version", "")) + if version and not version.startswith(("0.1", "0.0")): + return # already v0.2+ (or future) — nothing to do + + models = self.osi.get("semantic_model", []) + if not isinstance(models, list): + return + + for model in models: + for ds in model.get("datasets", []) or []: + # Promote legacy primary_key / unique_keys from OBSL extras + # only if the dataset doesn't already declare them. + legacy = self._extract_obml_extras(ds) + if not legacy: + continue + if "primary_key" not in ds and legacy.get("obml_primary_key"): + pk = legacy["obml_primary_key"] + if isinstance(pk, list) and all(isinstance(c, str) for c in pk): + ds["primary_key"] = list(pk) + if "unique_keys" not in ds and legacy.get("obml_unique_keys"): + uk = legacy["obml_unique_keys"] + if isinstance(uk, list) and all( + isinstance(g, list) and all(isinstance(c, str) for c in g) for g in uk + ): + ds["unique_keys"] = [list(g) for g in uk] + + if version.startswith(("0.0", "0.1")): + self.warnings.append( + f"OSI input declares version '{version}'; legacy v0.1.x " + f"compatibility shim applied. Output target is v{_OSI_VERSION}." + ) + + def convert(self) -> dict: + # Reset per-conversion accumulators so calling convert() twice on the + # same instance is idempotent (no duplicated warnings or preserved + # metrics). Both are populated as a side effect of conversion below. + self.warnings = [] + self._unconverted_metrics = [] + + # v0.1.x inputs need the legacy shim to promote pre-v0.2 + # custom_extensions into v0.2 first-class fields before we parse. + self._normalize_legacy_v01() + + models = self.osi.get("semantic_model", []) + if not models: + raise ValueError("No semantic_model found in OSI input") + + # Take the first semantic model (OBML is a single-model format) + model = models[0] + if len(models) > 1: + self.warnings.append( + f"OSI contains {len(models)} semantic models; " + f"only the first ('{model.get('name')}') is converted." + ) + + obml: dict[str, Any] = {"version": 1.0} + + # ── Model description ───────────────────────────────────── + if model.get("description"): + obml["description"] = model["description"] + + # ── DataObjects ───────────────────────────────────────────── + datasets = model.get("datasets", []) + relationships = model.get("relationships", []) + + # Build lookup: dataset_name → dataset + ds_map = {ds["name"]: ds for ds in datasets} + + # Build relationship index: from_dataset → [relationship, ...] + rel_by_from: dict[str, list] = {} + for rel in relationships: + rel_by_from.setdefault(rel["from"], []).append(rel) + + # Collect join key columns: (dataset_name, field_name) pairs + # These should NOT become dimensions (they are FK/PK join keys) + self._join_key_columns: set[tuple[str, str]] = set() + for rel in relationships: + for col in rel.get("from_columns", []): + self._join_key_columns.add((rel["from"], col)) + for col in rel.get("to_columns", []): + self._join_key_columns.add((rel["to"], col)) + + data_objects: dict[str, Any] = {} + for ds in datasets: + do_name, do_obj = self._convert_dataset(ds, rel_by_from) + data_objects[do_name] = do_obj + + obml["dataObjects"] = data_objects + + # ── Dimensions (extracted from OSI fields with dimension metadata) ── + dimensions = self._extract_dimensions(datasets) + if dimensions: + obml["dimensions"] = dimensions + + # ── Measures & Metrics ────────────────────────────────────── + osi_metrics = model.get("metrics", []) + measures, metrics = self._convert_metrics(osi_metrics, ds_map) + if measures: + obml["measures"] = measures + if metrics: + obml["metrics"] = metrics + + # Metrics that have no OBML representation are not dropped: stash the + # original OSI metric verbatim under the OSI vendor so the reverse + # (OBML -> OSI) direction re-emits them and a full OSI -> OBML -> OSI + # roundtrip stays lossless. They are not queryable in OBML; a LOSSY + # warning was already recorded per metric. + if self._unconverted_metrics: + obml.setdefault("customExtensions", []).append( + { + "vendor": _VENDOR_OSI, + "data": json.dumps({"obml_unconverted_metrics": self._unconverted_metrics}), + } + ) + + # ── Restore model-level properties from custom_extensions ──── + for ext in model.get("custom_extensions", []): + if ext.get("vendor_name") in _OBML_VENDOR_READ: + try: + ext_data = json.loads(ext.get("data", "{}")) + if ext_data.get("obml_filters"): + obml["filters"] = ext_data["obml_filters"] + if ext_data.get("obml_settings"): + obml["settings"] = ext_data["obml_settings"] + if ext_data.get("obml_owner"): + obml["owner"] = ext_data["obml_owner"] + except (json.JSONDecodeError, TypeError): + pass + break + + # Preserve third-party vendor extensions verbatim + self._carry_foreign_extensions(model.get("custom_extensions"), obml) + + return obml + + @staticmethod + def _carry_foreign_extensions(osi_exts: list[dict] | None, obml_target: dict[str, Any]) -> None: + """Carry third-party OSI custom_extensions verbatim into OBML. + + Our own payloads and OSI-native stashes are reconstructed elsewhere; + any other vendor's extension is preserved unchanged on the OBML side + so a full OSI -> OBML -> OSI roundtrip keeps the original vendor. + """ + for ext in osi_exts or []: + vendor = ext.get("vendor_name") + if vendor and vendor not in _INTERNAL_VENDORS: + obml_target.setdefault("customExtensions", []).append( + {"vendor": vendor, "data": ext.get("data", "")} + ) + + def _parse_source(self, source: str) -> tuple[str, str, str]: + """Parse 'database.schema.table' into parts.""" + parts = source.split(".") + if len(parts) == 3: + return parts[0], parts[1], parts[2] + elif len(parts) == 2: + return self.default_database, parts[0], parts[1] + else: + return self.default_database, self.default_schema, parts[0] + + def _convert_dataset(self, ds: dict, rel_by_from: dict) -> tuple[str, dict]: + """Convert an OSI dataset to an OBML dataObject. + + Uses the exact OSI dataset name as the OBML data object key. + """ + name = ds["name"] + + source = ds.get("source", name) + database, schema, table = self._parse_source(source) + + do: dict[str, Any] = { + "code": table, + "database": database, + "schema": schema, + } + + # ── Columns ───────────────────────────────────────────────── + columns: dict[str, Any] = {} + fields = ds.get("fields", []) + for field in fields: + col_name, col_obj = self._convert_field(field) + columns[col_name] = col_obj + + # ── Primary key flag propagation (OSI v0.2 first-class) ── + # ``primary_key`` lists physical column codes; mark every matching + # column with ``primaryKey: true``. Unknown PK columns surface as + # a warning (the spec couples PK to relationship cardinality, so + # silently dropping is unsafe). + pk_codes = ds.get("primary_key") or [] + if pk_codes: + code_to_col = {col.get("code"): (cname, col) for cname, col in columns.items()} + unknown_pks: list[str] = [] + for pk_code in pk_codes: + hit = code_to_col.get(pk_code) + if hit is None: + unknown_pks.append(pk_code) + continue + _, col = hit + col["primaryKey"] = True + if unknown_pks: + self.warnings.append( + f"Dataset '{name}' primary_key references unknown columns: " + f"{unknown_pks}. Ignored." + ) + + if columns: + do["columns"] = columns + else: + self.warnings.append(f"Dataset '{name}' has no fields; adding placeholder column.") + do["columns"] = {f"{name}_id": {"code": f"{table}_id", "abstractType": "string"}} + + # ── Joins (from relationships where this dataset is on 'from' side) ── + joins = [] + for rel in rel_by_from.get(name, []): + join_obj = self._convert_relationship_to_join(rel) + joins.append(join_obj) + + if joins: + do["joins"] = joins + + # ── Description (semantic, from OSI) ───────────────────────── + if ds.get("description"): + do["description"] = ds["description"] + + # ── Extract ai_context: synonyms → native, rest → customExtensions ─ + ai_ctx = ds.get("ai_context") + if ai_ctx: + ai_data = ai_ctx if isinstance(ai_ctx, dict) else {"instructions": ai_ctx} + # Extract synonyms directly into OBML synonyms property + if "synonyms" in ai_data: + do["synonyms"] = list(ai_data["synonyms"]) + # Store remaining ai_context keys in customExtensions + remaining = {k: v for k, v in ai_data.items() if k != "synonyms"} + if remaining: + do["customExtensions"] = [ + { + "vendor": "OSI", + "data": json.dumps(remaining), + } + ] + + # Restore DataObject owner / comment / refresh from custom_extensions + for ext in ds.get("custom_extensions", []): + if ext.get("vendor_name") in _OBML_VENDOR_READ: + try: + ext_data = json.loads(ext.get("data", "{}")) + if ext_data.get("obml_owner"): + do["owner"] = ext_data["obml_owner"] + if ext_data.get("obml_comment"): + do["comment"] = ext_data["obml_comment"] + if ext_data.get("obml_refresh"): + do["refresh"] = ext_data["obml_refresh"] + except (json.JSONDecodeError, TypeError): + pass + break + + # ── Unique keys roundtrip (OBML has no native concept) ── + # Persist the OSI ``unique_keys`` array into the OBSL-vendor + # customExtensions so the OBML → OSI direction can emit it back. + unique_keys = ds.get("unique_keys") or [] + if unique_keys: + do.setdefault("customExtensions", []).append( + { + "vendor": _VENDOR_OSI, + "data": json.dumps({"obml_unique_keys": [list(g) for g in unique_keys]}), + } + ) + + # Preserve third-party vendor extensions verbatim + self._carry_foreign_extensions(ds.get("custom_extensions"), do) + + return name, do + + def _convert_field(self, field: dict) -> tuple[str, dict]: + """Convert an OSI field to an OBML column. + + Uses the exact OSI field name as the OBML column key. + """ + name = field["name"] + + # Get expression (prefer ANSI_SQL dialect) + expr_obj = field.get("expression", {}) + code = name # fallback + if isinstance(expr_obj, dict): + dialects = expr_obj.get("dialects", []) + for d in dialects: + if d.get("dialect") == "ANSI_SQL": + code = d.get("expression", name) + break + if not dialects: + code = name + elif code == name and dialects: + code = dialects[0].get("expression", name) + + # Determine abstract type: prefer explicit data_type, fall back to heuristic + osi_type = field.get("data_type", "") + if osi_type and osi_type in OSI_TO_OBML_TYPE: + abstract_type = OSI_TO_OBML_TYPE[osi_type] + else: + abstract_type = self._infer_obml_type(field) + + col: dict[str, Any] = { + "code": code, + "abstractType": abstract_type, + } + + if field.get("description"): + col["description"] = field["description"] + + # Extract field-level ai_context: synonyms → native, rest → customExtensions + ai_ctx = field.get("ai_context") + if ai_ctx: + ai_data = ai_ctx if isinstance(ai_ctx, dict) else {"instructions": ai_ctx} + # Extract synonyms directly into OBML synonyms property + if "synonyms" in ai_data: + col["synonyms"] = list(ai_data["synonyms"]) + # Store remaining ai_context keys in customExtensions + remaining = {k: v for k, v in ai_data.items() if k != "synonyms"} + if remaining: + col["customExtensions"] = [ + { + "vendor": "OSI", + "data": json.dumps(remaining), + } + ] + + # Restore OBML-only column properties from custom_extensions + for ext in field.get("custom_extensions", []): + if ext.get("vendor_name") in _OBML_VENDOR_READ: + try: + ext_data = json.loads(ext.get("data", "{}")) + if ext_data.get("obml_sql_type"): + col["sqlType"] = ext_data["obml_sql_type"] + if ext_data.get("obml_sql_precision") is not None: + col["sqlPrecision"] = ext_data["obml_sql_precision"] + if ext_data.get("obml_sql_scale") is not None: + col["sqlScale"] = ext_data["obml_sql_scale"] + if ext_data.get("obml_num_class"): + col["numClass"] = ext_data["obml_num_class"] + if ext_data.get("obml_comment"): + col["comment"] = ext_data["obml_comment"] + if ext_data.get("obml_owner"): + col["owner"] = ext_data["obml_owner"] + except (json.JSONDecodeError, TypeError): + pass + break + + # ── Field label roundtrip (OSI v0.2 first-class) ── + # OBML has no native column label today; preserve via OBSL-vendor + # customExtensions so the reverse direction can emit it back. + if field.get("label"): + col.setdefault("customExtensions", []).append( + { + "vendor": _VENDOR_OSI, + "data": json.dumps({"obml_field_label": field["label"]}), + } + ) + + # Preserve third-party vendor extensions verbatim + self._carry_foreign_extensions(field.get("custom_extensions"), col) + + return name, col + + def _infer_obml_type(self, field: dict) -> str: + """Infer OBML abstractType from OSI field metadata.""" + + dim = field.get("dimension", {}) + if isinstance(dim, dict) and dim.get("is_time"): + return "date" + + name_lower = field.get("name", "").lower() + + # Helper: match keywords at word boundaries to avoid false positives + # (e.g. "country" should NOT match "count") + def _has_keyword(keywords: tuple[str, ...]) -> bool: + for kw in keywords: + if kw.startswith("_") or kw.endswith("_"): + # Substring match for prefix/suffix patterns like "_sk", "is_" + if kw in name_lower: + return True + else: + # Word-boundary match for standalone keywords + if re.search(r"(?:^|_)" + re.escape(kw) + r"(?:$|_)", name_lower): + return True + return False + + if _has_keyword( + ( + "_sk", + "_id", + "_key", + "name", + "desc", + "email", + "address", + "city", + "state", + "zip", + "phone", + "status", + "type", + "category", + "class", + ) + ): + return "string" + if _has_keyword( + ( + "price", + "cost", + "amount", + "sales", + "profit", + "revenue", + "tax", + "discount", + "rate", + "percent", + "ratio", + "margin", + ) + ): + return "float" + if _has_keyword(("qty", "quantity", "count", "num", "number", "cnt")): + return "int" + if _has_keyword(("date", "time", "year", "month", "day", "week")): + return "date" + if _has_keyword(("flag", "is_", "has_")): + return "boolean" + + return "string" + + # OSI relationship type → OBML joinType mapping + _REL_TYPE_MAP: dict[str, str] = { + "many_to_one": "many-to-one", + "many-to-one": "many-to-one", + "one_to_many": "one-to-many", + "one-to-many": "one-to-many", + "one_to_one": "one-to-one", + "one-to-one": "one-to-one", + "many_to_many": "many-to-many", + "many-to-many": "many-to-many", + } + + def _convert_relationship_to_join(self, rel: dict) -> dict: + """Convert an OSI relationship to an OBML join. + + Uses exact OSI names for joinTo and column references. + Maps OSI relationship 'type' to OBML joinType if present, + defaults to many-to-one with a warning otherwise. + """ + rel_type = rel.get("type", "") + join_type = self._REL_TYPE_MAP.get(rel_type.lower(), "") if rel_type else "" + if not join_type: + join_type = "many-to-one" + if rel_type: + self.warnings.append( + f"Relationship '{rel.get('name', '?')}': unknown type " + f"'{rel_type}', defaulting to many-to-one." + ) + else: + self.warnings.append( + f"Relationship '{rel.get('name', '?')}': no type specified, " + f"defaulting to many-to-one." + ) + + join: dict[str, Any] = { + "joinType": join_type, + "joinTo": rel["to"], + "columnsFrom": list(rel["from_columns"]), + "columnsTo": list(rel["to_columns"]), + } + return join + + def _extract_dimensions(self, datasets: list) -> dict: + """Extract dimension definitions from OSI fields marked as dimensions. + + Skips fields that are join keys (FK/PK columns used in relationships), + since those are structural and not analytical dimensions. + """ + dimensions: dict[str, Any] = {} + for ds in datasets: + ds_name = ds["name"] + for field in ds.get("fields", []): + dim = field.get("dimension") + if dim is None: + continue + field_name = field["name"] + # Skip join key columns — they are FK/PK, not analytical dims + if (ds_name, field_name) in self._join_key_columns: + continue + abstract_type = self._infer_obml_type(field) + dim_def: dict[str, Any] = { + "dataObject": ds_name, + "column": field_name, + "resultType": abstract_type, + } + # Extract synonyms from field-level ai_context + ai_ctx = field.get("ai_context") + if isinstance(ai_ctx, dict) and ai_ctx.get("synonyms"): + dim_def["synonyms"] = list(ai_ctx["synonyms"]) + # Restore OBML-only dimension properties from custom_extensions + for ext in field.get("custom_extensions", []): + if ext.get("vendor_name") in _OBML_VENDOR_READ: + try: + ext_data = json.loads(ext.get("data", "{}")) + if ext_data.get("obml_time_grain"): + dim_def["timeGrain"] = ext_data["obml_time_grain"] + if ext_data.get("obml_dimension_format"): + dim_def["format"] = ext_data["obml_dimension_format"] + if ext_data.get("obml_dimension_result_type"): + dim_def["resultType"] = ext_data["obml_dimension_result_type"] + if ext_data.get("obml_dimension_description"): + dim_def["description"] = ext_data["obml_dimension_description"] + if ext_data.get("obml_dimension_owner"): + dim_def["owner"] = ext_data["obml_dimension_owner"] + if ext_data.get("obml_dimension_via"): + dim_def["via"] = ext_data["obml_dimension_via"] + except (json.JSONDecodeError, TypeError): + pass + break + dimensions[field_name] = dim_def + return dimensions + + def _convert_metrics(self, osi_metrics: list, ds_map: dict) -> tuple[dict, dict]: + """ + Convert OSI metrics to OBML measures and metrics. + + OSI has a single 'metrics' concept with SQL expressions. + OBML separates 'measures' (simple aggregations on single columns) + from 'metrics' (cross-fact expressions referencing measures). + + Strategy: + - Simple single-aggregation metrics → OBML measures + - Aggregation over expression (e.g. SUM(a.x * a.y)) → expression measure + - Complex/multi-aggregation metrics → OBML metrics referencing auto-measures + """ + + measures: dict[str, Any] = {} + metrics: dict[str, Any] = {} + + # Case-insensitive dataset/field index for resolving SQL identifiers + # back to their canonical OSI names (Snowflake/Databricks expressions + # commonly upper-case or quote them). + ds_lc = {name.lower(): name for name in ds_map} + fields_lc = { + name: { + f["name"].lower(): f["name"] + for f in ds.get("fields", []) or [] + if isinstance(f, dict) and f.get("name") + } + for name, ds in ds_map.items() + } + + for m in osi_metrics: + name = m["name"] + + osi_description = m.get("description") + + # Extract synonyms from OSI ai_context + osi_ai_ctx = m.get("ai_context") + osi_synonyms: list[str] = [] + if isinstance(osi_ai_ctx, dict) and osi_ai_ctx.get("synonyms"): + osi_synonyms = list(osi_ai_ctx["synonyms"]) + + # Restore OBML-only properties from custom_extensions + obml_extras = self._extract_obml_extras(m) + + # Check for cumulative metric stored in custom_extensions + if obml_extras.get("obml_metric_type") == "cumulative": + cum_metric = self._reconstruct_cumulative_metric( + name, obml_extras, osi_description, osi_synonyms + ) + metrics[name] = cum_metric + continue + + # Check for period-over-period metric stored in custom_extensions + if obml_extras.get("obml_metric_type") == "period_over_period": + pop_metric = self._reconstruct_pop_metric( + name, obml_extras, osi_description, osi_synonyms + ) + metrics[name] = pop_metric + continue + + # Check for window metric (rank/lag/lead/ntile/first_value/last_value) + if obml_extras.get("obml_metric_type") == "window": + window_metric = self._reconstruct_window_metric( + name, obml_extras, osi_description, osi_synonyms + ) + metrics[name] = window_metric + continue + + # Engine-delegated aggregation (Databricks Metric View). Round-trip + # marker comes from the OBML → OSI direction; on input we restore + # ``aggregation: measure`` without touching the OSI expression + # (which is a literal ``MEASURE("[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' +) +_OSI_KNOWN_VENDORS = ( + "COMMON", + "ORIONBELT", + "SNOWFLAKE", + "SALESFORCE", + "DBT", + "DATABRICKS", + "GOODDATA", +) + +# Vendor identities for custom_extensions. +# ORIONBELT - OrionBelt/OBML-proprietary payloads we author on OBML -> OSI. +# OSI - OSI-native fields OBML can't hold (unique_keys, field label, +# ai_context leftovers), stashed into OBML on OSI -> OBML. +# Read paths also accept the legacy tags we emitted before this scheme so older +# documents still round-trip; foreign vendors (SNOWFLAKE, DBT, ...) are +# preserved verbatim, never relabelled. +_VENDOR_OBML = "ORIONBELT" +_VENDOR_OSI = "OSI" +_OBML_VENDOR_READ = ("ORIONBELT", "COMMON") +_OSI_VENDOR_READ = ("OSI", "OBSL") +# Vendors the converter handles internally (its own payloads + native-field +# stashes). Any custom_extension from a vendor outside this set is third-party +# and is carried through verbatim in both directions, never relabelled. +_INTERNAL_VENDORS = frozenset({"ORIONBELT", "COMMON", "OSI", "OBSL"}) + +# ─── Type mapping ─────────────────────────────────────────────────────────── + +OBML_TO_OSI_TYPE = { + "string": "string", + "json": "string", + "int": "integer", + "float": "number", + "date": "date", + "time": "time", + "time_tz": "time", + "timestamp": "timestamp", + "timestamp_tz": "timestamp", + "boolean": "boolean", +} + +OSI_TO_OBML_TYPE = { + "string": "string", + "integer": "int", + "number": "float", + "date": "date", + "time": "time", + "timestamp": "timestamp", + "boolean": "boolean", +} diff --git a/converters/orionbelt/src/osi_orionbelt/converter.py b/converters/orionbelt/src/osi_orionbelt/converter.py index 6ecf64d2..46a4aa03 100644 --- a/converters/orionbelt/src/osi_orionbelt/converter.py +++ b/converters/orionbelt/src/osi_orionbelt/converter.py @@ -10,2854 +10,120 @@ v0.2 first-class fields before regular parsing runs. Author: OrionBelt / RALFORION + +This module is a thin **facade**. The converter implementation is split across +sibling modules to keep each file focused: + +* :mod:`osi_orionbelt._common` — shared constants and mapping tables +* :mod:`osi_orionbelt.osi_to_obml` — :class:`OSItoOBML` +* :mod:`osi_orionbelt.obml_to_osi` — :class:`OBMLtoOSI` +* :mod:`osi_orionbelt.ontology` — :class:`OBMLtoOSIOntology` +* :mod:`osi_orionbelt.validation` — :class:`ValidationResult` + ``validate_*`` + +Every public name is re-exported here so ``osi_orionbelt.converter.`` +continues to work unchanged. """ +from __future__ import annotations + import argparse -import json -import re import sys from pathlib import Path -from typing import Any import yaml -# ─── Spec version pin ─────────────────────────────────────────────────────── -# Single source of truth for the OSI spec we emit. Bump when upstream cuts -# a stable v0.2.0 (drop the ``.dev0`` suffix). All read paths accept both -# 0.1.x (via the legacy shim) and 0.2.x. -_OSI_VERSION = "0.2.0.dev0" - -# Dialect / vendor enum extras new in v0.2.0.dev0 -_OSI_KNOWN_DIALECTS = ("ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL") -# SQL dialects (of the OSI enum) whose aggregation expressions our regex-based -# metric parser can read, in preference order. ANSI_SQL first; SNOWFLAKE and -# DATABRICKS are SQL engines OrionBelt also targets, and their simple/expression -# aggregations (``SUM(t.c)``, ``SUM(t.a * t.b)``) are syntactically identical to -# ANSI. MDX / TABLEAU / MAQL are non-SQL languages and are never parsed as SQL. -_SQL_PARSEABLE_DIALECTS = ("ANSI_SQL", "SNOWFLAKE", "DATABRICKS") - -# Matches a ``dataset.column`` reference inside a SQL expression, where each -# side is a bare identifier or a quoted identifier (double quotes, backticks, or -# brackets). The leading lookbehind prevents matching the tail of a longer path -# (``a.b.c``) or a mid-token boundary; the bare form must start with a letter or -# underscore so numeric literals (``1.5``) are never treated as references. -_COLUMN_REF_RE = re.compile( - r'(?[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' - r"\s*\.\s*" - r'(?P[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' +from osi_orionbelt._common import ( + _COLUMN_REF_RE as _COLUMN_REF_RE, ) -_OSI_KNOWN_VENDORS = ( - "COMMON", - "ORIONBELT", - "SNOWFLAKE", - "SALESFORCE", - "DBT", - "DATABRICKS", - "GOODDATA", +from osi_orionbelt._common import ( + _INTERNAL_VENDORS as _INTERNAL_VENDORS, ) - -# Vendor identities for custom_extensions. -# ORIONBELT - OrionBelt/OBML-proprietary payloads we author on OBML -> OSI. -# OSI - OSI-native fields OBML can't hold (unique_keys, field label, -# ai_context leftovers), stashed into OBML on OSI -> OBML. -# Read paths also accept the legacy tags we emitted before this scheme so older -# documents still round-trip; foreign vendors (SNOWFLAKE, DBT, ...) are -# preserved verbatim, never relabelled. -_VENDOR_OBML = "ORIONBELT" -_VENDOR_OSI = "OSI" -_OBML_VENDOR_READ = ("ORIONBELT", "COMMON") -_OSI_VENDOR_READ = ("OSI", "OBSL") -# Vendors the converter handles internally (its own payloads + native-field -# stashes). Any custom_extension from a vendor outside this set is third-party -# and is carried through verbatim in both directions, never relabelled. -_INTERNAL_VENDORS = frozenset({"ORIONBELT", "COMMON", "OSI", "OBSL"}) - -# ─── Type mapping ─────────────────────────────────────────────────────────── - -OBML_TO_OSI_TYPE = { - "string": "string", - "json": "string", - "int": "integer", - "float": "number", - "date": "date", - "time": "time", - "time_tz": "time", - "timestamp": "timestamp", - "timestamp_tz": "timestamp", - "boolean": "boolean", -} - -OSI_TO_OBML_TYPE = { - "string": "string", - "integer": "int", - "number": "float", - "date": "date", - "time": "time", - "timestamp": "timestamp", - "boolean": "boolean", -} - - -# ═══════════════════════════════════════════════════════════════════════════ -# OSI → OBML Converter -# ═══════════════════════════════════════════════════════════════════════════ - - -class OSItoOBML: - """Convert an OSI semantic model YAML to OBML format.""" - - def __init__( - self, osi: dict, default_database: str = "ANALYTICS", default_schema: str = "PUBLIC" - ): - self.osi = osi - self.default_database = default_database - self.default_schema = default_schema - self.warnings: list[str] = [] - # OSI metrics that have no OBML representation (non-SQL dialect only, - # or an expression our parser cannot decompose). Preserved verbatim - # rather than dropped — see ``_preserve_unconverted_metric``. - self._unconverted_metrics: list[dict] = [] - - def _normalize_legacy_v01(self) -> None: - """Promote OSI v0.1.x payloads to the v0.2 shape, in place. - - The v0.2 spec promotes ``primary_key`` and ``unique_keys`` to - first-class dataset fields. v0.1.x serializers (including ours - pre-bump) stash both under ``custom_extensions`` with vendor - ``OBSL`` and keys ``obml_primary_key`` / ``obml_unique_keys``. - This shim runs before parsing so the rest of the converter can - assume v0.2 shape regardless of input version. - - No-op for documents that already declare ``version`` >= 0.2 or - that have nothing to migrate. - """ - version = str(self.osi.get("version", "")) - if version and not version.startswith(("0.1", "0.0")): - return # already v0.2+ (or future) — nothing to do - - models = self.osi.get("semantic_model", []) - if not isinstance(models, list): - return - - for model in models: - for ds in model.get("datasets", []) or []: - # Promote legacy primary_key / unique_keys from OBSL extras - # only if the dataset doesn't already declare them. - legacy = self._extract_obml_extras(ds) - if not legacy: - continue - if "primary_key" not in ds and legacy.get("obml_primary_key"): - pk = legacy["obml_primary_key"] - if isinstance(pk, list) and all(isinstance(c, str) for c in pk): - ds["primary_key"] = list(pk) - if "unique_keys" not in ds and legacy.get("obml_unique_keys"): - uk = legacy["obml_unique_keys"] - if isinstance(uk, list) and all( - isinstance(g, list) and all(isinstance(c, str) for c in g) for g in uk - ): - ds["unique_keys"] = [list(g) for g in uk] - - if version.startswith(("0.0", "0.1")): - self.warnings.append( - f"OSI input declares version '{version}'; legacy v0.1.x " - f"compatibility shim applied. Output target is v{_OSI_VERSION}." - ) - - def convert(self) -> dict: - # Reset per-conversion accumulators so calling convert() twice on the - # same instance is idempotent (no duplicated warnings or preserved - # metrics). Both are populated as a side effect of conversion below. - self.warnings = [] - self._unconverted_metrics = [] - - # v0.1.x inputs need the legacy shim to promote pre-v0.2 - # custom_extensions into v0.2 first-class fields before we parse. - self._normalize_legacy_v01() - - models = self.osi.get("semantic_model", []) - if not models: - raise ValueError("No semantic_model found in OSI input") - - # Take the first semantic model (OBML is a single-model format) - model = models[0] - if len(models) > 1: - self.warnings.append( - f"OSI contains {len(models)} semantic models; " - f"only the first ('{model.get('name')}') is converted." - ) - - obml: dict[str, Any] = {"version": 1.0} - - # ── Model description ───────────────────────────────────── - if model.get("description"): - obml["description"] = model["description"] - - # ── DataObjects ───────────────────────────────────────────── - datasets = model.get("datasets", []) - relationships = model.get("relationships", []) - - # Build lookup: dataset_name → dataset - ds_map = {ds["name"]: ds for ds in datasets} - - # Build relationship index: from_dataset → [relationship, ...] - rel_by_from: dict[str, list] = {} - for rel in relationships: - rel_by_from.setdefault(rel["from"], []).append(rel) - - # Collect join key columns: (dataset_name, field_name) pairs - # These should NOT become dimensions (they are FK/PK join keys) - self._join_key_columns: set[tuple[str, str]] = set() - for rel in relationships: - for col in rel.get("from_columns", []): - self._join_key_columns.add((rel["from"], col)) - for col in rel.get("to_columns", []): - self._join_key_columns.add((rel["to"], col)) - - data_objects: dict[str, Any] = {} - for ds in datasets: - do_name, do_obj = self._convert_dataset(ds, rel_by_from) - data_objects[do_name] = do_obj - - obml["dataObjects"] = data_objects - - # ── Dimensions (extracted from OSI fields with dimension metadata) ── - dimensions = self._extract_dimensions(datasets) - if dimensions: - obml["dimensions"] = dimensions - - # ── Measures & Metrics ────────────────────────────────────── - osi_metrics = model.get("metrics", []) - measures, metrics = self._convert_metrics(osi_metrics, ds_map) - if measures: - obml["measures"] = measures - if metrics: - obml["metrics"] = metrics - - # Metrics that have no OBML representation are not dropped: stash the - # original OSI metric verbatim under the OSI vendor so the reverse - # (OBML -> OSI) direction re-emits them and a full OSI -> OBML -> OSI - # roundtrip stays lossless. They are not queryable in OBML; a LOSSY - # warning was already recorded per metric. - if self._unconverted_metrics: - obml.setdefault("customExtensions", []).append( - { - "vendor": _VENDOR_OSI, - "data": json.dumps({"obml_unconverted_metrics": self._unconverted_metrics}), - } - ) - - # ── Restore model-level properties from custom_extensions ──── - for ext in model.get("custom_extensions", []): - if ext.get("vendor_name") in _OBML_VENDOR_READ: - try: - ext_data = json.loads(ext.get("data", "{}")) - if ext_data.get("obml_filters"): - obml["filters"] = ext_data["obml_filters"] - if ext_data.get("obml_settings"): - obml["settings"] = ext_data["obml_settings"] - if ext_data.get("obml_owner"): - obml["owner"] = ext_data["obml_owner"] - except (json.JSONDecodeError, TypeError): - pass - break - - # Preserve third-party vendor extensions verbatim - self._carry_foreign_extensions(model.get("custom_extensions"), obml) - - return obml - - @staticmethod - def _carry_foreign_extensions(osi_exts: list[dict] | None, obml_target: dict[str, Any]) -> None: - """Carry third-party OSI custom_extensions verbatim into OBML. - - Our own payloads and OSI-native stashes are reconstructed elsewhere; - any other vendor's extension is preserved unchanged on the OBML side - so a full OSI -> OBML -> OSI roundtrip keeps the original vendor. - """ - for ext in osi_exts or []: - vendor = ext.get("vendor_name") - if vendor and vendor not in _INTERNAL_VENDORS: - obml_target.setdefault("customExtensions", []).append( - {"vendor": vendor, "data": ext.get("data", "")} - ) - - def _parse_source(self, source: str) -> tuple[str, str, str]: - """Parse 'database.schema.table' into parts.""" - parts = source.split(".") - if len(parts) == 3: - return parts[0], parts[1], parts[2] - elif len(parts) == 2: - return self.default_database, parts[0], parts[1] - else: - return self.default_database, self.default_schema, parts[0] - - def _convert_dataset(self, ds: dict, rel_by_from: dict) -> tuple[str, dict]: - """Convert an OSI dataset to an OBML dataObject. - - Uses the exact OSI dataset name as the OBML data object key. - """ - name = ds["name"] - - source = ds.get("source", name) - database, schema, table = self._parse_source(source) - - do: dict[str, Any] = { - "code": table, - "database": database, - "schema": schema, - } - - # ── Columns ───────────────────────────────────────────────── - columns: dict[str, Any] = {} - fields = ds.get("fields", []) - for field in fields: - col_name, col_obj = self._convert_field(field) - columns[col_name] = col_obj - - # ── Primary key flag propagation (OSI v0.2 first-class) ── - # ``primary_key`` lists physical column codes; mark every matching - # column with ``primaryKey: true``. Unknown PK columns surface as - # a warning (the spec couples PK to relationship cardinality, so - # silently dropping is unsafe). - pk_codes = ds.get("primary_key") or [] - if pk_codes: - code_to_col = {col.get("code"): (cname, col) for cname, col in columns.items()} - unknown_pks: list[str] = [] - for pk_code in pk_codes: - hit = code_to_col.get(pk_code) - if hit is None: - unknown_pks.append(pk_code) - continue - _, col = hit - col["primaryKey"] = True - if unknown_pks: - self.warnings.append( - f"Dataset '{name}' primary_key references unknown columns: " - f"{unknown_pks}. Ignored." - ) - - if columns: - do["columns"] = columns - else: - self.warnings.append(f"Dataset '{name}' has no fields; adding placeholder column.") - do["columns"] = {f"{name}_id": {"code": f"{table}_id", "abstractType": "string"}} - - # ── Joins (from relationships where this dataset is on 'from' side) ── - joins = [] - for rel in rel_by_from.get(name, []): - join_obj = self._convert_relationship_to_join(rel) - joins.append(join_obj) - - if joins: - do["joins"] = joins - - # ── Description (semantic, from OSI) ───────────────────────── - if ds.get("description"): - do["description"] = ds["description"] - - # ── Extract ai_context: synonyms → native, rest → customExtensions ─ - ai_ctx = ds.get("ai_context") - if ai_ctx: - ai_data = ai_ctx if isinstance(ai_ctx, dict) else {"instructions": ai_ctx} - # Extract synonyms directly into OBML synonyms property - if "synonyms" in ai_data: - do["synonyms"] = list(ai_data["synonyms"]) - # Store remaining ai_context keys in customExtensions - remaining = {k: v for k, v in ai_data.items() if k != "synonyms"} - if remaining: - do["customExtensions"] = [ - { - "vendor": "OSI", - "data": json.dumps(remaining), - } - ] - - # Restore DataObject owner / comment / refresh from custom_extensions - for ext in ds.get("custom_extensions", []): - if ext.get("vendor_name") in _OBML_VENDOR_READ: - try: - ext_data = json.loads(ext.get("data", "{}")) - if ext_data.get("obml_owner"): - do["owner"] = ext_data["obml_owner"] - if ext_data.get("obml_comment"): - do["comment"] = ext_data["obml_comment"] - if ext_data.get("obml_refresh"): - do["refresh"] = ext_data["obml_refresh"] - except (json.JSONDecodeError, TypeError): - pass - break - - # ── Unique keys roundtrip (OBML has no native concept) ── - # Persist the OSI ``unique_keys`` array into the OBSL-vendor - # customExtensions so the OBML → OSI direction can emit it back. - unique_keys = ds.get("unique_keys") or [] - if unique_keys: - do.setdefault("customExtensions", []).append( - { - "vendor": _VENDOR_OSI, - "data": json.dumps({"obml_unique_keys": [list(g) for g in unique_keys]}), - } - ) - - # Preserve third-party vendor extensions verbatim - self._carry_foreign_extensions(ds.get("custom_extensions"), do) - - return name, do - - def _convert_field(self, field: dict) -> tuple[str, dict]: - """Convert an OSI field to an OBML column. - - Uses the exact OSI field name as the OBML column key. - """ - name = field["name"] - - # Get expression (prefer ANSI_SQL dialect) - expr_obj = field.get("expression", {}) - code = name # fallback - if isinstance(expr_obj, dict): - dialects = expr_obj.get("dialects", []) - for d in dialects: - if d.get("dialect") == "ANSI_SQL": - code = d.get("expression", name) - break - if not dialects: - code = name - elif code == name and dialects: - code = dialects[0].get("expression", name) - - # Determine abstract type: prefer explicit data_type, fall back to heuristic - osi_type = field.get("data_type", "") - if osi_type and osi_type in OSI_TO_OBML_TYPE: - abstract_type = OSI_TO_OBML_TYPE[osi_type] - else: - abstract_type = self._infer_obml_type(field) - - col: dict[str, Any] = { - "code": code, - "abstractType": abstract_type, - } - - if field.get("description"): - col["description"] = field["description"] - - # Extract field-level ai_context: synonyms → native, rest → customExtensions - ai_ctx = field.get("ai_context") - if ai_ctx: - ai_data = ai_ctx if isinstance(ai_ctx, dict) else {"instructions": ai_ctx} - # Extract synonyms directly into OBML synonyms property - if "synonyms" in ai_data: - col["synonyms"] = list(ai_data["synonyms"]) - # Store remaining ai_context keys in customExtensions - remaining = {k: v for k, v in ai_data.items() if k != "synonyms"} - if remaining: - col["customExtensions"] = [ - { - "vendor": "OSI", - "data": json.dumps(remaining), - } - ] - - # Restore OBML-only column properties from custom_extensions - for ext in field.get("custom_extensions", []): - if ext.get("vendor_name") in _OBML_VENDOR_READ: - try: - ext_data = json.loads(ext.get("data", "{}")) - if ext_data.get("obml_sql_type"): - col["sqlType"] = ext_data["obml_sql_type"] - if ext_data.get("obml_sql_precision") is not None: - col["sqlPrecision"] = ext_data["obml_sql_precision"] - if ext_data.get("obml_sql_scale") is not None: - col["sqlScale"] = ext_data["obml_sql_scale"] - if ext_data.get("obml_num_class"): - col["numClass"] = ext_data["obml_num_class"] - if ext_data.get("obml_comment"): - col["comment"] = ext_data["obml_comment"] - if ext_data.get("obml_owner"): - col["owner"] = ext_data["obml_owner"] - except (json.JSONDecodeError, TypeError): - pass - break - - # ── Field label roundtrip (OSI v0.2 first-class) ── - # OBML has no native column label today; preserve via OBSL-vendor - # customExtensions so the reverse direction can emit it back. - if field.get("label"): - col.setdefault("customExtensions", []).append( - { - "vendor": _VENDOR_OSI, - "data": json.dumps({"obml_field_label": field["label"]}), - } - ) - - # Preserve third-party vendor extensions verbatim - self._carry_foreign_extensions(field.get("custom_extensions"), col) - - return name, col - - def _infer_obml_type(self, field: dict) -> str: - """Infer OBML abstractType from OSI field metadata.""" - - dim = field.get("dimension", {}) - if isinstance(dim, dict) and dim.get("is_time"): - return "date" - - name_lower = field.get("name", "").lower() - - # Helper: match keywords at word boundaries to avoid false positives - # (e.g. "country" should NOT match "count") - def _has_keyword(keywords: tuple[str, ...]) -> bool: - for kw in keywords: - if kw.startswith("_") or kw.endswith("_"): - # Substring match for prefix/suffix patterns like "_sk", "is_" - if kw in name_lower: - return True - else: - # Word-boundary match for standalone keywords - if re.search(r"(?:^|_)" + re.escape(kw) + r"(?:$|_)", name_lower): - return True - return False - - if _has_keyword( - ( - "_sk", - "_id", - "_key", - "name", - "desc", - "email", - "address", - "city", - "state", - "zip", - "phone", - "status", - "type", - "category", - "class", - ) - ): - return "string" - if _has_keyword( - ( - "price", - "cost", - "amount", - "sales", - "profit", - "revenue", - "tax", - "discount", - "rate", - "percent", - "ratio", - "margin", - ) - ): - return "float" - if _has_keyword(("qty", "quantity", "count", "num", "number", "cnt")): - return "int" - if _has_keyword(("date", "time", "year", "month", "day", "week")): - return "date" - if _has_keyword(("flag", "is_", "has_")): - return "boolean" - - return "string" - - # OSI relationship type → OBML joinType mapping - _REL_TYPE_MAP: dict[str, str] = { - "many_to_one": "many-to-one", - "many-to-one": "many-to-one", - "one_to_many": "one-to-many", - "one-to-many": "one-to-many", - "one_to_one": "one-to-one", - "one-to-one": "one-to-one", - "many_to_many": "many-to-many", - "many-to-many": "many-to-many", - } - - def _convert_relationship_to_join(self, rel: dict) -> dict: - """Convert an OSI relationship to an OBML join. - - Uses exact OSI names for joinTo and column references. - Maps OSI relationship 'type' to OBML joinType if present, - defaults to many-to-one with a warning otherwise. - """ - rel_type = rel.get("type", "") - join_type = self._REL_TYPE_MAP.get(rel_type.lower(), "") if rel_type else "" - if not join_type: - join_type = "many-to-one" - if rel_type: - self.warnings.append( - f"Relationship '{rel.get('name', '?')}': unknown type " - f"'{rel_type}', defaulting to many-to-one." - ) - else: - self.warnings.append( - f"Relationship '{rel.get('name', '?')}': no type specified, " - f"defaulting to many-to-one." - ) - - join: dict[str, Any] = { - "joinType": join_type, - "joinTo": rel["to"], - "columnsFrom": list(rel["from_columns"]), - "columnsTo": list(rel["to_columns"]), - } - return join - - def _extract_dimensions(self, datasets: list) -> dict: - """Extract dimension definitions from OSI fields marked as dimensions. - - Skips fields that are join keys (FK/PK columns used in relationships), - since those are structural and not analytical dimensions. - """ - dimensions: dict[str, Any] = {} - for ds in datasets: - ds_name = ds["name"] - for field in ds.get("fields", []): - dim = field.get("dimension") - if dim is None: - continue - field_name = field["name"] - # Skip join key columns — they are FK/PK, not analytical dims - if (ds_name, field_name) in self._join_key_columns: - continue - abstract_type = self._infer_obml_type(field) - dim_def: dict[str, Any] = { - "dataObject": ds_name, - "column": field_name, - "resultType": abstract_type, - } - # Extract synonyms from field-level ai_context - ai_ctx = field.get("ai_context") - if isinstance(ai_ctx, dict) and ai_ctx.get("synonyms"): - dim_def["synonyms"] = list(ai_ctx["synonyms"]) - # Restore OBML-only dimension properties from custom_extensions - for ext in field.get("custom_extensions", []): - if ext.get("vendor_name") in _OBML_VENDOR_READ: - try: - ext_data = json.loads(ext.get("data", "{}")) - if ext_data.get("obml_time_grain"): - dim_def["timeGrain"] = ext_data["obml_time_grain"] - if ext_data.get("obml_dimension_format"): - dim_def["format"] = ext_data["obml_dimension_format"] - if ext_data.get("obml_dimension_result_type"): - dim_def["resultType"] = ext_data["obml_dimension_result_type"] - if ext_data.get("obml_dimension_description"): - dim_def["description"] = ext_data["obml_dimension_description"] - if ext_data.get("obml_dimension_owner"): - dim_def["owner"] = ext_data["obml_dimension_owner"] - if ext_data.get("obml_dimension_via"): - dim_def["via"] = ext_data["obml_dimension_via"] - except (json.JSONDecodeError, TypeError): - pass - break - dimensions[field_name] = dim_def - return dimensions - - def _convert_metrics(self, osi_metrics: list, ds_map: dict) -> tuple[dict, dict]: - """ - Convert OSI metrics to OBML measures and metrics. - - OSI has a single 'metrics' concept with SQL expressions. - OBML separates 'measures' (simple aggregations on single columns) - from 'metrics' (cross-fact expressions referencing measures). - - Strategy: - - Simple single-aggregation metrics → OBML measures - - Aggregation over expression (e.g. SUM(a.x * a.y)) → expression measure - - Complex/multi-aggregation metrics → OBML metrics referencing auto-measures - """ - - measures: dict[str, Any] = {} - metrics: dict[str, Any] = {} - - # Case-insensitive dataset/field index for resolving SQL identifiers - # back to their canonical OSI names (Snowflake/Databricks expressions - # commonly upper-case or quote them). - ds_lc = {name.lower(): name for name in ds_map} - fields_lc = { - name: { - f["name"].lower(): f["name"] - for f in ds.get("fields", []) or [] - if isinstance(f, dict) and f.get("name") - } - for name, ds in ds_map.items() - } - - for m in osi_metrics: - name = m["name"] - - osi_description = m.get("description") - - # Extract synonyms from OSI ai_context - osi_ai_ctx = m.get("ai_context") - osi_synonyms: list[str] = [] - if isinstance(osi_ai_ctx, dict) and osi_ai_ctx.get("synonyms"): - osi_synonyms = list(osi_ai_ctx["synonyms"]) - - # Restore OBML-only properties from custom_extensions - obml_extras = self._extract_obml_extras(m) - - # Check for cumulative metric stored in custom_extensions - if obml_extras.get("obml_metric_type") == "cumulative": - cum_metric = self._reconstruct_cumulative_metric( - name, obml_extras, osi_description, osi_synonyms - ) - metrics[name] = cum_metric - continue - - # Check for period-over-period metric stored in custom_extensions - if obml_extras.get("obml_metric_type") == "period_over_period": - pop_metric = self._reconstruct_pop_metric( - name, obml_extras, osi_description, osi_synonyms - ) - metrics[name] = pop_metric - continue - - # Check for window metric (rank/lag/lead/ntile/first_value/last_value) - if obml_extras.get("obml_metric_type") == "window": - window_metric = self._reconstruct_window_metric( - name, obml_extras, osi_description, osi_synonyms - ) - metrics[name] = window_metric - continue - - # Engine-delegated aggregation (Databricks Metric View). Round-trip - # marker comes from the OBML → OSI direction; on input we restore - # ``aggregation: measure`` without touching the OSI expression - # (which is a literal ``MEASURE("[A-Za-z_]\w*|"[^"]+"|`[^`]+`|\[[^\]]+\])' ) -_OSI_KNOWN_VENDORS = ( - "COMMON", - "ORIONBELT", - "SNOWFLAKE", - "SALESFORCE", - "DBT", - "DATABRICKS", - "GOODDATA", -) - # Vendor identities for custom_extensions. # ORIONBELT - OrionBelt/OBML-proprietary payloads we author on OBML -> OSI. # OSI - OSI-native fields OBML can't hold (unique_keys, field label, diff --git a/converters/orionbelt/src/ossie_orionbelt/converter.py b/converters/orionbelt/src/ossie_orionbelt/converter.py index 3acd2f7a..9d08ff8b 100644 --- a/converters/orionbelt/src/ossie_orionbelt/converter.py +++ b/converters/orionbelt/src/ossie_orionbelt/converter.py @@ -41,12 +41,6 @@ from ossie_orionbelt._common import ( _OBML_VENDOR_READ as _OBML_VENDOR_READ, ) -from ossie_orionbelt._common import ( - _OSI_KNOWN_DIALECTS as _OSI_KNOWN_DIALECTS, -) -from ossie_orionbelt._common import ( - _OSI_KNOWN_VENDORS as _OSI_KNOWN_VENDORS, -) from ossie_orionbelt._common import ( _OSI_VENDOR_READ as _OSI_VENDOR_READ, ) @@ -78,12 +72,6 @@ from ossie_orionbelt.validation import ( _OBML_SCHEMA_PATH as _OBML_SCHEMA_PATH, ) -from ossie_orionbelt.validation import ( - _OSI_CORE_SPEC_RAW_URL as _OSI_CORE_SPEC_RAW_URL, -) -from ossie_orionbelt.validation import ( - _OSI_ONTOLOGY_SCHEMA_PATH as _OSI_ONTOLOGY_SCHEMA_PATH, -) from ossie_orionbelt.validation import ( _OSI_SCHEMA_PATH as _OSI_SCHEMA_PATH, ) @@ -96,9 +84,6 @@ from ossie_orionbelt.validation import ( ValidationResult as ValidationResult, ) -from ossie_orionbelt.validation import ( - _osi_core_registry as _osi_core_registry, -) from ossie_orionbelt.validation import ( _validate_json_schema as _validate_json_schema, ) diff --git a/converters/orionbelt/src/ossie_orionbelt/schemas/osi-ontology-schema.json b/converters/orionbelt/src/ossie_orionbelt/schemas/osi-ontology-schema.json deleted file mode 100644 index 473f69b3..00000000 --- a/converters/orionbelt/src/ossie_orionbelt/schemas/osi-ontology-schema.json +++ /dev/null @@ -1,299 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", - "title": "OSI Ontology Metadata Specification", - "description": "JSON Schema for validating OSI (Open Semantic Interoperability) ontology definitions", - "type": "object", - "properties": { - "version": { - "type": "string", - "const": "0.2.0.dev0", - "description": "Ontology specification version" - }, - "name": { - "type": "string", - "description": "Unique identifier for the ontology" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/AIContext" - }, - "ontology": { - "type": "array", - "items": { - "$ref": "#/$defs/OntologyComponent" - }, - "minItems": 1, - "description": "Components that define the concepts and relationships in this ontology" - }, - "ontology_mappings": { - "type": "array", - "description": "Collection of ontology maps from logical models", - "items": { - "$ref": "#/$defs/OntologyMap" - } - } - }, - "required": ["version", "name", "ontology"], - "additionalProperties": false, - "$defs": { - "OntologyComponent": { - "type": "object", - "description": "Ontology component that defines a single concept and any relationships that are keyed primarily by that concept", - "properties": { - "description": { - "type": "string", - "description": "Human-readable description of the component" - }, - "concept": { - "$ref": "#/$defs/Concept" - }, - "relationships": { - "type": "array", - "items": { - "$ref": "#/$defs/Relationship" - }, - "description": "Defines relationships that pertain primarily to the concept defined in this component" - } - }, - "required": ["concept"], - "additionalProperties": false - }, - "Expression": { - "type": "string", - "description": "ANSI SQL expression" - }, - "Relationship": { - "type": "object", - "description": "Relationship between concepts in the ontology", - "properties": { - "name": { - "type": "string", - "description": "Name of the relationship" - }, - "description": { - "type": "string", - "description": "Human-readable description of the relationship" - }, - "roles": { - "type": "array", - "items": { - "$ref": "#/$defs/Role" - }, - "description": "Additional roles in this relationship" - }, - "multiplicity": { - "$ref": "#/$defs/Multiplicity" - }, - "derived_by": { - "type": "array", - "items": { - "$ref": "#/$defs/Expression" - }, - "description": "Expressions that define how this concept is derived" - }, - "verbalizes": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Natural language expressions that verbalize this relationship" - } - }, - "required": ["name", "verbalizes"], - "additionalProperties": false - }, - "Concept": { - "type": "object", - "description": "Defines a concept in the ontology", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the concept" - }, - "type": { - "$ref": "#/$defs/ConceptType" - }, - "description": { - "type": "string", - "description": "Human-readable description of the concept" - }, - "extends": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Indicates that this concept extends one or more other concepts" - }, - "derived_by": { - "type": "array", - "items": { - "$ref": "#/$defs/Expression" - }, - "description": "Expressions that define how this concept is derived" - }, - "identify_by": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Names of relationships to use as the preferred identifier of this concept" - }, - "requires": { - "type": "array", - "items": { - "$ref": "#/$defs/Expression" - }, - "description": "Expressions that constrain the population of this concept" - } - }, - "required": ["name", "type"], - "additionalProperties": false - }, - "ConceptMapping": { - "type": "object", - "description": "Mappings from logical model constructs to some ontology component", - "properties": { - "concept": { - "type": "string", - "description": "Name of the concept whose part of the ontology we are mapping to" - }, - "object_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/ObjectMapping" - }, - "description": "Mappings from logical constructs that populate the concept in this component. Valid only when the concept is an entity type" - }, - "link_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/LinkMapping" - }, - "description": "Mappings from logical model relationships to ontology relationships pertaining to the mapped concept" - } - }, - "required": ["concept"], - "additionalProperties": false - }, - "ConceptType": { - "type": "string", - "enum": [ "EntityType", "ValueType" ], - "description": "A concept is either an entity type or a value type" - }, - "ReferentMapping": { - "type": "object", - "description": "Mapping from logical model constructs to a relationship used to references some entity type in the ontology", - "properties": { - "relationship": { - "type": "string", - "description": "Name of referent relationship" - }, - "expression": { - "$ref": "#/$defs/Expression" - }, - "referent_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/ReferentMapping" - } - } - }, - "required": ["relationship"], - "additionalProperties": false - }, - "Role": { - "type": "object", - "description": "Role in some relationship (the container)", - "properties": { - "concept": { - "type": "string", - "description": "Name of the concept playing this role" - }, - "name": { - "type": "string", - "description": "Optional name of this role, used when the same concept plays multiple roles in the same relationship" - } - }, - "required": ["concept"], - "additionalProperties": false - }, - "ObjectMapping": { - "type": "object", - "description": "Pattern of logical-level expressions for identifying objects of some concept using the values in one or more fields", - "properties": { - "concept": { - "type": "string", - "description": "Name of the concept whose objects we are mapping to" - }, - "referent_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/ReferentMapping" - }, - "description": "Maps logical-model constructs to referent relationships of this entity type" - }, - "expression": { - "$ref": "#/$defs/Expression" - } - }, - "additionalProperties": false - }, - "LinkMapping": { - "type": "object", - "description": "Mapping from logical schema to the links of relationships in the ontology", - "properties": { - "relationship": { - "type": "string", - "description": "Name of relationship being populated by this mapping node" - }, - "object_mapping": { - "$ref": "#/$defs/ObjectMapping" - }, - "children": { - "type": "array", - "items": { - "$ref": "#/$defs/LinkMapping" - }, - "description": "Relationship maps at the next level in this hierarchy" - } - }, - "required": ["object_mapping"], - "additionalProperties": false - }, - "Multiplicity": { - "type": "string", - "enum": [ "ManyToOne", "OneToOne" ], - "description": "Relationship multiplicity" - }, - "OntologyMap": { - "type": "object", - "description": "Map from the constructs of some logical model to some ontology", - "properties": { - "name": { - "type": "string", - "description": "Name of this ontology map" - }, - "description": { - "type": "string", - "description": "Human-readable description of this ontology map" - }, - "semantic_model": { - "$ref": "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json#/$defs/SemanticModel" - }, - "concept_mappings": { - "type": "array", - "items": { - "$ref": "#/$defs/ConceptMapping" - }, - "description": "Maps logical model constructs to some concept and its relationships in the ontology" - } - }, - "required": ["semantic_model", "concept_mappings"], - "additionalProperties": false - } - } -} diff --git a/converters/orionbelt/src/ossie_orionbelt/schemas/osi-schema.json b/converters/orionbelt/src/ossie_orionbelt/schemas/osi-schema.json deleted file mode 100644 index 72cb164d..00000000 --- a/converters/orionbelt/src/ossie_orionbelt/schemas/osi-schema.json +++ /dev/null @@ -1,330 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/open-semantic-interchange/OSI/core-spec/osi-schema.json", - "title": "OSI Core Metadata Specification", - "description": "JSON Schema for validating OSI (Open Semantic Interoperability) semantic model definitions", - "type": "object", - "properties": { - "version": { - "type": "string", - "const": "0.2.0.dev0", - "description": "OSI specification version" - }, - "semantic_model": { - "type": "array", - "description": "Collection of semantic model definitions", - "items": { - "$ref": "#/$defs/SemanticModel" - } - } - }, - "required": ["version", "semantic_model"], - "additionalProperties": false, - "$defs": { - "Dialect": { - "type": "string", - "enum": ["ANSI_SQL", "SNOWFLAKE", "MDX", "TABLEAU", "DATABRICKS", "MAQL"], - "description": "Supported SQL and expression language dialects" - }, - "Vendor": { - "type": "string", - "examples": ["COMMON", "SNOWFLAKE", "SALESFORCE", "DBT", "DATABRICKS", "GOODDATA"], - "description": "Vendor name for custom extensions. Any string value is accepted." - }, - "AIContext": { - "description": "Additional context for AI tools", - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - "properties": { - "instructions": { - "type": "string", - "description": "Instructions for AI on how to use this entity" - }, - "synonyms": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Alternative names and terms" - }, - "examples": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Sample questions or use cases" - } - }, - "additionalProperties": true - } - ] - }, - "CustomExtension": { - "type": "object", - "description": "Vendor-specific attributes for extensibility", - "properties": { - "vendor_name": { - "$ref": "#/$defs/Vendor" - }, - "data": { - "type": "string", - "description": "JSON string containing vendor-specific data" - } - }, - "required": ["vendor_name", "data"], - "additionalProperties": false - }, - "DialectExpression": { - "type": "object", - "description": "Expression in a specific dialect", - "properties": { - "dialect": { - "$ref": "#/$defs/Dialect" - }, - "expression": { - "type": "string", - "description": "SQL or dialect-specific expression" - } - }, - "required": ["dialect", "expression"], - "additionalProperties": false - }, - "Expression": { - "type": "object", - "description": "Expression definition with multi-dialect support", - "properties": { - "dialects": { - "type": "array", - "items": { - "$ref": "#/$defs/DialectExpression" - }, - "minItems": 1 - } - }, - "required": ["dialects"], - "additionalProperties": false - }, - "Dimension": { - "type": "object", - "description": "Dimension metadata", - "properties": { - "is_time": { - "type": "boolean", - "description": "Indicates if this is a time-based dimension for temporal filtering" - } - }, - "additionalProperties": false - }, - "Field": { - "type": "object", - "description": "Row-level attribute for grouping, filtering, and metric expressions", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the field within the dataset" - }, - "expression": { - "$ref": "#/$defs/Expression" - }, - "dimension": { - "$ref": "#/$defs/Dimension" - }, - "label": { - "type": "string", - "description": "Label for categorization" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "expression"], - "additionalProperties": false - }, - "Dataset": { - "type": "object", - "description": "Logical dataset representing a business entity (fact or dimension table)", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the dataset" - }, - "source": { - "type": "string", - "description": "Reference to underlying physical table/view (database.schema.table) or query" - }, - "primary_key": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Primary key columns (single or composite)" - }, - "unique_keys": { - "type": "array", - "items": { - "type": "array", - "items": { - "type": "string" - } - }, - "description": "Array of unique key definitions (each can be single or composite)" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "fields": { - "type": "array", - "items": { - "$ref": "#/$defs/Field" - } - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "source"], - "additionalProperties": false - }, - "Relationship": { - "type": "object", - "description": "Foreign key relationship between datasets", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the relationship" - }, - "from": { - "type": "string", - "description": "Dataset on the many side of the relationship" - }, - "to": { - "type": "string", - "description": "Dataset on the one side of the relationship" - }, - "from_columns": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "description": "Foreign key columns in the 'from' dataset" - }, - "to_columns": { - "type": "array", - "items": { - "type": "string" - }, - "minItems": 1, - "description": "Primary/unique key columns in the 'to' dataset" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "from", "to", "from_columns", "to_columns"], - "additionalProperties": false - }, - "Metric": { - "type": "object", - "description": "Quantitative measure defined on business data", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the metric" - }, - "expression": { - "$ref": "#/$defs/Expression" - }, - "description": { - "type": "string", - "description": "Human-readable description of what the metric measures" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "expression"], - "additionalProperties": false - }, - "SemanticModel": { - "type": "object", - "description": "Top-level container representing a complete semantic model", - "properties": { - "name": { - "type": "string", - "description": "Unique identifier for the semantic model" - }, - "description": { - "type": "string", - "description": "Human-readable description" - }, - "ai_context": { - "$ref": "#/$defs/AIContext" - }, - "datasets": { - "type": "array", - "items": { - "$ref": "#/$defs/Dataset" - }, - "minItems": 1, - "description": "Collection of logical datasets" - }, - "relationships": { - "type": "array", - "items": { - "$ref": "#/$defs/Relationship" - }, - "description": "Defines how datasets are connected" - }, - "metrics": { - "type": "array", - "items": { - "$ref": "#/$defs/Metric" - }, - "description": "Quantifiable measures spanning datasets" - }, - "custom_extensions": { - "type": "array", - "items": { - "$ref": "#/$defs/CustomExtension" - } - } - }, - "required": ["name", "datasets"], - "additionalProperties": false - } - } -} diff --git a/converters/orionbelt/src/ossie_orionbelt/validation.py b/converters/orionbelt/src/ossie_orionbelt/validation.py index 738478c0..64c9f515 100644 --- a/converters/orionbelt/src/ossie_orionbelt/validation.py +++ b/converters/orionbelt/src/ossie_orionbelt/validation.py @@ -13,20 +13,39 @@ _SCRIPT_DIR = Path(__file__).resolve().parent _SCHEMAS_DIR = _SCRIPT_DIR / "schemas" -# All three schemas are tracked files beside the converter, so the package is -# self-contained (no repo-root dependency, sdist/wheel build in isolation). The -# OBML schema is a vendored snapshot of the canonical repo-root -# schema/obml-schema.json, kept in sync by a drift-guard test. +# The OBML schema is OrionBelt's own format, always vendored beside the package +# and kept in sync with the repo-root schema/obml-schema.json by a drift-guard +# test. _OBML_SCHEMA_PATH = _SCHEMAS_DIR / "obml-schema.json" -_OSI_SCHEMA_PATH = _SCHEMAS_DIR / "osi-schema.json" -_OSI_ONTOLOGY_SCHEMA_PATH = _SCHEMAS_DIR / "osi-ontology-schema.json" -# The OSI ontology schema $refs the core-spec schema by its public raw URL for -# ``ai_context`` and the embedded ``semantic_model``. Resolve that URL against -# the vendored local copy so validation never touches the network. -_OSI_CORE_SPEC_RAW_URL = ( - "https://raw.githubusercontent.com/open-semantic-interchange/OSI/main/core-spec/osi-schema.json" -) + +def _osi_schema_path(filename: str) -> Path: + """Resolve an OSI core-spec schema without forcing a duplicate copy where a + canonical one already exists. + + Resolution order: + 1. A copy vendored beside this package (``schemas/``). The + standalone PyPI wheel and the OrionBelt product API rely on this, so the + package validates OSI documents self-contained and offline. + 2. Otherwise ``core-spec/`` in an enclosing Ossie monorepo + checkout, found by walking up from this file. This lets the in-tree + converter drop its vendored copy and link the single canonical schema + rather than duplicating it. + + Returns the vendored path unchanged when neither exists, so downstream + ``.exists()`` checks degrade to skip-with-warning rather than raising. + """ + vendored = _SCHEMAS_DIR / filename + if vendored.exists(): + return vendored + for parent in _SCRIPT_DIR.parents: + candidate = parent / "core-spec" / filename + if candidate.exists(): + return candidate + return vendored + + +_OSI_SCHEMA_PATH = _osi_schema_path("osi-schema.json") class ValidationResult: @@ -101,30 +120,6 @@ def _validate_json_schema( result.schema_errors.append(f"[{path}] {error.message}") -def _osi_core_registry() -> Any | None: - """Build a ``referencing.Registry`` that resolves the OSI core-spec schema - URL (referenced by the ontology schema) to the vendored local copy. Returns - ``None`` if the dependencies or the local core schema are unavailable, in - which case the caller falls back to default (network) resolution.""" - try: - from referencing import Registry, Resource - from referencing.jsonschema import DRAFT202012 - except ImportError: - return None - if not _OSI_SCHEMA_PATH.exists(): - return None - with open(_OSI_SCHEMA_PATH) as f: - core = json.load(f) - core_res = Resource.from_contents(core, default_specification=DRAFT202012) - # Register under both the raw URL used by the ontology schema's $refs and - # the core schema's own canonical $id (so its internal #/$defs refs resolve). - resources = [(_OSI_CORE_SPEC_RAW_URL, core_res)] - core_id = core_res.id() - if core_id: - resources.append((core_id, core_res)) - return Registry().with_resources(resources) - - # ── OBML Validation ────────────────────────────────────────────────────── @@ -262,30 +257,19 @@ def validate_osi(osi_dict: dict[str, Any], schema_path: Path | None = None) -> V return result -def validate_osi_ontology( - onto_dict: dict[str, Any], schema_path: Path | None = None -) -> ValidationResult: - """Validate an OSI ontology dict against JSON Schema and semantic rules. +def validate_osi_ontology(onto_dict: dict[str, Any]) -> ValidationResult: + """Validate an OSI ontology dict against semantic rules. - 1. **JSON Schema** — structural correctness against ``osi-ontology-schema.json`` - (Draft 2020-12). External ``$ref``s to the core-spec schema are resolved - against the vendored local copy via a ``referencing`` registry. - 2. **Unique concept names** across the ``ontology`` components. - 3. **Reference integrity** — relationship roles and concept_mappings + Ossie ships no OSI ontology schema (the converter emits ontology documents + but never consumes them), so JSON-Schema conformance is not checked here. + Runs the converter-owned semantic checks: + 1. **Unique concept names** across the ``ontology`` components. + 2. **Reference integrity** — relationship roles and concept_mappings reference concepts defined in the ontology. """ result = ValidationResult("OSI-ONTOLOGY") - # 1. JSON Schema validation (offline external-ref resolution). - _validate_json_schema( - onto_dict, - schema_path or _OSI_ONTOLOGY_SCHEMA_PATH, - result, - draft="draft2020", - registry=_osi_core_registry(), - ) - - # 2. Unique concept names + collect the defined set. + # 1. Unique concept names + collect the defined set. defined: set[str] = set() for comp in onto_dict.get("ontology", []): name = comp.get("concept", {}).get("name", "") @@ -293,7 +277,7 @@ def validate_osi_ontology( result.semantic_errors.append(f"[DUPLICATE_CONCEPT] Duplicate concept name '{name}'") defined.add(name) - # 3. Reference integrity — roles reference defined concepts. + # 2. Reference integrity — roles reference defined concepts. for comp in onto_dict.get("ontology", []): for rel in comp.get("relationships", []): rel_name = rel.get("name", "") diff --git a/converters/orionbelt/tests/test_osi_converter_ontology.py b/converters/orionbelt/tests/test_osi_converter_ontology.py index 8c2a6d7e..7669387e 100644 --- a/converters/orionbelt/tests/test_osi_converter_ontology.py +++ b/converters/orionbelt/tests/test_osi_converter_ontology.py @@ -1,9 +1,9 @@ """Tests for the OBML → OSI **ontology** converter (OBMLtoOSIOntology). -Validates that the derived ontology document conforms to the vendored -``osi-ontology-schema.json`` (with external core-spec refs resolved offline), -that OBML join cardinality maps to OSI multiplicity, and that the documented -gaps (many-to-many, composite keys, missing PK) surface as warnings. +Validates that the derived ontology document passes the converter's semantic +checks (unique concepts, reference integrity), that OBML join cardinality maps +to OSI multiplicity, and that the documented gaps (many-to-many, composite keys, +missing PK) surface as warnings. """ from __future__ import annotations @@ -105,11 +105,10 @@ def test_concept_mappings_bind_keys_and_fks(self) -> None: "expression": "ORDERS.CUSTOMER_ID", } - def test_validates_against_ontology_schema_offline(self) -> None: + def test_passes_ontology_semantic_validation(self) -> None: doc = conv.OBMLtoOSIOntology(_OBML, model_name="sales").convert() result = conv.validate_osi_ontology(doc) - assert result.valid, result.schema_errors + result.semantic_errors - assert not result.schema_errors + assert result.valid, result.semantic_errors assert not result.semantic_errors def test_one_to_one_multiplicity(self) -> None: diff --git a/converters/orionbelt/tests/test_osi_v02_compat.py b/converters/orionbelt/tests/test_osi_v02_compat.py index a6c78171..f5c110b9 100644 --- a/converters/orionbelt/tests/test_osi_v02_compat.py +++ b/converters/orionbelt/tests/test_osi_v02_compat.py @@ -8,7 +8,7 @@ - Dataset ``unique_keys`` round-trips lossly via OBSL custom_extensions - Field ``label`` round-trips via OBSL custom_extensions - Legacy v0.1.1 inputs are normalized in place by the shim -- Every emitted document validates against the vendored v0.2 schema +- Every emitted document validates against the resolved v0.2 core schema """ from __future__ import annotations @@ -25,16 +25,13 @@ # Fixtures # --------------------------------------------------------------------------- -_SCHEMA_PATH = ( - Path(__file__).resolve().parents[1] / "src" / "ossie_orionbelt" / "schemas" / "osi-schema.json" -) - @pytest.fixture(scope="module") def schema_validator() -> Any: - """Draft 2020-12 validator pinned to the vendored OSI v0.2 schema.""" + """Draft 2020-12 validator pinned to the resolved OSI v0.2 core schema + (vendored copy if present, otherwise the ossie ``core-spec/`` copy).""" jsonschema = pytest.importorskip("jsonschema") - with open(_SCHEMA_PATH) as f: + with open(conv._OSI_SCHEMA_PATH) as f: schema = json.load(f) return jsonschema.Draft202012Validator(schema) From a84318afebfccc7a73ebb51e9addb77eaa262b44 Mon Sep 17 00:00:00 2001 From: Ralf Becher Date: Tue, 14 Jul 2026 19:24:36 +0200 Subject: [PATCH 7/7] orionbelt converter: bundle OSI core schema into the wheel from core-spec The vendored osi-schema.json copy was removed so the source tree carries no duplicate of the spec. But a pip-installed wheel has no core-spec/ alongside it, so validate_osi would silently skip schema validation. force-include the single canonical core-spec/osi-schema.json into the built wheel: no tracked duplicate, yet a pip install still validates OSI documents self-contained. In-tree runs still resolve it from core-spec/ via validation._osi_schema_path. --- converters/orionbelt/pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/converters/orionbelt/pyproject.toml b/converters/orionbelt/pyproject.toml index 474571a7..d0396570 100644 --- a/converters/orionbelt/pyproject.toml +++ b/converters/orionbelt/pyproject.toml @@ -36,6 +36,13 @@ ossie-orionbelt = "ossie_orionbelt.cli:main" [tool.hatch.build.targets.wheel] packages = ["src/ossie_orionbelt"] +# The OSI core schema is not tracked as a copy in this package (it lives once at +# the repo's core-spec/). Bundle it into the built wheel from that single source +# so a pip-installed converter still validates OSI documents self-contained. +# In-tree (dev/test) runs resolve it from core-spec/ via validation._osi_schema_path. +[tool.hatch.build.targets.wheel.force-include] +"../../core-spec/osi-schema.json" = "ossie_orionbelt/schemas/osi-schema.json" + # Only the OBML schema (OrionBelt's own format) is vendored under # src/ossie_orionbelt/schemas/. OSI core documents are validated against the # repo's core-spec/osi-schema.json, resolved at runtime by