Initial OSI <> dbt Semantic Layer converters

.gitignore

@@ -1 +1,2 @@
-__pycache__/
+**/__pycache__/
+**/.venv/

converters/dbt/README.md

@@ -0,0 +1,122 @@
+# osi-dbt
+
+Converts between dbt's [MetricFlow Semantic Interface](https://docs.getdbt.com/docs/build/about-metricflow) (MSI) and the [Open Semantic Interchange](https://github.com/open-semantic-interchange/OSI) (OSI) format.
+
+Both conversion directions are supported:
+
+- `msi-to-osi` — `semantic_manifest.json` (dbt output) → OSI YAML
+- `osi-to-msi` — OSI YAML → `semantic_manifest.json`
+
+## Requirements
+
+- Python 3.11+
+- [uv](https://docs.astral.sh/uv/) (recommended) or pip
+
+## Installation
+
+```bash
+pip install osi-dbt
+```
+
+Or with uv:
+
+```bash
+uv add osi-dbt
+```
+
+## CLI usage
+
+### dbt → OSI
+
+Generate `semantic_manifest.json` from your dbt project first:
+
+```bash
+dbt parse
+# output: target/semantic_manifest.json
+```
+
+Then convert to OSI YAML:
+
+```bash
+osi-dbt msi-to-osi -i target/semantic_manifest.json -o semantic_model.yaml
+```
+
+By default the OSI semantic model is named `semantic_model`. Override it with `--model-name`:
+
+```bash
+osi-dbt msi-to-osi -i target/semantic_manifest.json -o semantic_model.yaml --model-name my_project
+```
+
+Conversion issues (e.g. dropped CONVERSION or PRIVATE metrics) are printed as warnings to stderr. The output file is still written.
+
+### OSI → dbt
+
+```bash
+osi-dbt osi-to-msi -i semantic_model.yaml -o semantic_manifest.json
+```
+
+Produces a `semantic_manifest.json` that metricflow can load.
+
+### Help
+
+```bash
+osi-dbt --help
+osi-dbt msi-to-osi --help
+osi-dbt osi-to-msi --help
+```
+
+## Python API
+
+```python
+from osi_dbt import MSIToOSIConverter, OSIToMSIConverter
+from metricflow_semantics.model.dbt_manifest_parser import parse_manifest_from_dbt_generated_manifest
+
+# dbt → OSI
+manifest = parse_manifest_from_dbt_generated_manifest(Path("target/semantic_manifest.json").read_text())
+result = MSIToOSIConverter().convert(manifest, osi_model_name="my_project")
+
+for issue in result.issues:
+    print(f"[warning] {issue.issue_type.value}: {issue.element_name}")
+
+osi_yaml = result.output.to_osi_yaml()
+
+# OSI → dbt
+import yaml
+from osi import OSIDocument
+
+document = OSIDocument.model_validate(yaml.safe_load(Path("semantic_model.yaml").read_text()))
+result = OSIToMSIConverter().convert(document)
+manifest_json = result.output.model_dump_json(by_alias=True, exclude_none=True, indent=2)
+```
+
+### Conversion notes
+
+**MSI → OSI** is lossy in the following ways, each recorded as a `ConverterIssue` in the result:
+
+| Issue type | Reason |
+|---|---|
+| `CONVERSION_METRIC_DROPPED` | OSI has no conversion-funnel metric type |
+| `PRIVATE_METRIC_DROPPED` | OSI has no visibility modifiers |
+| `NATURAL_ENTITY_DROPPED` | OSI has no natural-key entity type |
+| `CUMULATIVE_SEMANTICS_LOSS` | Window/grain semantics cannot be expressed in an OSI expression string; the base aggregation is preserved |
+
+**OSI → MSI** reconstructs a best-effort MSI manifest from OSI's simpler schema. Nothing is dropped, but OSI carries less structural information than MSI, so the converter makes the following choices:
+
+- Single aggregations (`SUM(col)`, `COUNT(DISTINCT col)`, etc.) → SIMPLE metric with `metric_aggregation_params`
+- `(expr_a) / (expr_b)` → RATIO metric with auto-generated sub-metrics
+- Anything else → SIMPLE metric with the raw expression stored verbatim
+- Time dimensions always receive `TimeGranularity.DAY` (OSI carries no granularity field)
+
+## Development
+
+```bash
+cd converters/dbt
+uv sync
+uv run pytest
+```
+
+Generate initial syrupy snapshots on first run:
+
+```bash
+uv run pytest --snapshot-update
+```

converters/dbt/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "osi-dbt"
+version = "0.2.0.dev0"
+description = "dbt (MetricFlow Semantic Interface) <> OSI converter"
+requires-python = ">=3.11"
+dependencies = [
+    "osi-python>=0.2.0.dev0",
+    "metricflow>=0.200",
+    "sqlglot>=20.0",
+    "jinja2>=3.0",
+    "PyYAML>=6.0",
+]
+
+[project.license]
+text = "Apache-2.0"
+
+[project.scripts]
+osi-dbt = "osi_dbt.cli:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/osi_dbt"]
+
+[tool.uv]
+dev-dependencies = [
+    "pytest>=8.0",
+    "syrupy>=4.0",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

converters/dbt/src/osi_dbt/__init__.py

@@ -0,0 +1,11 @@
+from osi_dbt.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult
+from osi_dbt.msi_to_osi import MSIToOSIConverter
+from osi_dbt.osi_to_msi import OSIToMSIConverter
+
+__all__ = [
+    "ConverterIssue",
+    "ConverterIssueType",
+    "ConverterResult",
+    "MSIToOSIConverter",
+    "OSIToMSIConverter",
+]

converters/dbt/src/osi_dbt/cli.py

@@ -0,0 +1,91 @@
+"""CLI entry point for the osi-dbt converter.
+
+Usage:
+    osi-dbt msi-to-osi -i semantic_manifest.json -o output.yaml
+    osi-dbt osi-to-msi -i input.yaml -o semantic_manifest.json
+"""
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+import yaml
+
+from osi import OSIDocument
+from osi_dbt.converter_issues import ConverterIssueType
+from osi_dbt.msi_to_osi import MSIToOSIConverter
+from osi_dbt.osi_to_msi import OSIToMSIConverter
+
+from metricflow_semantics.model.dbt_manifest_parser import parse_manifest_from_dbt_generated_manifest
+
+_ISSUE_REASON: dict[ConverterIssueType, str] = {
+    ConverterIssueType.CONVERSION_METRIC_DROPPED: "OSI has no conversion-funnel metric type",
+    ConverterIssueType.PRIVATE_METRIC_DROPPED: "OSI has no visibility modifiers",
+    ConverterIssueType.NATURAL_ENTITY_DROPPED: "OSI has no natural-key entity type",
+    ConverterIssueType.CUMULATIVE_SEMANTICS_LOSS: "OSI expressions cannot represent window or grain semantics; the base aggregation was preserved",
+}
+
+_DROPPED_ISSUE_TYPES = {
+    ConverterIssueType.CONVERSION_METRIC_DROPPED,
+    ConverterIssueType.PRIVATE_METRIC_DROPPED,
+    ConverterIssueType.NATURAL_ENTITY_DROPPED,
+}
+
+
+def _cmd_msi_to_osi(args: argparse.Namespace) -> None:
+    input_path = Path(args.input)
+    output_path = Path(args.output)
+
+    manifest = parse_manifest_from_dbt_generated_manifest(input_path.read_text())
+    result = MSIToOSIConverter().convert(manifest, osi_model_name=args.model_name)
+
+    if result.issues:
+        for issue in result.issues:
+            verb = "was dropped" if issue.issue_type in _DROPPED_ISSUE_TYPES else "was converted with loss"
+            reason = _ISSUE_REASON[issue.issue_type]
+            print(f"[WARNING] {issue.issue_type.value}: {issue.element_name} {verb} during conversion because {reason}", file=sys.stderr)
+
+    output_path.write_text(result.output.to_osi_yaml())
+    print(f"Written to {output_path}", file=sys.stderr)
+
+
+def _cmd_osi_to_msi(args: argparse.Namespace) -> None:
+    input_path = Path(args.input)
+    output_path = Path(args.output)
+
+    raw = yaml.safe_load(input_path.read_text())
+    document = OSIDocument.model_validate(raw)
+    result = OSIToMSIConverter().convert(document)
+
+    output_path.write_text(result.output.model_dump_json(by_alias=True, exclude_none=True, indent=2))
+    print(f"Written to {output_path}", file=sys.stderr)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="osi-dbt",
+        description="Convert between dbt semantic_manifest.json and OSI YAML.",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    msi_to_osi = subparsers.add_parser("msi-to-osi", help="Convert semantic_manifest.json → OSI YAML")
+    msi_to_osi.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to semantic_manifest.json")
+    msi_to_osi.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output OSI YAML")
+    msi_to_osi.add_argument(
+        "--model-name", default="semantic_model", metavar="NAME", help="OSI semantic model name (default: semantic_model)"
+    )
+
+    osi_to_msi = subparsers.add_parser("osi-to-msi", help="Convert OSI YAML → semantic_manifest.json")
+    osi_to_msi.add_argument("-i", "--input", required=True, metavar="FILE", help="Path to OSI YAML")
+    osi_to_msi.add_argument("-o", "--output", required=True, metavar="FILE", help="Path for output semantic_manifest.json")
+
+    args = parser.parse_args()
+    if args.command == "msi-to-osi":
+        _cmd_msi_to_osi(args)
+    elif args.command == "osi-to-msi":
+        _cmd_osi_to_msi(args)
+
+
+if __name__ == "__main__":
+    main()

converters/dbt/src/osi_dbt/converter_issues.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Generic, List, TypeVar
+
+
+class ConverterIssueType(Enum):
+    """Identifies the kind of information loss that occurred during conversion."""
+
+    CONVERSION_METRIC_DROPPED = "CONVERSION_METRIC_DROPPED"
+    PRIVATE_METRIC_DROPPED = "PRIVATE_METRIC_DROPPED"
+    NATURAL_ENTITY_DROPPED = "NATURAL_ENTITY_DROPPED"
+    CUMULATIVE_SEMANTICS_LOSS = "CUMULATIVE_SEMANTICS_LOSS"
+
+
+@dataclass(frozen=True)
+class ConverterIssue:
+    """Records a single instance of information loss during conversion."""
+
+    issue_type: ConverterIssueType
+    element_name: str
+
+
+T = TypeVar("T")
+
+
+@dataclass(frozen=True)
+class ConverterResult(Generic[T]):
+    """Return value of a converter's convert() method, pairing the output with any conversion issues."""
+
+    output: T
+    issues: List[ConverterIssue]