From bca1500993391e44f292f0f0a654c7b8c10b354c Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Wed, 29 Jul 2026 23:12:48 +0500 Subject: [PATCH 01/25] Add Apache Ossie <-> Cube converter (import direction) Scaffolds converters/cube/ following the osi-omni and osi-databricks converters: a pure offline YAML transform with no Cube deployment, API token, or network access required. This commit lands the import direction (Cube -> Ossie). Cubes become datasets, cube joins become relationships, cube measures are hoisted to model-level metrics, and the mapped view supplies the model's name, description, and AI context -- the view is the model boundary because Cube users are view-first and Cube's agent reads meta.ai_context only from views and members, not cubes. Design decisions worth calling out: - Fan-out. Cube corrects row multiplication at query time by deduplicating on declared primary keys, which a static Ossie expression cannot inherit. So a bare `type: count` maps to COUNT(DISTINCT ) -- exactly equal to both forms Cube renders, and correct in every join context -- and a non-idempotent aggregate on a dataset the graph fans out is refused by default, mirroring Cube's own refusal. --no-strict-fanout downgrades it to a recorded issue. - Calculated measures inline their {other_measure} references, because that is what Cube itself does; Ossie has no metric-to-metric reference. Cycles are rejected. - Measure filters fold into CASE WHEN ... END inside the aggregate, matching Cube's own applyMeasureFilters rendering. - Dimension `type: number` omits Ossie `datatype` rather than assert a precision the model does not carry; `type: geo` splits into two fields since an Ossie field holds one expression. - Losses that cannot be avoided surface as structured ConverterIssues rather than bare warnings, following the osi-dbt converter, so a pipeline can gate on them. Jinja-templated YAML, .js/.ts models, and `extends` are refused or preserved verbatim rather than half-converted. Everything Cube-only round-trips through custom_extensions[CUBE]. 49 tests pass; the fixture output validates against core-spec/osi-schema.json via validation/validate.py. Co-Authored-By: Claude Opus 5 --- .github/workflows/converter-cube-ci.yml | 63 ++ converters/cube/README.md | 237 +++++ converters/cube/pyproject.toml | 67 ++ converters/cube/src/ossie_cube/__init__.py | 37 + converters/cube/src/ossie_cube/_common.py | 593 +++++++++++++ converters/cube/src/ossie_cube/cli.py | 117 +++ .../cube/src/ossie_cube/converter_issues.py | 110 +++ converters/cube/src/ossie_cube/cube_to_osi.py | 839 ++++++++++++++++++ converters/cube/tests/_util.py | 98 ++ converters/cube/tests/conftest.py | 25 + .../fixtureA_cube/model/cubes/orders.yml | 68 ++ .../fixtureA_cube/model/cubes/users.yml | 48 + .../fixtureA_cube/model/views/sales.yml | 33 + converters/cube/tests/test_cube_to_osi.py | 518 +++++++++++ converters/cube/uv.lock | 216 +++++ 15 files changed, 3069 insertions(+) create mode 100644 .github/workflows/converter-cube-ci.yml create mode 100644 converters/cube/README.md create mode 100644 converters/cube/pyproject.toml create mode 100644 converters/cube/src/ossie_cube/__init__.py create mode 100644 converters/cube/src/ossie_cube/_common.py create mode 100644 converters/cube/src/ossie_cube/cli.py create mode 100644 converters/cube/src/ossie_cube/converter_issues.py create mode 100644 converters/cube/src/ossie_cube/cube_to_osi.py create mode 100644 converters/cube/tests/_util.py create mode 100644 converters/cube/tests/conftest.py create mode 100644 converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml create mode 100644 converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml create mode 100644 converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml create mode 100644 converters/cube/tests/test_cube_to_osi.py create mode 100644 converters/cube/uv.lock diff --git a/.github/workflows/converter-cube-ci.yml b/.github/workflows/converter-cube-ci.yml new file mode 100644 index 00000000..a8cafb84 --- /dev/null +++ b/.github/workflows/converter-cube-ci.yml @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Cube CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/cube/**' + - '.github/workflows/converter-cube-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/cube/**' + - '.github/workflows/converter-cube-ci.yml' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + working-directory: converters/cube + run: | + uv sync + + - name: Unit Tests + working-directory: converters/cube + run: | + uv run pytest diff --git a/converters/cube/README.md b/converters/cube/README.md new file mode 100644 index 00000000..4fe445fe --- /dev/null +++ b/converters/cube/README.md @@ -0,0 +1,237 @@ + + +# Apache Ossie <-> Cube converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and a [Cube](https://cube.dev/docs/product/data-modeling/overview) +data model. No Cube deployment, API token, or network access required. + +> **Status:** the **import** direction (Cube -> Ossie) is implemented. The +> **export** direction (Ossie -> Cube) is in progress; the mapping table below +> describes the agreed behavior for both. + +A Cube data model is a *directory* of YAML files rather than a single document, so +this converter maps one Ossie YAML document to/from the Cube model layout: + +``` +model/cubes/.yml # one per Ossie dataset +model/views/.yml # the view the Ossie model maps to +``` + +Import accepts any layout: `cubes:` and `views:` may live in any `.yml`/`.yaml` +file at any depth, several per file, and original file paths are preserved through +a round trip. + +- **Import** (`ossie-cube import`): Cube files -> Ossie. Cube features Ossie has + no native field for are preserved in `custom_extensions[CUBE]`, so + **Cube -> Ossie -> Cube is lossless**. +- **Export** (`ossie-cube export`): Ossie -> Cube files. Ossie features with no + Cube slot are parked under `meta.ossie` rather than dropped -- Cube has a `meta` + field at every level -- so **Ossie -> Cube -> Ossie is lossless too**. + +Any input that breaks a [requirement](#requirements) **raises a +`ConversionError`** -- the converter never silently drops a field or produces an +invalid result. Losses it *can* absorb are returned as structured +[issues](#conversion-issues) rather than printed and forgotten. + +## Installation + +```bash +pip install apache-ossie-cube # once published to PyPI +# or, from a checkout of this directory: +pip install -e . +``` + +The only runtime dependency is `PyYAML`. Python 3.11+. + +## Usage + +### Command line + +```bash +ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] + [--no-strict-fanout] +``` + +With no `-o` the Ossie YAML goes to stdout; issues always go to stderr. `--view` +picks which view's name/description/AI context map onto the Ossie model when the +directory holds several. `--name` overrides the model name. + +### Python API + +```python +from ossie_cube import convert_cube_to_ossie + +ossie_yaml, issues = convert_cube_to_ossie(files) # {relative filename: YAML str} +for issue in issues: + print(issue) +``` + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is specific +to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). + +| Apache Ossie | Cube | Notes | +|---|---|---| +| `semantic_model` | a **view** | Cube users are view-first, and Cube's agent reads `meta.ai_context` only from views and members -- so the view, not any cube, is the model boundary. | +| `semantic_model.name` | view name | Import: the mapped view's name (override with `--name`). | +| `model.description` / `ai_context.instructions` | view `description` / `meta.ai_context` | Import: taken from the sole view, or `--view`. | +| dataset | `cubes[]` entry in `model/cubes/.yml` | Import: a non-canonical original path is stashed and restored on export. | +| `dataset.source` (dotted) | `sql_table` | Passed through verbatim; Cube interpolates it straight into `FROM`, so no catalog/schema split is needed. | +| `dataset.source` (`SELECT ...`) | `sql` | Cube requires exactly one of `sql` / `sql_table`. | +| `dataset.description` | cube `description` | | +| `dataset.ai_context` | cube `meta.ai_context` | Preserved for the round trip, but **inert in Cube** -- its agent ignores cube-level `ai_context`. Recorded as an issue. | +| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. Export: a key column no field covers becomes a `public: false` dimension. | +| `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot; parked rather than dropped. | +| field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | +| `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | +| `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back except for `number`, where it **omits `datatype`** -- Cube collapses three Ossie types into one, and the spec says to omit rather than assert. The original `type` is stashed. | +| `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. | +| `field.label` / `description` | dimension `title` / `description` | | +| `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | +| — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. | +| relationship | `joins[]` on a cube | `many_to_one` on cube A -> `from: A`(many), `to: B`(one). `one_to_many` is flipped so Ossie's `from` is the many side; the declared side and type are stashed so export restores the original. | +| `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities between two member references maps. Anything else (non-equi, range, literal, third cube) is preserved verbatim in the stash. | +| metric | `measures[]` on the cube its expression references | Import hoists cube-scoped measures to the model level, qualifying a colliding name as `__` and stashing the original name and owning cube. | +| `SUM`/`AVG`/`MIN`/`MAX(x)` | `type: sum`/`avg`/`min`/`max` + `sql` | | +| `COUNT(DISTINCT x)` | `type: count_distinct` | | +| `APPROX_COUNT_DISTINCT(x)` | `type: count_distinct_approx` | Cube resolves the warehouse-specific function itself. | +| `COUNT(DISTINCT )` | bare `type: count` | See [Fan-out](#fan-out) -- the primary key is load-bearing here. | +| anything else | `type: number` (calculated) | A `{other_measure}` reference is **inlined**, because that is what Cube itself does; Ossie has no metric-to-metric reference. | +| — | measure `filters` | Folded into `CASE WHEN … THEN … END` inside the aggregate, exactly as Cube's own `applyMeasureFilters` renders it. | +| `metric.datatype` | — | Import emits `Integer` for the count family, whose result type Cube does know, and omits it otherwise. | +| `metric.description` / `ai_context` | measure `description` / `meta.ai_context` | | +| `custom_extensions[CUBE]` | everything Cube-only | Import stashes; export restores -- keeping `Cube -> Ossie -> Cube` lossless. | +| foreign-vendor `custom_extensions` | `meta.ossie.custom_extensions` | Parked so a multi-vendor Ossie model survives the round trip. | + +**Stashed on import** (and restored on export): the views verbatim (minus the +natively mapped description/AI context), the mapped view's identity, original file +paths, cube extras (`title`, `sql_alias`, `data_source`, `public`, `refresh_key`, +`segments`, `pre_aggregations`, `hierarchies`, `access_policy`, `calendar`, ...), +dimension extras (`format`, `currency`, `granularities`, `case`, `sub_query`, +`order`, `aliases`, `meta`, ...), measure extras and any non-reconstructible +measure, joins with no Ossie form, Jinja-templated members, and files with no +Ossie form (`.js`/`.ts` models, non-model YAML). + +**Expression dialects**: Cube SQL is the SQL of the model's data source, and the +Ossie dialect enum has no `CUBE` entry -- so import emits `ANSI_SQL`, and export +prefers `ANSI_SQL` with `--dialect` prepending a warehouse dialect (e.g. +`SNOWFLAKE` for a Snowflake-backed Cube model). + +## Fan-out + +This is the one place where Cube carries semantics an Ossie expression cannot, and +it is handled deliberately rather than papered over. + +When a cube sits on the multiplied side of a join, Cube does **not** aggregate over +the flattened join. It builds `SELECT DISTINCT FROM `, joins +that key set back to the measure's own cube, and aggregates there -- so each source +row is counted once. If the measures themselves span cubes that fan out, Cube +refuses the query outright. Correctness comes from a *runtime rewrite keyed on +declared primary keys*, and a static SQL string has no way to inherit it. + +So the converter emits the fan-out-safe form wherever one exists, and refuses to +emit a silently-wrong one: + +| Cube measure | Ossie expression | Safe under fan-out? | +|---|---|---| +| bare `count` | `COUNT(DISTINCT )` | **Yes, exactly.** Cube renders `count(pk)` normally and `count(distinct pk)` when multiplied; `COUNT(DISTINCT pk)` equals both. A composite key is concatenated with `CAST` + `CONCAT`, as Cube does. | +| `count_distinct` | `COUNT(DISTINCT x)` | Yes, inherently | +| `count_distinct_approx` | `APPROX_COUNT_DISTINCT(x)` | Yes, inherently | +| `min` / `max` | `MIN(x)` / `MAX(x)` | Yes -- idempotent under duplication | +| `sum`, `avg`, `count` + `sql` | `SUM(x)`, `AVG(x)`, `COUNT(x)` | **No** | + +Only the last row is at risk, and only when its own cube is the `to` (one) side of +a relationship in the model. The converter computes that from the Ossie graph and, +**by default, refuses** -- mirroring Cube's own refusal. Pass +`--no-strict-fanout` to emit the metric with a `FANOUT_UNSAFE_METRIC` issue +instead, naming the metric, the dataset, and the relationship responsible. + +Because a bare `count` maps through the primary key, a cube carrying one **must** +declare `primary_key: true` on a dimension; its absence is an error, not a +different number. + +> Ossie has no additivity or grain declaration to record this properly -- dbt's +> `non_additive_dimension` is the nearest precedent, and this repo's dbt converter +> already loses the same information. Worth raising on `dev@`. + +## Conversion issues + +`convert_cube_to_ossie` returns `(yaml, IssueLog)`. Each issue carries a type, the +element it concerns, and a detail string. + +| Issue type | Meaning | +|---|---| +| `FANOUT_UNSAFE_METRIC` | A non-idempotent aggregate on a dataset the graph fans out; see [Fan-out](#fan-out) | +| `MULTI_STAGE_MEASURE_DROPPED` | A `multi_stage` measure (`group_by`/`reduce_by`/`time_shift`/`rank`) renders as a window function over another grain | +| `CUBE_LEVEL_AI_CONTEXT_INERT` | Cube's agent ignores cube-level `meta.ai_context` | +| `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | +| `TEMPLATED_MEMBER_DROPPED` | Jinja templating, or a `.js`/`.ts` model file | +| `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | +| `PARKED_IN_META` | An element preserved in the stash with no native mapping | + +## Requirements + +Conversion raises a `ConversionError` (rather than guessing or emitting something +invalid) when an input breaks one of these: + +- a cube has neither or both of `sql` / `sql_table` (Cube requires exactly one); +- a cube uses `extends` -- resolving it means reproducing Cube's definition-merge + semantics exactly, so it is refused rather than half-applied; +- a bare `type: count` measure's cube declares no primary key; +- a join names a cube that is not in the model, or an unknown `relationship`; +- a measure has an unknown `type`, or a measure reference cycle; +- two cubes, two views, or two derived metric names collide; +- a dimension has an unknown `type`, or a `geo` dimension is missing + `latitude.sql` / `longitude.sql`; +- there are no convertible cubes at all; the input YAML is malformed. + +## Notes and limitations + +- **YAML data models only.** `.js`/`.ts` models and Jinja-templated YAML are + preserved verbatim for the round trip but no cube inside them is converted -- + matching what Cube's own `CubeSchemaConverter` does for the Rollup Designer. +- **camelCase is normalized.** Cube accepts `sqlTable` and `sql_table` alike; + import normalizes to snake_case and export always emits snake_case, so a + camelCase source file comes back snake_cased. +- A filter or computed operand written with bare column names (rather than + `{CUBE}.col`) cannot be qualified into `dataset.column` form, so it is emitted + as-is. Cube's own idiom uses the reference form, which converts fully. +- View curation (`prefix`, `alias`, `includes`/`excludes`, `folders`, + `default_filters`, `view_group`) is stash-and-restore only; Ossie field names + are always *cube* member names, so prefixed view members never leak into them. +- `type: switch` dimensions, `hierarchies`, `pre_aggregations`, `access_policy`, + and multiple `data_source`s have no Ossie semantics and round-trip via the stash. + +## Development + +```bash +uv sync +uv run pytest +``` + +## Future effort + +Both the Apache Ossie specification and Cube's data model are still evolving. As +either side adds or changes fields, this converter will be updated to track them. +Known next steps: the export direction, offline `extends` resolution, and a +first-class Ossie representation for measure additivity so the fan-out caveat can +be recorded in the model instead of an issue log. diff --git a/converters/cube/pyproject.toml b/converters/cube/pyproject.toml new file mode 100644 index 00000000..4b60853e --- /dev/null +++ b/converters/cube/pyproject.toml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "apache-ossie-cube" +version = "0.2.0.dev0" +description = "Bidirectional converter between Apache Ossie semantic models and Cube data models" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Cube", +] +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "PyYAML>=6.0", +] + +[dependency-groups] +dev = [ + "pytest>=8.0", + "hypothesis>=6.0", +] + +[project.scripts] +ossie-cube = "ossie_cube.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_cube"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev", +] diff --git a/converters/cube/src/ossie_cube/__init__.py b/converters/cube/src/ossie_cube/__init__.py new file mode 100644 index 00000000..fcf01fd5 --- /dev/null +++ b/converters/cube/src/ossie_cube/__init__.py @@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bidirectional converter between Apache Ossie semantic models and Cube data +models. Pure offline transforms: Ossie YAML string <-> {relative filename: YAML +string}. + + from ossie_cube import convert_cube_to_ossie + + ossie_yaml, issues = convert_cube_to_ossie(files) +""" + +from ._common import ConversionError +from .converter_issues import ConverterIssue, IssueLog, IssueType +from .cube_to_osi import convert_cube_to_ossie + +__all__ = [ + "ConversionError", + "ConverterIssue", + "IssueLog", + "IssueType", + "convert_cube_to_ossie", +] diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py new file mode 100644 index 00000000..9bfaee02 --- /dev/null +++ b/converters/cube/src/ossie_cube/_common.py @@ -0,0 +1,593 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared helpers for the Apache Ossie <-> Cube converters. + +Both directions are pure offline YAML transforms. The cross-cutting concerns +live here: version constants, the `custom_extensions` stash protocol, Cube +identifier rules, key-spelling normalization, the type/aggregate mapping tables, +and the member-reference translation between Cube's f-string SQL and the plain +column references Ossie expressions use. +""" + +import json +import re + +import yaml + +# Ossie semantic model spec version this converter targets (see core-spec). +OSSIE_VERSION = "0.2.0.dev0" + +# Vendor id used for the `custom_extensions` stash. +VENDOR = "CUBE" + +# Cube SQL is the SQL of the model's data source, so there is no CUBE entry in +# the Ossie dialect enum. Import emits ANSI_SQL; export prefers ANSI_SQL and lets +# the caller prepend a warehouse dialect the actual data source would accept. +DIALECT_ANSI = "ANSI_SQL" + +# Bump when the shape of a stashed `data` blob changes. +STASH_VERSION = 1 + +# Cube's default data model directory layout (`CUBEJS_SCHEMA_PATH` defaults to +# `model`, and `cube create` scaffolds these two subdirectories). +CUBE_DIR = "model/cubes" +VIEW_DIR = "model/views" + +# A valid Cube identifier -- `identifierRegex` in Cube's CubeValidator. +_CUBE_NAME_RE = re.compile(r"^[_a-zA-Z][_a-zA-Z0-9]*$") + +# A bare SQL identifier (single column reference), e.g. `c_name`. +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +# `cube.member` -- a dotted reference an Ossie expression uses to point into a +# dataset. Guarded so `a.b.c` and `1.5` do not match. +DOTTED_REF_RE = re.compile( + r"(? Cube -> Ossie` lossless for models carrying several vendors. + """ + return [ + ext + for ext in (obj or {}).get("custom_extensions") or [] + if ext.get("vendor_name") != VENDOR + ] + + +# --- expressions ---------------------------------------------------------------- + +def pick_expression(ossie_expression, preferred=None): + """Choose the SQL string for an Ossie expression. + + Preference order: the caller-chosen warehouse dialect (Cube passes SQL + through to the data source, so e.g. SNOWFLAKE SQL is valid on a + Snowflake-backed Cube model), then ANSI_SQL. Returns None if neither is + present (the caller records an issue and skips). + """ + dialects = { + d.get("dialect"): d.get("expression") + for d in (ossie_expression or {}).get("dialects") or [] + } + expr = None + if preferred: + expr = dialects.get(preferred) + if expr is None: + expr = dialects.get(DIALECT_ANSI) + if expr is not None and not isinstance(expr, str): + raise ConversionError( + f"expression must be a string, got {type(expr).__name__}") + return expr + + +def synonyms_of(ai_context): + """Extract the synonyms list from an Ossie ai_context (object form only).""" + if isinstance(ai_context, dict): + return list(ai_context.get("synonyms") or []) + return [] + + +def examples_of(ai_context): + if isinstance(ai_context, dict): + return list(ai_context.get("examples") or []) + return [] + + +def instructions_of(ai_context): + """The free-text part of an Ossie ai_context: the string itself, or the + object form's `instructions`.""" + if isinstance(ai_context, str) and ai_context.strip(): + return ai_context + if isinstance(ai_context, dict): + text = ai_context.get("instructions") + if isinstance(text, str) and text.strip(): + return text + return None + + +def cube_sql_to_ossie(sql, own_cube, resolve_ref=None, self_prefix=None): + """Translate Cube member references in a SQL string to the plain references + Ossie expressions use. Returns (translated, changed). + + - `{CUBE}.col` / `{TABLE}.col` -> `col` (a raw column of the own cube) + - `{CUBE.member}` -> `member` (own-cube member reference) + - `{member}` -> `member` (same, unqualified) + - `{other.member}` -> `other.member` + - `{own_cube.member}` -> `member` + + Ossie has no field-vs-column distinction, so both flavors flatten to names. + `\\{` / `\\}` (Cube's escape for a literal brace) survive as plain braces. + + `self_prefix`, when given, qualifies own-cube references with it instead of + reducing them to a bare name -- so `{CUBE}.col` becomes `orders.col`. Ossie + field expressions are dataset-scoped and want the bare form, but model-level + metric expressions address columns as `dataset.column`, so measure conversion + passes the owning cube's name here. + + `resolve_ref`, when given, is called with each raw reference body before the + rules above are applied; returning a string uses it verbatim instead, and + returning None falls through. Measure conversion uses this to inline a + `{other_measure}` reference, which Cube resolves to that measure's own + aggregate SQL and Ossie has no reference form for. + """ + if not isinstance(sql, str): + sql = str(sql) + changed = False + protected = sql.replace("\\{", _ESC_OPEN).replace("\\}", _ESC_CLOSE) + + def repl(m): + nonlocal changed + body = m.group(1).strip() + if resolve_ref is not None: + override = resolve_ref(body) + if override is not None: + changed = True + return override + changed = True + head, _, rest = body.partition(".") + if not rest: + # A lone `{name}`: either `{CUBE}`/`{TABLE}`, the cube's own name + # spelled out, or an unqualified member reference. The first two are + # an alias that a trailing `.column` attaches to, so they are marked + # for removal along with that dot; a member name is an own-cube + # reference. + if body in _SELF_REFS or (own_cube and body == own_cube): + return _SELF_MARK + return f"{self_prefix}.{body}" if self_prefix else body + if head in _SELF_REFS or (own_cube and head == own_cube): + return f"{self_prefix}.{rest}" if self_prefix else rest + return body + + out = _CUBE_REF_RE.sub(repl, protected) + # `{CUBE}.column` -- the alias marker plus the dot the column hangs off. + out = out.replace(f"{_SELF_MARK}.", f"{self_prefix}." if self_prefix else "") + out = out.replace(_SELF_MARK, "") + out = out.replace(_ESC_OPEN, "{").replace(_ESC_CLOSE, "}") + return out, changed + + +def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=()): + """Rewrite an Ossie expression into Cube member-reference form. + + Only *dotted* `cube.name` references are rewritten -- a bare identifier stays + bare, because in Ossie it is a physical column of the owning dataset and + rewriting it to `{CUBE.name}` would make a member's own `sql` self-referential. + + A dotted reference resolves to whichever form Cube expects: + - `own_cube.member` where `member` is declared -> `{CUBE.member}` + (compile-time checked, and inlines the member's own SQL) + - `own_cube.column` where it is not -> `{CUBE}.column` + (a raw physical column, passed through to the database) + - `other_cube.member` -> `{other_cube.member}` + (which is also what triggers the implicit join a cross-dataset metric needs) + + The own cube is always referenced as `{CUBE}` rather than by name, so the + model keeps working when the cube is extended. Literal braces in the incoming + expression are escaped. + """ + escaped = str(expr).replace("{", "\\{").replace("}", "\\}") + known = set(cube_names) + members = set(own_members) + + def repl(m): + head, name = m.group(1), m.group(2) + if head == own_cube: + return "{CUBE." + name + "}" if name in members else "{CUBE}." + name + if head in known: + return "{" + head + "." + name + "}" + # Not a dataset in this model -- a genuine schema-qualified table + # reference or an unrelated dotted token. Leave it alone. + return m.group(0) + + return DOTTED_REF_RE.sub(repl, escaped) + + +# --- source --------------------------------------------------------------------- + +def parse_source(source, dataset_name): + """Classify an Ossie dataset `source` for placement on a Cube cube. + + Returns ("sql", sql_text) for a SELECT/WITH subquery source, or + ("sql_table", table_ref) for a table reference. Cube's `sql_table` takes the + reference verbatim (it is interpolated straight into FROM), so no splitting + into catalog/schema/table is needed -- unlike Omni, Cube has no separate + `schema` key, which also means a bare one-part table name is fine. + """ + if not source or not str(source).strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = str(source).strip() + if re.match(r"(?i)(select|with)\b", s): + return ("sql", s) + return ("sql_table", s) + + +def join_source(cube, cube_name): + """Rebuild an Ossie dataset `source` string from a Cube cube dict. + + Cube's schema requires exactly one of `sql` / `sql_table` (an `xor` in + CubeValidator), so anything else is rejected rather than guessed at. + """ + sql = cube.get("sql") + table = cube.get("sql_table") + if sql is not None and table is not None: + raise ConversionError( + f"Cube '{cube_name}': has both 'sql' and 'sql_table'; Cube allows " + f"exactly one") + if table is not None: + return str(table).strip() + if sql is not None: + return str(sql).strip() + raise ConversionError( + f"Cube '{cube_name}': has neither 'sql' nor 'sql_table' (an `extends`-only " + f"cube?); Ossie datasets require a source") + + +# --- type mapping --------------------------------------------------------------- + +# Cube dimension `type` -> Ossie `datatype`. `number` is deliberately absent: +# Cube collapses Integer/Decimal/Float into one type, and Ossie says to omit +# `datatype` when it is unknown rather than assert a precision the model does not +# have. (Cube's SQL API reports `number` as Double, but that is a wire-protocol +# floor, not a claim about the column.) `geo` is absent because such a dimension +# is split into two numeric fields. +DIM_TYPE_TO_DATATYPE = { + "string": "String", + "boolean": "Boolean", + "time": "DateTime", + "switch": "String", +} + +# Ossie `datatype` -> Cube dimension `type`, which is required on every +# dimension. Lossy in the numeric and temporal directions by construction. +DATATYPE_TO_DIM_TYPE = { + "String": "string", + "Integer": "number", + "Decimal": "number", + "Float": "number", + "Boolean": "boolean", + "Date": "time", + "Time": "time", + "DateTime": "time", + "DateTimeTz": "time", + "Opaque": "string", +} + +# Ossie datatypes whose temporal role makes `is_time` default to true (spec.md, +# "DataType and is_time"). +TEMPORAL_DATATYPES = frozenset({"Date", "Time", "DateTime", "DateTimeTz"}) + +# Cube measure `type` -> the Ossie aggregate function that reproduces it. +# `count` is absent: it maps through the cube's primary key, see +# primary_key_count_expression(). +AGG_TO_OSSIE_FUNC = { + "sum": "SUM", + "avg": "AVG", + "min": "MIN", + "max": "MAX", + "count_distinct": "COUNT_DISTINCT", + "count_distinct_approx": "APPROX_COUNT_DISTINCT", +} + +OSSIE_FUNC_TO_AGG = { + "SUM": "sum", + "AVG": "avg", + "MIN": "min", + "MAX": "max", + "COUNT_DISTINCT": "count_distinct", + "APPROX_COUNT_DISTINCT": "count_distinct_approx", +} + +# Cube measure types whose aggregation is written out in the `sql` itself +# (CubeSymbols.isCalculatedMeasureType). Their sql is emitted verbatim. +CALCULATED_MEASURE_TYPES = frozenset({"number", "string", "boolean", "time"}) + +# Aggregates whose value is unaffected by duplicate input rows, so a static Ossie +# expression stays correct even when the relationship graph fans the dataset out. +# `count` belongs here only in its bare form, which maps to COUNT(DISTINCT ). +FANOUT_SAFE_AGGS = frozenset({ + "count_distinct", "count_distinct_approx", "min", "max", +}) + +# Aggregates that over-count under row multiplication. Cube corrects for these at +# query time by deduplicating on the primary key; an Ossie expression cannot. +FANOUT_UNSAFE_AGGS = frozenset({"sum", "avg"}) + +# The Ossie result datatype Cube itself declares for each aggregate. Only the +# count family is listed: those are exactly the aggregates whose result type does +# not depend on the operand. +AGG_TO_RESULT_DATATYPE = { + "count": "Integer", + "count_distinct": "Integer", + "count_distinct_approx": "Integer", +} + + +def primary_key_operand(cube_name, primary_keys): + """The single scalar expression standing for a cube's primary key. + + A composite key is concatenated the same way Cube does it (CAST + CONCAT, in + `primaryKeyCount`); both are REQUIRED functions in the Ossie expression + language, so the result stays portable. + """ + if not primary_keys: + raise ConversionError( + f"Cube '{cube_name}': a bare `type: count` measure needs the cube's " + f"primary key to convert safely, but no dimension declares " + f"`primary_key: true`") + if len(primary_keys) == 1: + return f"{cube_name}.{primary_keys[0]}" + parts = ", ".join(f"CAST({cube_name}.{pk} AS VARCHAR)" for pk in primary_keys) + return f"CONCAT({parts})" + + +def primary_key_count_expression(cube_name, primary_keys, filter_exprs=()): + """The Ossie expression for Cube's bare `type: count` measure. + + Cube renders such a measure as `count()` normally and + `count(distinct )` when the cube sits on the multiplied side of a join + (BaseQuery `primaryKeyCount`). `COUNT(DISTINCT )` equals both -- a primary + key is unique, so the DISTINCT is free when there is no fan-out and + load-bearing when there is -- making it the one static form that is correct in + every join context. + """ + operand = filtered_operand(primary_key_operand(cube_name, primary_keys), + filter_exprs) + return f"COUNT(DISTINCT {operand})" + + +def filtered_operand(operand, filter_sqls): + """Fold Cube measure `filters` into the operand, the way Cube itself does. + + Cube's `applyMeasureFilters` wraps the operand as + `CASE WHEN THEN END` inside the aggregate, + which is the filtered-aggregation idiom the Ossie expression language + endorses. The `ELSE` is omitted, matching Cube. + """ + if not filter_sqls: + return operand + where = " AND ".join(f"({f})" for f in filter_sqls) + return f"CASE WHEN {where} THEN {operand} END" diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py new file mode 100644 index 00000000..51bc336c --- /dev/null +++ b/converters/cube/src/ossie_cube/cli.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the Apache Ossie <-> Cube converter. + + ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] + +`import` converts a Cube data model directory (any `.yml` holding `cubes:` / +`views:`) into an Apache Ossie semantic model; with no `-o` the Ossie YAML goes to +stdout. Conversions that could not carry something across print an issue list to +stderr. + +By default a metric whose value a static Ossie expression cannot keep correct +under row multiplication is refused, mirroring Cube's own refusal to answer such +a query; pass `--no-strict-fanout` to emit it with a recorded issue instead. +""" + +import argparse +import os +import sys + +from ._common import ConversionError +from .cube_to_osi import convert_cube_to_ossie + + +def _build_parser(): + parser = argparse.ArgumentParser( + prog="ossie-cube", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command") + sub.required = True + + imp = sub.add_parser( + "import", help="Cube data model directory -> Apache Ossie semantic model YAML") + imp.add_argument("-i", "--input", required=True, help="Cube model directory") + imp.add_argument("-o", "--output", + help="output Ossie YAML file (default: stdout)") + imp.add_argument("--name", + help="Ossie model name (default: the mapped view's name)") + imp.add_argument("--view", + help="view whose name/description/AI context map onto the " + "Ossie model (default: the sole view, if there is one)") + imp.add_argument("--no-strict-fanout", dest="strict_fanout", + action="store_false", default=True, + help="record fan-out-unsafe metrics as issues instead of " + "refusing the conversion") + return parser + + +def _read_model_dir(path): + """Collect every file under a Cube model directory as {relative path: text}. + + Everything is collected, not just YAML: a `.js` data model has no Ossie form, + but the converter preserves it so a round trip does not lose the file. Hidden + files and directories (including `node_modules`) are skipped. + """ + if not os.path.isdir(path): + raise ConversionError(f"'{path}' is not a directory") + files = {} + for dirpath, dirnames, filenames in os.walk(path): + dirnames[:] = [d for d in sorted(dirnames) + if not d.startswith(".") and d != "node_modules"] + for fname in sorted(filenames): + if fname.startswith("."): + continue + rel = os.path.relpath(os.path.join(dirpath, fname), path) + rel = rel.replace(os.sep, "/") + with open(os.path.join(dirpath, fname)) as fh: + files[rel] = fh.read() + if not files: + raise ConversionError(f"'{path}' holds no files") + return files + + +def _report(issues): + if not len(issues): + return + print(f"{len(issues)} conversion issue(s):", file=sys.stderr) + for issue in issues: + print(f" {issue}", file=sys.stderr) + + +def main(argv=None): + args = _build_parser().parse_args(argv) + try: + files = _read_model_dir(args.input) + out, issues = convert_cube_to_ossie( + files, model_name=args.name, view=args.view, + strict_fanout=args.strict_fanout) + if args.output: + with open(args.output, "w") as fh: + fh.write(out) + else: + sys.stdout.write(out) + _report(issues) + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py new file mode 100644 index 00000000..64dfe4b1 --- /dev/null +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -0,0 +1,110 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Structured record of what a conversion could not carry across. + +A bare `warnings.warn` is fine for "this label was dropped", but Cube carries +semantics that an Apache Ossie expression string genuinely cannot hold -- most +importantly the row-multiplication correction Cube applies at query time (see +`FANOUT_UNSAFE_METRIC`). Those need to reach the caller as data, not as text on +stderr, so a pipeline can gate on them. Same approach as the osi-dbt converter's +`ConverterIssue`. +""" + +from dataclasses import dataclass, field +from enum import Enum + + +class IssueType(Enum): + """Identifies the kind of information loss that occurred during conversion.""" + + # A non-idempotent aggregate (sum/avg, or count over an expression) on a + # dataset that the relationship graph can fan out. Cube corrects for this at + # query time by deduplicating on the primary key; a static Ossie expression + # cannot, so a downstream consumer may over-count. See README "Fan-out". + FANOUT_UNSAFE_METRIC = "FANOUT_UNSAFE_METRIC" + + # A `multi_stage` measure (group_by / reduce_by / time_shift / rank). These + # render as window functions over a grain other than the query's, which an + # Ossie expression has no form for; the measure is preserved in the stash + # and omitted from `metrics`. + MULTI_STAGE_MEASURE_DROPPED = "MULTI_STAGE_MEASURE_DROPPED" + + # A cube-level `meta.ai_context`. Cube's own agent only consumes ai_context + # on views and on individual members, so this value is inert in Cube; it is + # preserved so the round trip stays lossless. + CUBE_LEVEL_AI_CONTEXT_INERT = "CUBE_LEVEL_AI_CONTEXT_INERT" + + # A `type: geo` dimension, split into two Ossie fields (latitude/longitude) + # because an Ossie field holds a single expression. + GEO_DIMENSION_SPLIT = "GEO_DIMENSION_SPLIT" + + # A dimension or measure whose `sql` uses Jinja templating, or a cube using + # `extends`: no static form, so it is preserved in the stash only. + TEMPLATED_MEMBER_DROPPED = "TEMPLATED_MEMBER_DROPPED" + + # An Ossie field or metric with no usable expression dialect (export). + NO_USABLE_DIALECT = "NO_USABLE_DIALECT" + + # An Ossie construct Cube has no slot for, parked under `meta.ossie`. + PARKED_IN_META = "PARKED_IN_META" + + +@dataclass(frozen=True) +class ConverterIssue: + """One instance of information loss, addressed to a named element.""" + + issue_type: IssueType + element_name: str + detail: str = "" + + def __str__(self): + suffix = f": {self.detail}" if self.detail else "" + return f"[{self.issue_type.value}] {self.element_name}{suffix}" + + +@dataclass +class IssueLog: + """Collects issues during a conversion. + + `strict_types` names the issue types that should abort the conversion + instead of being recorded. The CLI puts `FANOUT_UNSAFE_METRIC` in there by + default, mirroring Cube's own refusal to answer a query whose measures + reference cubes that lead to row multiplication. + """ + + issues: list = field(default_factory=list) + strict_types: frozenset = frozenset() + + def add(self, issue_type, element_name, detail=""): + issue = ConverterIssue(issue_type, element_name, detail) + if issue_type in self.strict_types: + # Imported here to avoid a circular import at module load. + from ._common import ConversionError + + raise ConversionError(f"{issue} (refused under strict mode)") + self.issues.append(issue) + return issue + + def of_type(self, issue_type): + return [i for i in self.issues if i.issue_type is issue_type] + + def __len__(self): + return len(self.issues) + + def __iter__(self): + return iter(self.issues) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py new file mode 100644 index 00000000..6e165e8e --- /dev/null +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -0,0 +1,839 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert a Cube data model to an Apache Ossie semantic model. + +Pure offline conversion -- no Cube deployment required. Accepts a Cube model +directory as {relative filename: YAML string}: any `.yml`/`.yaml` file holding +top-level `cubes:` and/or `views:`. Cubes become Ossie datasets, cube joins +become relationships, cube measures are hoisted to model-level metrics, and the +mapped view supplies the model's name, description, and AI context. + +Cube features Ossie has no native field for (segments, pre-aggregations, +hierarchies, folders, view curation, formats, access policies, ...) are preserved +in `custom_extensions[CUBE]` so that converting back reproduces the original +files. See README.md. + +Usage (CLI): + ossie-cube import -i model/ [-o model.yaml] [--name NAME] [--view VIEW] +""" + +import re + +from ._common import ( + AGG_TO_OSSIE_FUNC, + AGG_TO_RESULT_DATATYPE, + CALCULATED_MEASURE_TYPES, + DIALECT_ANSI, + DIM_TYPE_TO_DATATYPE, + DOTTED_REF_RE, + FANOUT_UNSAFE_AGGS, + JINJA_RE, + OSSIE_VERSION, + ConversionError, + cube_file, + cube_sql_to_ossie, + dump_yaml, + filtered_operand, + is_simple_identifier, + join_source, + load_yaml, + primary_key_count_expression, + require_str, + snake, + snake_keys, + view_file, + write_stash, +) +from .converter_issues import IssueLog, IssueType + +# Cube keys the converter maps natively at the cube level; everything else is +# stashed verbatim in the dataset's `cube_extras` and restored on export. +_CUBE_NATIVE_KEYS = frozenset({ + "name", "sql", "sql_table", "description", "dimensions", "measures", + "joins", "meta", +}) + +# Dimension keys mapped natively; the rest stash flat on the field. +_DIM_NATIVE_KEYS = frozenset({ + "name", "sql", "type", "primary_key", "title", "description", "meta", + "latitude", "longitude", +}) + +# Measure keys an Ossie metric represents natively. Any other key forces the +# full-measure stash, because export could not rebuild the measure without it. +_MEASURE_NATIVE_KEYS = frozenset({ + "name", "sql", "type", "filters", "title", "description", "meta", +}) + +# `relationship` values, normalized. Cube accepts the legacy `belongsTo` / +# `hasMany` / `hasOne` spellings alongside the modern ones, in either case style. +_RELATIONSHIP_ALIASES = { + "belongs_to": "many_to_one", + "many_to_one": "many_to_one", + "has_many": "one_to_many", + "one_to_many": "one_to_many", + "has_one": "one_to_one", + "one_to_one": "one_to_one", +} + +_AND_SPLIT_RE = re.compile(r"\s+AND\s+", re.IGNORECASE) + + +def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True): + """Convert Cube model files ({relative filename: YAML str}) to Ossie YAML. + + Returns (ossie_yaml_str, IssueLog). `model_name` overrides the Ossie model + name (default: the mapped view's name, else 'cube_model'). `view` names the + view whose name/description/AI context map onto the Ossie model when the + directory holds more than one. `strict_fanout` refuses metrics whose value a + static Ossie expression cannot keep correct under row multiplication -- see + README "Fan-out". + """ + if not isinstance(files, dict) or not files: + raise ConversionError("expected a non-empty mapping of {filename: YAML}") + + strict = {IssueType.FANOUT_UNSAFE_METRIC} if strict_fanout else set() + issues = IssueLog(strict_types=frozenset(strict)) + + cubes, cube_paths, views, view_paths, extra_files = _collect(files, issues) + if not cubes: + raise ConversionError( + "no convertible cubes found (a `.yml` file with a top-level `cubes:` " + "list); nothing to convert") + + # The mapped view supplies the Ossie model's identity. Cube users are + # view-first, and Cube's own agent reads `meta.ai_context` only from views and + # individual members -- so the view, not any cube, is the model boundary. + mapped_name = _pick_view(views, view, issues) + mapped_view = views.get(mapped_name) or {} + + model = {"name": model_name or mapped_name or "cube_model"} + if mapped_view.get("description"): + model["description"] = mapped_view["description"] + ai = _ai_context_from_meta(mapped_view.get("meta")) + if ai: + model["ai_context"] = ai + + # Joins are decomposed first: a join with no Ossie form is parked on its + # declaring cube's stash, which has to be known before the dataset is built. + relationships, extra_joins = _convert_joins(cubes, issues) + + datasets = [] + pk_by_cube = {} + for cname, cube in cubes.items(): + ds, primary_key = _convert_cube(cname, cube, extra_joins.get(cname), issues) + datasets.append(ds) + pk_by_cube[cname] = primary_key + model["datasets"] = datasets + if relationships: + model["relationships"] = relationships + + # A dataset on the `to` (one) side of a relationship can be fanned out by rows + # from the `from` (many) side. Derived entirely from the Ossie graph. + fanned_out = {rel["to"]: rel["name"] for rel in relationships} + + metrics = _convert_measures(cubes, pk_by_cube, fanned_out, issues) + if metrics: + model["metrics"] = metrics + + # Model-level stash: the views verbatim (minus natively mapped properties), + # the mapped view's identity, non-canonical file paths, and any file with no + # Ossie form. `views` is stashed even when empty, so a lossless re-export does + # not invent a view the original model never had. + stash = {"views": {}} + for vname, vdict in views.items(): + vdict = dict(vdict) + if vname == mapped_name: + vdict.pop("description", None) + leftover = _meta_without_ai_context(vdict.get("meta")) + vdict.pop("meta", None) + if leftover: + vdict["meta"] = leftover + stash["views"][vname] = vdict + off_layout_views = {v: p for v, p in view_paths.items() if p != view_file(v)} + if off_layout_views: + stash["view_files"] = off_layout_views + off_layout_cubes = {c: p for c, p in cube_paths.items() if p != cube_file(c)} + if off_layout_cubes: + stash["cube_files"] = off_layout_cubes + if mapped_name is not None: + stash["mapped_view"] = mapped_name + if extra_files: + stash["extra_files"] = extra_files + write_stash(model, stash) + + return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}), issues + + +# --- collection ----------------------------------------------------------------- + +def _collect(files, issues): + """Partition the input files into cubes, views, and everything else.""" + cubes, views = {}, {} + cube_paths, view_paths = {}, {} + extra_files = {} + for fname in sorted(files): + text = files[fname] + if not fname.lower().endswith((".yml", ".yaml")): + # A `.js`/`.ts` data model needs Cube's own transpiler and a `.py` one + # is Jinja-driven. Preserved verbatim so the round trip keeps the file, + # but no cube inside it is converted. + issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, fname, + "not a YAML data model; preserved in custom_extensions only") + extra_files[fname] = text + continue + if JINJA_RE.search(text): + issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, fname, + "uses Jinja templating, which has no static form; " + "preserved in custom_extensions only") + extra_files[fname] = text + continue + parsed = load_yaml(text, fname) + if not isinstance(parsed, dict) or not ("cubes" in parsed or "views" in parsed): + issues.add(IssueType.PARKED_IN_META, fname, + "no top-level `cubes:` or `views:`; preserved in " + "custom_extensions only") + extra_files[fname] = text + continue + for entry in _as_named_list(parsed.get("cubes"), f"'{fname}' cubes"): + name = require_str(entry, "name", f"'{fname}': cube") + if name in cubes: + raise ConversionError( + f"cube '{name}' is defined twice " + f"('{cube_paths[name]}' and '{fname}')") + if "extends" in entry: + # Resolving `extends` means reproducing Cube's definition-merge + # semantics exactly; refused rather than half-applied. + raise ConversionError( + f"cube '{name}' uses `extends`, which this converter does not " + f"resolve yet; flatten the cube or exclude the file") + cubes[name] = entry + cube_paths[name] = fname + for entry in _as_named_list(parsed.get("views"), f"'{fname}' views"): + name = require_str(entry, "name", f"'{fname}': view") + if name in views: + raise ConversionError( + f"view '{name}' is defined twice " + f"('{view_paths[name]}' and '{fname}')") + views[name] = entry + view_paths[name] = fname + return cubes, cube_paths, views, view_paths, extra_files + + +def _as_named_list(value, what): + """Normalize a Cube collection to a list of dicts carrying `name`. + + YAML data models write `cubes:` / `dimensions:` / `joins:` as lists whose + entries carry a `name`; the JavaScript form (and Cube's post-transpile schema) + uses a mapping keyed by name. Both are accepted, and keys are normalized to + snake_case so the mapping code only has to know one spelling. + """ + if value is None: + return [] + if isinstance(value, list): + out = [] + for entry in value: + if not isinstance(entry, dict): + raise ConversionError( + f"{what}: expected a mapping, got {type(entry).__name__}") + out.append(snake_keys(entry)) + return out + if isinstance(value, dict): + out = [] + for name, entry in value.items(): + entry = snake_keys(entry or {}) + entry.setdefault("name", name) + out.append(entry) + return out + raise ConversionError( + f"{what}: expected a list or mapping, got {type(value).__name__}") + + +def _pick_view(views, requested, issues): + if requested is not None: + if requested not in views: + raise ConversionError( + f"requested view '{requested}' not found; views present: " + f"{sorted(views) or 'none'}") + return requested + if len(views) == 1: + return next(iter(views)) + if len(views) > 1: + issues.add(IssueType.PARKED_IN_META, "model", + f"{len(views)} views found and none chosen with --view; view " + f"metadata is preserved in custom_extensions only") + return None + + +# --- ai_context ----------------------------------------------------------------- + +def _ai_context_from_meta(meta): + """Build an Ossie `ai_context` from a Cube `meta`. + + `meta.ai_context` is Cube's documented AI-only context field. A structured + copy parked by a previous export under `meta.ossie.ai_context` wins, since it + carries the synonyms/examples lists that the prose form flattens. + """ + if not isinstance(meta, dict): + return None + parked = (meta.get("ossie") or {}).get("ai_context") + if parked: + return parked + text = meta.get("ai_context") + if isinstance(text, str) and text.strip(): + return {"instructions": text.strip()} + return None + + +def _meta_without_ai_context(meta): + """The part of a Cube `meta` with no Ossie home, for the stash. + + `meta.ossie` is this converter's own parking spot; its contents are restored + into native Ossie fields, so it never rides in the stash. + """ + if not isinstance(meta, dict): + return {} + return {k: v for k, v in meta.items() if k not in ("ai_context", "ossie")} + + +# --- cubes ---------------------------------------------------------------------- + +def _convert_cube(cname, cube, extra_joins, issues): + """Build one Ossie dataset from a Cube cube. Returns (dataset, primary_key).""" + scope = f"cube '{cname}'" + ds = {"name": cname} + stash = {} + + ds["source"] = join_source(cube, cname) + if cube.get("description"): + ds["description"] = cube["description"] + + meta = cube.get("meta") if isinstance(cube.get("meta"), dict) else {} + parked = meta.get("ossie") or {} + ai = _ai_context_from_meta(meta) + if ai: + ds["ai_context"] = ai + if meta.get("ai_context"): + issues.add(IssueType.CUBE_LEVEL_AI_CONTEXT_INERT, scope, + "Cube's agent reads ai_context only on views and members, " + "so a cube-level value has no effect in Cube") + if parked.get("unique_keys"): + ds["unique_keys"] = [list(k) for k in parked["unique_keys"]] + + fields = [] + primary_key = [] + templated = {} + for dim in _as_named_list(cube.get("dimensions"), f"{scope} dimensions"): + dname = require_str(dim, "name", f"{scope}: dimension") + if JINJA_RE.search(str(dim.get("sql", ""))): + issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, f"{cname}.{dname}", + "dimension sql uses Jinja templating; preserved in " + "custom_extensions only") + templated[dname] = dim + continue + if dim.get("primary_key"): + primary_key.append(dname) + fields.extend(_convert_dimension(cname, dname, dim, issues)) + if fields: + ds["fields"] = fields + if primary_key: + ds["primary_key"] = primary_key + if templated: + stash["extra_dimensions"] = templated + if extra_joins: + stash["extra_joins"] = extra_joins + + extras = {snake(k): v for k, v in cube.items() + if snake(k) not in _CUBE_NATIVE_KEYS} + leftover_meta = _meta_without_ai_context(cube.get("meta")) + if leftover_meta: + extras["meta"] = leftover_meta + if extras: + stash["cube_extras"] = extras + write_stash(ds, stash) + + # Foreign-vendor extensions parked by a previous export are restored after the + # stash is written, so the CUBE entry stays first and both survive. + if parked.get("custom_extensions"): + ds.setdefault("custom_extensions", []).extend(parked["custom_extensions"]) + return ds, primary_key + + +def _convert_dimension(cname, dname, dim, issues): + """Build the Ossie field(s) for one Cube dimension. + + Returns a list because a `type: geo` dimension carries two SQL expressions + (latitude and longitude) where an Ossie field holds one, so it splits into two + fields. Every other dimension yields exactly one. + """ + dtype = snake(dim.get("type") or "string") + if dtype == "geo": + return _convert_geo_dimension(cname, dname, dim, issues) + + stash = {} + sql = dim.get("sql") + if sql is None: + # No `sql` means the same-named physical column. + expr = dname + else: + expr, changed = cube_sql_to_ossie(sql, cname) + if changed or str(sql).strip() == dname: + # Stashed when the Ossie expression differs from the Cube sql, and also + # when the sql is an explicit same-named bare column -- which export + # would otherwise normalize away to the implicit form. + stash["sql"] = sql + + field = { + "name": dname, + "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, + } + datatype = DIM_TYPE_TO_DATATYPE.get(dtype) + if datatype: + field["datatype"] = datatype + elif dtype == "number": + # Cube collapses Integer/Decimal/Float into `number`, so no Ossie datatype + # is asserted -- the spec says to omit it when unknown. The original type + # rides in the stash so export reproduces it. + stash["type"] = dtype + else: + raise ConversionError( + f"cube '{cname}': dimension '{dname}' has unknown type '{dtype}'") + if dtype == "time": + field["dimension"] = {"is_time": True} + if dim.get("title"): + field["label"] = dim["title"] + if dim.get("description"): + field["description"] = dim["description"] + ai = _ai_context_from_meta(dim.get("meta")) + if ai: + field["ai_context"] = ai + + for key, value in dim.items(): + skey = snake(key) + if skey not in _DIM_NATIVE_KEYS: + stash[skey] = value + leftover_meta = _meta_without_ai_context(dim.get("meta")) + if leftover_meta: + stash["meta"] = leftover_meta + write_stash(field, stash) + return [field] + + +def _convert_geo_dimension(cname, dname, dim, issues): + """Split a `type: geo` dimension into a latitude and a longitude field. + + The reconstruction data rides on the latitude half (`geo.host` holds the + dimension's other keys), so export can rebuild the single geo dimension. + """ + issues.add(IssueType.GEO_DIMENSION_SPLIT, f"{cname}.{dname}", + f"split into '{dname}_latitude' and '{dname}_longitude'; an Ossie " + f"field holds a single expression") + host_extras = { + snake(k): v for k, v in dim.items() + if snake(k) not in ("name", "type", "latitude", "longitude") + } + out = [] + for part in ("latitude", "longitude"): + sub = (dim.get(part) or {}).get("sql") + if sub is None: + raise ConversionError( + f"cube '{cname}': geo dimension '{dname}' is missing '{part}.sql'") + expr, _ = cube_sql_to_ossie(sub, cname) + field = { + "name": f"{dname}_{part}", + "expression": { + "dialects": [{"dialect": DIALECT_ANSI, "expression": expr}] + }, + "datatype": "Float", + } + geo = {"of": dname, "part": part, "sql": sub} + if part == "latitude" and host_extras: + geo["host"] = host_extras + write_stash(field, {"geo": geo}) + out.append(field) + return out + + +# --- joins ---------------------------------------------------------------------- + +def _convert_joins(cubes, issues): + """Turn every cube's `joins` into Ossie relationships. + + Ossie's `from` is always the many side. A `many_to_one` join declared on cube + A points A(many) -> B(one) directly; a `one_to_many` join is flipped, and the + declared side and type are stashed so export restores the original. + + Returns (relationships, {cube name: [unconvertible join, ...]}). + """ + relationships = [] + extra_joins = {} + taken = set() + for cname, cube in cubes.items(): + for index, join in enumerate( + _as_named_list(cube.get("joins"), f"cube '{cname}' joins")): + target = require_str(join, "name", f"cube '{cname}': join") + what = f"join '{cname}' -> '{target}'" + if target not in cubes: + raise ConversionError( + f"{what}: '{target}' is not a cube in this model") + raw_rel = snake(require_str(join, "relationship", what)) + rel_type = _RELATIONSHIP_ALIASES.get(raw_rel) + if rel_type is None: + raise ConversionError( + f"{what}: unknown relationship '{join['relationship']}'") + sql = require_str(join, "sql", what) + + pairs = _decompose_join_sql(sql, cname, target, what, issues) + if pairs is None: + extra_joins.setdefault(cname, []).append( + {"index": index, "join": join}) + continue + + from_cube, to_cube = cname, target + from_cols = [p[0] for p in pairs] + to_cols = [p[1] for p in pairs] + stash = {"declared_on": cname, "relationship": raw_rel} + if rel_type == "one_to_many": + from_cube, to_cube = to_cube, from_cube + from_cols, to_cols = to_cols, from_cols + elif rel_type == "one_to_one": + # Neither side multiplies, so Ossie's many/one orientation is not + # meaningful; the declared orientation is kept. + issues.add(IssueType.PARKED_IN_META, what, + "one_to_one has no Ossie orientation; the declared " + "orientation is kept and the type preserved") + if sql != _rebuild_join_sql(target, pairs): + stash["sql"] = sql + for key, value in join.items(): + if snake(key) not in ("name", "sql", "relationship"): + stash[snake(key)] = value + + # Ossie relationship names are unique per model; several joins between + # one cube pair would generate the same `_to_`, so repeats + # are suffixed. Export never reads the name, so this stays lossless. + name = f"{from_cube}_to_{to_cube}" + base, k = name, 2 + while name in taken: + name, k = f"{base}_{k}", k + 1 + taken.add(name) + + rel = {"name": name, "from": from_cube, "to": to_cube, + "from_columns": from_cols, "to_columns": to_cols} + write_stash(rel, stash) + relationships.append(rel) + return relationships, extra_joins + + +def _decompose_join_sql(sql, own_cube, target, what, issues): + """Split a Cube join `sql` into (own_column, target_column) pairs. + + Only an AND-chain of equalities between one own-cube reference and one + target-cube reference has an Ossie relationship form. Anything else -- a + range/non-equi condition, a comparison against a literal, a third cube -- + returns None, and the caller preserves the join in the stash instead. + """ + pairs = [] + for clause in _AND_SPLIT_RE.split(sql): + sides = clause.split("=") + if len(sides) != 2: + issues.add(IssueType.PARKED_IN_META, what, + f"join clause '{clause.strip()}' is not a single equality; " + f"preserved in custom_extensions only") + return None + left = _ref_target(sides[0], own_cube, target) + right = _ref_target(sides[1], own_cube, target) + if left is None or right is None: + issues.add(IssueType.PARKED_IN_META, what, + f"join clause '{clause.strip()}' is not between two member " + f"references; preserved in custom_extensions only") + return None + (lcube, lcol), (rcube, rcol) = left, right + if lcube == own_cube and rcube == target: + pairs.append((lcol, rcol)) + elif lcube == target and rcube == own_cube: + pairs.append((rcol, lcol)) + else: + issues.add(IssueType.PARKED_IN_META, what, + f"join clause '{clause.strip()}' references cubes other than " + f"'{own_cube}'/'{target}'; preserved in custom_extensions only") + return None + return pairs or None + + +def _ref_target(side, own_cube, target): + """Resolve one side of a join equality to (cube_name, column), or None.""" + translated, _ = cube_sql_to_ossie(side, own_cube) + translated = translated.strip() + if is_simple_identifier(translated): + # A bare name came from `{CUBE}.col`, `{CUBE.col}`, or `{col}` -- all of + # which address the cube the join is declared on. + return (own_cube, translated) + m = DOTTED_REF_RE.fullmatch(translated) + if m and m.group(1) in (own_cube, target): + return (m.group(1), m.group(2)) + return None + + +def _rebuild_join_sql(target, pairs): + """The canonical form export emits, used to decide whether the original has to + be stashed. The own side is always `{CUBE}` so the join keeps working when the + cube is extended.""" + return " AND ".join( + "{CUBE}." + own + " = {" + target + "." + other + "}" + for own, other in pairs + ) + + +# --- measures ------------------------------------------------------------------- + +class _MeasureResolver: + """Computes the Ossie expression for a Cube measure. + + Kept as a class because a calculated measure (`type: number`, and the other + types in `CALCULATED_MEASURE_TYPES`) can reference other measures, which Cube + resolves by inlining their full aggregate SQL -- so producing one measure's + expression may require producing another's first. Results are memoized and + reference cycles are rejected rather than recursed into. + """ + + def __init__(self, cubes, pk_by_cube, issues): + self._pk = pk_by_cube + self._issues = issues + self._raw = {} + self._dimensions = {} + for cname, cube in cubes.items(): + for m in _as_named_list(cube.get("measures"), f"cube '{cname}' measures"): + self._raw[(cname, require_str(m, "name", f"cube '{cname}': measure"))] = m + self._dimensions[cname] = { + d["name"] + for d in _as_named_list(cube.get("dimensions"), + f"cube '{cname}' dimensions") + } + + def measures(self): + return self._raw + + def is_measure(self, cube, name): + return (cube, name) in self._raw + + def aggregate_of(self, cname, mname): + """The normalized Cube `type` of a measure.""" + return snake(self._raw[(cname, mname)].get("type") or "") + + def expression(self, cname, mname, stack=()): + """The Ossie expression reproducing this measure, or None when the measure + has no static form (multi-stage, Jinja-templated).""" + key = (cname, mname) + if key in stack: + chain = " -> ".join(f"{c}.{m}" for c, m in stack + (key,)) + raise ConversionError(f"measure reference cycle: {chain}") + measure = self._raw[key] + scope = f"{cname}.{mname}" + mtype = snake(measure.get("type") or "") + if not mtype: + raise ConversionError(f"measure '{scope}': missing required 'type'") + + if measure.get("multi_stage"): + # group_by / reduce_by / time_shift / rank render as window functions + # over a grain other than the query's; Ossie has no form for that. + self._issues.add( + IssueType.MULTI_STAGE_MEASURE_DROPPED, scope, + f"multi_stage measure (type '{mtype}'); preserved in " + f"custom_extensions only") + return None + if JINJA_RE.search(str(measure.get("sql", ""))): + self._issues.add( + IssueType.TEMPLATED_MEMBER_DROPPED, scope, + "measure sql uses Jinja templating; preserved in " + "custom_extensions only") + return None + + sql = measure.get("sql") + filter_exprs = [ + self._translate(f["sql"], cname, stack + (key,)) + for f in (measure.get("filters") or []) + if isinstance(f, dict) and f.get("sql") + ] + + if mtype in CALCULATED_MEASURE_TYPES: + if sql is None: + raise ConversionError( + f"measure '{scope}': type '{mtype}' requires 'sql'") + expr = self._translate(sql, cname, stack + (key,)) + return filtered_operand(expr, filter_exprs) + if mtype == "count": + if sql is None: + return primary_key_count_expression( + cname, self._pk.get(cname) or [], filter_exprs) + operand = filtered_operand( + self._operand(cname, sql, stack + (key,)), filter_exprs) + return f"COUNT({operand})" + func = AGG_TO_OSSIE_FUNC.get(mtype) + if func is None: + raise ConversionError( + f"measure '{scope}': unknown aggregate type '{mtype}'") + if sql is None: + raise ConversionError( + f"measure '{scope}': type '{mtype}' requires 'sql'") + operand = filtered_operand( + self._operand(cname, sql, stack + (key,)), filter_exprs) + return (f"COUNT(DISTINCT {operand})" if func == "COUNT_DISTINCT" + else f"{func}({operand})") + + def _translate(self, sql, cname, stack): + """Translate a Cube SQL string, inlining any measure reference. + + `self_prefix` is the owning cube: Ossie metrics are model-level, so a + column reads as `dataset.column` here, unlike in a dataset-scoped field + expression. + """ + out, _ = cube_sql_to_ossie( + sql, cname, resolve_ref=lambda body: self._inline(body, cname, stack), + self_prefix=cname) + return out + + def _inline(self, body, cname, stack): + """Resolve one `{...}` body when it names a measure, else fall through. + + Cube inlines a measure reference to that measure's own aggregate SQL + (`isCalculatedMeasureType` emits the sql as-is), so `{revenue} / {count}` + becomes a complete ratio expression -- which is exactly the shape Ossie + metrics use. Parenthesized to keep the referenced measure's precedence. + """ + head, _, rest = body.partition(".") + if rest: + target_cube = cname if head in ("CUBE", "TABLE") else head + target_name = rest + else: + target_cube, target_name = cname, body + if not self.is_measure(target_cube, target_name): + return None + inner = self.expression(target_cube, target_name, stack) + if inner is None: + raise ConversionError( + f"measure '{cname}': references '{target_cube}.{target_name}', " + f"which has no static Ossie form") + return f"({inner})" + + def _operand(self, cname, sql, stack): + """Translate an aggregate's operand into an Ossie reference. + + A same-cube member or bare column becomes `cube.name` -- the qualified form + Ossie model-level metrics use. A computed operand keeps its own qualifiers + and is emitted as-is; the owning cube rides in the stash either way, so + export still puts the measure back on the right cube. + """ + translated = self._translate(sql, cname, stack).strip() + if is_simple_identifier(translated): + return f"{cname}.{translated}" + return translated + + +def _convert_measures(cubes, pk_by_cube, fanned_out, issues): + """Hoist every cube's measures into Ossie model-level metrics. + + A metric name is the measure name when globally unique, else + `__`; the original name and owning cube are stashed so export + puts the measure back where it came from. + """ + resolver = _MeasureResolver(cubes, pk_by_cube, issues) + + counts = {} + for (_cname, mname) in resolver.measures(): + counts[mname] = counts.get(mname, 0) + 1 + + metrics = [] + seen = set() + for cname, cube in cubes.items(): + for measure in _as_named_list(cube.get("measures"), + f"cube '{cname}' measures"): + mname = measure["name"] + metric_name = mname if counts[mname] == 1 else f"{cname}__{mname}" + if metric_name in seen: + raise ConversionError( + f"metric name '{metric_name}' derived twice; rename the " + f"colliding measures in Cube") + seen.add(metric_name) + metric = _convert_measure(cname, mname, metric_name, measure, resolver, + fanned_out, issues) + if metric is not None: + metrics.append(metric) + return metrics + + +def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, + issues): + scope = f"{cname}.{mname}" + expr = resolver.expression(cname, mname) + if expr is None: + # No static form; the resolver already recorded why. + return None + mtype = resolver.aggregate_of(cname, mname) + sql = measure.get("sql") + + # Reconstructible = export can rebuild this measure from the Ossie expression + # alone. A calculated measure never is: export would re-parse its expression + # into a structured measure, and the inlined references cannot be un-inlined. + # Neither is a filtered one -- recovering `filters` would mean parsing the + # folded CASE back apart, so the original rides along instead. + reconstructible = ( + {snake(k) for k in measure} <= _MEASURE_NATIVE_KEYS + and mtype not in CALCULATED_MEASURE_TYPES + and not measure.get("filters") + ) + + # Fan-out: a non-idempotent aggregate on a dataset the graph can multiply. + # Cube fixes this at query time by deduplicating on the primary key; a static + # expression cannot, so the caller has to be told. + unsafe = mtype in FANOUT_UNSAFE_AGGS or (mtype == "count" and sql is not None) + if unsafe and cname in fanned_out: + issues.add( + IssueType.FANOUT_UNSAFE_METRIC, scope, + f"'{mtype}' over dataset '{cname}', which relationship " + f"'{fanned_out[cname]}' fans out; Cube deduplicates on the primary key " + f"at query time but a static Ossie expression cannot, so a consumer " + f"joining through that relationship may over-count") + + metric = { + "name": metric_name, + "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, + } + datatype = AGG_TO_RESULT_DATATYPE.get(mtype) + if datatype: + metric["datatype"] = datatype + if measure.get("description"): + metric["description"] = measure["description"] + ai = _ai_context_from_meta(measure.get("meta")) + if ai: + metric["ai_context"] = ai + + stash = {"cube": cname} + if not reconstructible: + stash["measure"] = { + snake(k): v for k, v in measure.items() + if snake(k) not in ("description", "meta") + } + if metric_name != mname: + stash["name"] = mname + if measure.get("title"): + stash["title"] = measure["title"] + leftover_meta = _meta_without_ai_context(measure.get("meta")) + if leftover_meta: + stash["meta"] = leftover_meta + write_stash(metric, stash) + return metric diff --git a/converters/cube/tests/_util.py b/converters/cube/tests/_util.py new file mode 100644 index 00000000..feb7e253 --- /dev/null +++ b/converters/cube/tests/_util.py @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared test helpers: fixture loading and structural lookup.""" + +import copy +import json +import pathlib + +from ossie_cube._common import load_yaml # src is on sys.path via conftest.py + +FIXTURES = pathlib.Path(__file__).resolve().parent / "fixtures" +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] + + +def load_fixture(name): + with open(FIXTURES / name) as fh: + return fh.read() + + +def load_fixture_dir(name): + """Read a fixture Cube model directory as {relative posix path: text}.""" + root = FIXTURES / name + files = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + files[path.relative_to(root).as_posix()] = path.read_text() + return files + + +def parse(yaml_str): + return load_yaml(yaml_str) + + +def model_of(ossie_yaml): + """The sole semantic model of an Ossie document.""" + doc = parse(ossie_yaml) + assert len(doc["semantic_model"]) == 1 + return doc["semantic_model"][0] + + +def by_name(items): + """Index a list of named Ossie objects by `name`.""" + return {item["name"]: item for item in items or []} + + +def expr_of(item, dialect="ANSI_SQL"): + """The expression string of an Ossie field or metric in a given dialect.""" + for entry in item["expression"]["dialects"]: + if entry["dialect"] == dialect: + return entry["expression"] + raise AssertionError(f"{item['name']} has no {dialect} expression") + + +def stash_of(item, vendor="CUBE"): + """The parsed vendor stash on an Ossie object, or {} when absent.""" + for ext in item.get("custom_extensions") or []: + if ext["vendor_name"] == vendor: + data = json.loads(ext["data"]) + data.pop("_v", None) + return data + return {} + + +def canon(obj): + """Deep-copy with every `custom_extensions[].data` JSON string parsed into a + dict, so comparisons are insensitive to JSON key order and whitespace.""" + obj = copy.deepcopy(obj) + + def walk(node): + if isinstance(node, dict): + for key, value in node.items(): + if key == "custom_extensions" and isinstance(value, list): + for ext in value: + if isinstance(ext, dict) and isinstance(ext.get("data"), str): + ext["data"] = json.loads(ext["data"]) + else: + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(obj) + return obj diff --git a/converters/cube/tests/conftest.py b/converters/cube/tests/conftest.py new file mode 100644 index 00000000..254b3d75 --- /dev/null +++ b/converters/cube/tests/conftest.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib +import sys + +# Make the converter modules in ../src, and this directory's own helpers, +# importable from the tests. +_HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(_HERE)) +sys.path.insert(0, str(_HERE.parent / "src")) diff --git a/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml new file mode 100644 index 00000000..a28395ab --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The fact cube: sits on the many side of the join, so its sum/avg measures are +# not exposed to row multiplication. Exercises a bare `count` (which maps through +# the primary key), a filtered sum, a calculated measure that references two other +# measures, and `meta.ai_context` on both a dimension and a measure. +cubes: + - name: orders + sql_table: public.orders + description: Customer orders + joins: + - name: users + sql: "{CUBE}.user_id = {users}.id" + relationship: many_to_one + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: user_id + sql: user_id + type: number + - name: status + sql: status + type: string + title: Order Status + description: Current order status + meta: + ai_context: Values are pending, shipped, and completed. + - name: created_at + sql: created_at + type: time + - name: is_large + sql: "{CUBE}.amount > 500" + type: boolean + measures: + - name: count + type: count + - name: total_amount + sql: "{CUBE}.amount" + type: sum + description: Total order amount + format: currency + meta: + ai_context: Use this for revenue questions. + - name: completed_amount + sql: "{CUBE}.amount" + type: sum + filters: + - sql: "{CUBE}.status = 'completed'" + - name: avg_order_value + sql: "{total_amount} / {count}" + type: number diff --git a/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml new file mode 100644 index 00000000..d7b59532 --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The dimension cube: sits on the one side of the join, so it is exposed to row +# multiplication -- both of its measures are deliberately fan-out-safe (a bare +# `count`, which maps to COUNT(DISTINCT ), and a count_distinct). Also +# exercises a `sql`-defined cube, a geo dimension (which splits into two Ossie +# fields), and a segment (which has no Ossie form and rides in the stash). +cubes: + - name: users + sql: SELECT * FROM public.users WHERE deleted_at IS NULL + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: city + sql: city + type: string + - name: location + type: geo + latitude: + sql: "{CUBE}.lat" + longitude: + sql: "{CUBE}.lon" + measures: + - name: count + type: count + - name: cities + sql: "{CUBE}.city" + type: count_distinct + segments: + - name: active + sql: "{CUBE}.status = 'active'" diff --git a/converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml b/converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml new file mode 100644 index 00000000..b6f0d584 --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The sole view, so it is the mapped one: its name, description, and +# meta.ai_context become the Ossie model's. The `cubes:` curation has no Ossie +# form and round-trips through the model-level stash. +views: + - name: sales + description: Sales overview + meta: + ai_context: > + Primary view for revenue analysis. Use it for any question about + sales, orders, or customer spend. + cubes: + - join_path: orders + includes: "*" + - join_path: orders.users + includes: + - city diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py new file mode 100644 index 00000000..15e07a95 --- /dev/null +++ b/converters/cube/tests/test_cube_to_osi.py @@ -0,0 +1,518 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Cube data model -> Apache Ossie semantic model.""" + +import pytest +from _util import by_name, expr_of, load_fixture_dir, model_of, stash_of + +from ossie_cube import ConversionError, IssueType, convert_cube_to_ossie +from ossie_cube._common import OSSIE_VERSION, cube_sql_to_ossie + + +@pytest.fixture +def fixture_a(): + return load_fixture_dir("fixtureA_cube") + + +@pytest.fixture +def model_a(fixture_a): + out, issues = convert_cube_to_ossie(fixture_a) + return model_of(out), issues + + +# --- model identity ------------------------------------------------------------- + +def test_version_and_single_model(fixture_a): + out, _ = convert_cube_to_ossie(fixture_a) + from _util import parse + + doc = parse(out) + assert doc["version"] == OSSIE_VERSION + assert len(doc["semantic_model"]) == 1 + + +def test_mapped_view_supplies_model_identity(model_a): + model, _ = model_a + assert model["name"] == "sales" + assert model["description"] == "Sales overview" + assert "revenue analysis" in model["ai_context"]["instructions"] + + +def test_model_name_override(fixture_a): + out, _ = convert_cube_to_ossie(fixture_a, model_name="custom") + assert model_of(out)["name"] == "custom" + + +def test_unknown_view_is_rejected(fixture_a): + with pytest.raises(ConversionError, match="not found"): + convert_cube_to_ossie(fixture_a, view="nope") + + +def test_view_curation_rides_in_the_stash(model_a): + model, _ = model_a + stash = stash_of(model) + assert stash["mapped_view"] == "sales" + # The natively mapped description/ai_context are stripped; the curation stays. + view = stash["views"]["sales"] + assert "description" not in view + assert "meta" not in view + assert view["cubes"][0]["join_path"] == "orders" + + +# --- datasets ------------------------------------------------------------------- + +def test_cubes_become_datasets(model_a): + model, _ = model_a + datasets = by_name(model["datasets"]) + assert set(datasets) == {"orders", "users"} + assert datasets["orders"]["source"] == "public.orders" + assert datasets["orders"]["description"] == "Customer orders" + # A `sql`-defined cube keeps its query as the source. + assert datasets["users"]["source"].startswith("SELECT * FROM public.users") + + +def test_primary_key_from_dimension_flag(model_a): + model, _ = model_a + datasets = by_name(model["datasets"]) + assert datasets["orders"]["primary_key"] == ["id"] + assert datasets["users"]["primary_key"] == ["id"] + + +def test_segments_have_no_ossie_form_and_are_stashed(model_a): + model, _ = model_a + users = by_name(model["datasets"])["users"] + segments = stash_of(users)["cube_extras"]["segments"] + assert segments[0]["name"] == "active" + + +# --- fields --------------------------------------------------------------------- + +def test_dimension_types_map_to_datatypes(model_a): + model, _ = model_a + fields = by_name(by_name(model["datasets"])["orders"]["fields"]) + assert fields["status"]["datatype"] == "String" + assert fields["is_large"]["datatype"] == "Boolean" + assert fields["created_at"]["datatype"] == "DateTime" + assert fields["created_at"]["dimension"]["is_time"] is True + + +def test_number_dimension_asserts_no_datatype(model_a): + """Cube collapses Integer/Decimal/Float into `number`, so the converter omits + `datatype` rather than assert a precision the model does not carry.""" + model, _ = model_a + fields = by_name(by_name(model["datasets"])["orders"]["fields"]) + assert "datatype" not in fields["id"] + assert stash_of(fields["id"])["type"] == "number" + + +def test_dimension_title_becomes_label_and_ai_context_maps(model_a): + model, _ = model_a + status = by_name(by_name(model["datasets"])["orders"]["fields"])["status"] + assert status["label"] == "Order Status" + assert status["description"] == "Current order status" + assert status["ai_context"]["instructions"].startswith("Values are pending") + + +def test_cube_reference_is_stripped_in_a_field_expression(model_a): + """Field expressions are dataset-scoped, so `{CUBE}.amount` reads as `amount`.""" + model, _ = model_a + is_large = by_name(by_name(model["datasets"])["orders"]["fields"])["is_large"] + assert expr_of(is_large) == "amount > 500" + + +def test_geo_dimension_splits_into_two_fields(model_a): + model, issues = model_a + fields = by_name(by_name(model["datasets"])["users"]["fields"]) + assert "location" not in fields + assert expr_of(fields["location_latitude"]) == "lat" + assert expr_of(fields["location_longitude"]) == "lon" + assert fields["location_latitude"]["datatype"] == "Float" + assert stash_of(fields["location_latitude"])["geo"]["of"] == "location" + assert issues.of_type(IssueType.GEO_DIMENSION_SPLIT) + + +# --- relationships -------------------------------------------------------------- + +def test_many_to_one_join_becomes_a_relationship(model_a): + model, _ = model_a + rel = by_name(model["relationships"])["orders_to_users"] + assert rel["from"] == "orders" + assert rel["to"] == "users" + assert rel["from_columns"] == ["user_id"] + assert rel["to_columns"] == ["id"] + # The declaring side and the exact Cube spelling round-trip via the stash. + assert stash_of(rel)["declared_on"] == "orders" + assert stash_of(rel)["relationship"] == "many_to_one" + + +def test_one_to_many_join_is_flipped_to_many_side_first(): + """Ossie's `from` is always the many side, so a join declared as one_to_many on + the one side is flipped -- and the declared orientation stashed.""" + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: orders\n" + " sql: \"{CUBE}.id = {orders}.user_id\"\n" + " relationship: one_to_many\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + ) + } + out, _ = convert_cube_to_ossie(files) + rel = model_of(out)["relationships"][0] + assert rel["from"] == "orders" + assert rel["to"] == "users" + assert rel["from_columns"] == ["user_id"] + assert rel["to_columns"] == ["id"] + assert stash_of(rel)["relationship"] == "one_to_many" + assert stash_of(rel)["declared_on"] == "users" + + +def test_non_equi_join_is_preserved_not_guessed_at(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: rates\n" + " sql: \"{CUBE}.day >= {rates}.valid_from\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: day\n" + " sql: day\n" + " type: time\n" + " - name: rates\n" + " sql_table: public.rates\n" + " dimensions:\n" + " - name: valid_from\n" + " sql: valid_from\n" + " type: time\n" + ) + } + out, issues = convert_cube_to_ossie(files) + model = model_of(out) + assert "relationships" not in model + orders = by_name(model["datasets"])["orders"] + assert stash_of(orders)["extra_joins"][0]["join"]["name"] == "rates" + assert issues.of_type(IssueType.PARKED_IN_META) + + +def test_join_to_unknown_cube_is_rejected(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: ghosts\n" + " sql: \"{CUBE}.id = {ghosts}.id\"\n" + " relationship: many_to_one\n" + ) + } + with pytest.raises(ConversionError, match="not a cube in this model"): + convert_cube_to_ossie(files) + + +# --- metrics -------------------------------------------------------------------- + +def test_measures_are_hoisted_and_disambiguated(model_a): + """`count` exists on both cubes, so both are qualified and the original names + stashed; a globally unique measure keeps its own name.""" + model, _ = model_a + metrics = by_name(model["metrics"]) + assert "orders__count" in metrics + assert "users__count" in metrics + assert stash_of(metrics["orders__count"])["name"] == "count" + assert stash_of(metrics["orders__count"])["cube"] == "orders" + assert "total_amount" in metrics + + +def test_bare_count_maps_through_the_primary_key(model_a): + """Cube renders a bare `count` as count(pk), and count(distinct pk) when the + cube is fanned out. COUNT(DISTINCT pk) equals both, so it is the one static + form that stays correct in every join context.""" + model, _ = model_a + metrics = by_name(model["metrics"]) + assert expr_of(metrics["orders__count"]) == "COUNT(DISTINCT orders.id)" + assert expr_of(metrics["users__count"]) == "COUNT(DISTINCT users.id)" + assert metrics["orders__count"]["datatype"] == "Integer" + + +def test_bare_count_without_a_primary_key_is_rejected(): + """The primary key is load-bearing for a correct `count`, so its absence is an + error rather than a silently-different number.""" + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: status\n" + " sql: status\n" + " type: string\n" + " measures:\n" + " - name: count\n" + " type: count\n" + ) + } + with pytest.raises(ConversionError, match="primary key"): + convert_cube_to_ossie(files) + + +def test_aggregate_measures_become_qualified_expressions(model_a): + model, _ = model_a + metrics = by_name(model["metrics"]) + assert expr_of(metrics["total_amount"]) == "SUM(orders.amount)" + assert expr_of(metrics["cities"]) == "COUNT(DISTINCT users.city)" + assert metrics["total_amount"]["description"] == "Total order amount" + assert metrics["total_amount"]["ai_context"]["instructions"].startswith("Use this") + + +def test_measure_filters_fold_into_a_case_expression(model_a): + """Cube's own applyMeasureFilters wraps the operand as + CASE WHEN THEN END inside the aggregate.""" + model, _ = model_a + metric = by_name(model["metrics"])["completed_amount"] + assert expr_of(metric) == ( + "SUM(CASE WHEN (orders.status = 'completed') THEN orders.amount END)") + + +def test_filtered_and_calculated_measures_keep_the_original(model_a): + """Export cannot recover `filters` from the folded CASE, nor un-inline a + calculated measure's references, so both keep the original measure verbatim -- + which is what makes Cube -> Ossie -> Cube lossless.""" + model, _ = model_a + metrics = by_name(model["metrics"]) + assert stash_of(metrics["completed_amount"])["measure"]["filters"] + assert stash_of(metrics["avg_order_value"])["measure"]["sql"] == ( + "{total_amount} / {count}") + # A plain aggregate needs no such copy. + assert "measure" not in stash_of(metrics["orders__count"]) + + +def test_calculated_measure_inlines_its_measure_references(model_a): + """Cube resolves `{total_amount} / {count}` to the referenced measures' own + aggregate SQL; Ossie has no metric-to-metric reference, so it is inlined.""" + model, _ = model_a + metric = by_name(model["metrics"])["avg_order_value"] + assert expr_of(metric) == "(SUM(orders.amount)) / (COUNT(DISTINCT orders.id))" + + +def test_measure_reference_cycle_is_rejected(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " measures:\n" + " - name: a\n" + " sql: \"{b} + 1\"\n" + " type: number\n" + " - name: b\n" + " sql: \"{a} + 1\"\n" + " type: number\n" + ) + } + with pytest.raises(ConversionError, match="cycle"): + convert_cube_to_ossie(files) + + +def test_multi_stage_measure_is_dropped_with_an_issue(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " measures:\n" + " - name: rolling\n" + " sql: amount\n" + " type: sum\n" + " multi_stage: true\n" + ) + } + out, issues = convert_cube_to_ossie(files) + assert "metrics" not in model_of(out) + assert issues.of_type(IssueType.MULTI_STAGE_MEASURE_DROPPED) + + +# --- fan-out -------------------------------------------------------------------- + +_FANOUT_MODEL = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: lifetime_value\n" + " sql: \"{CUBE}.ltv\"\n" + " type: sum\n" + ) +} + + +def test_fanout_unsafe_metric_is_refused_by_default(): + """`users` is the one side of a many-to-one join, so summing over it after the + join over-counts. Cube deduplicates on the primary key at query time; a static + Ossie expression cannot, so the default is to refuse rather than emit a number + that silently disagrees with Cube.""" + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(_FANOUT_MODEL) + + +def test_fanout_unsafe_metric_is_recorded_when_not_strict(): + out, issues = convert_cube_to_ossie(_FANOUT_MODEL, strict_fanout=False) + metric = by_name(model_of(out)["metrics"])["lifetime_value"] + assert expr_of(metric) == "SUM(users.ltv)" + recorded = issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + assert len(recorded) == 1 + assert recorded[0].element_name == "users.lifetime_value" + assert "over-count" in recorded[0].detail + + +def test_idempotent_aggregates_are_never_flagged(fixture_a): + """count / count_distinct / min / max are unaffected by duplicate rows, so a + fanned-out dataset carrying only those converts cleanly under strict mode.""" + _, issues = convert_cube_to_ossie(fixture_a) + assert not issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + + +# --- rejections and preservation ------------------------------------------------ + +def test_jinja_templated_file_is_preserved_not_parsed(): + files = { + "model/cubes/dyn.yml": "cubes:\n - name: o{{ suffix }}\n sql_table: t\n", + "model/cubes/ok.yml": ( + "cubes:\n - name: orders\n sql_table: public.orders\n"), + } + out, issues = convert_cube_to_ossie(files) + model = model_of(out) + assert by_name(model["datasets"]).keys() == {"orders"} + assert "model/cubes/dyn.yml" in stash_of(model)["extra_files"] + assert issues.of_type(IssueType.TEMPLATED_MEMBER_DROPPED) + + +def test_javascript_model_is_preserved_not_parsed(): + files = { + "model/cubes/orders.js": "cube(`orders`, { sql_table: `public.orders` });", + "model/cubes/ok.yml": ( + "cubes:\n - name: orders_yaml\n sql_table: public.orders\n"), + } + out, issues = convert_cube_to_ossie(files) + assert "model/cubes/orders.js" in stash_of(model_of(out))["extra_files"] + assert issues.of_type(IssueType.TEMPLATED_MEMBER_DROPPED) + + +def test_extends_is_refused_rather_than_half_resolved(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: base\n" + " sql_table: public.orders\n" + " - name: derived\n" + " extends: base\n" + ) + } + with pytest.raises(ConversionError, match="extends"): + convert_cube_to_ossie(files) + + +def test_cube_without_a_source_is_rejected(): + files = {"model/cubes/m.yml": "cubes:\n - name: orders\n description: x\n"} + with pytest.raises(ConversionError, match="neither 'sql' nor 'sql_table'"): + convert_cube_to_ossie(files) + + +def test_cube_with_both_sources_is_rejected(): + files = { + "model/cubes/m.yml": ( + "cubes:\n - name: orders\n sql: SELECT 1\n sql_table: t\n") + } + with pytest.raises(ConversionError, match="exactly one"): + convert_cube_to_ossie(files) + + +def test_duplicate_cube_name_is_rejected(): + files = { + "model/cubes/a.yml": "cubes:\n - name: orders\n sql_table: a\n", + "model/cubes/b.yml": "cubes:\n - name: orders\n sql_table: b\n", + } + with pytest.raises(ConversionError, match="defined twice"): + convert_cube_to_ossie(files) + + +def test_model_with_no_cubes_is_rejected(): + with pytest.raises(ConversionError, match="no convertible cubes"): + convert_cube_to_ossie({"README.md": "not a model"}) + + +# --- reference translation ------------------------------------------------------ + +@pytest.mark.parametrize("sql,expected", [ + ("{CUBE}.status", "status"), + ("{TABLE}.status", "status"), + ("{CUBE.status}", "status"), + ("{status}", "status"), + ("{orders.status}", "status"), + ("{users.city}", "users.city"), + ("${CUBE}.status", "status"), + ("LOWER({CUBE}.email)", "LOWER(email)"), + (r"'\{literal\}'", "'{literal}'"), +]) +def test_reference_translation_in_a_field_context(sql, expected): + """A field expression is dataset-scoped, so own-cube references reduce to a + bare name. `\\{` stays a literal brace.""" + assert cube_sql_to_ossie(sql, "orders")[0] == expected + + +@pytest.mark.parametrize("sql,expected", [ + ("{CUBE}.amount", "orders.amount"), + ("{CUBE.amount}", "orders.amount"), + ("{amount}", "orders.amount"), + ("{users.city}", "users.city"), +]) +def test_reference_translation_in_a_metric_context(sql, expected): + """A metric expression is model-level, so own-cube references are qualified.""" + assert cube_sql_to_ossie(sql, "orders", self_prefix="orders")[0] == expected diff --git a/converters/cube/uv.lock b/converters/cube/uv.lock new file mode 100644 index 00000000..c213eb6b --- /dev/null +++ b/converters/cube/uv.lock @@ -0,0 +1,216 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "apache-ossie-cube" +version = "0.2.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "hypothesis" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] + +[package.metadata.requires-dev] +dev = [ + { name = "hypothesis", specifier = ">=6.0" }, + { name = "pytest", specifier = ">=8.0" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.163.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/08/4cbfa0327e9df00f57fc67f91847add2f0dd6c23408935273b095ee9a9f1/hypothesis-6.163.0.tar.gz", hash = "sha256:520480d4bd3a17557616c25923640953e360332c89d012fffcebd69857e674a9", size = 490145, upload-time = "2026-07-28T07:16:46.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/b1/3eea7a422de342bd0095a9672c3e03fe0466e4561a55499a1e01b4a9f098/hypothesis-6.163.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:331906cb029b6b360b8ebac3ec00c3cfa720037fe2efb294a503a1979c9a9a8f", size = 769898, upload-time = "2026-07-28T07:15:19.323Z" }, + { url = "https://files.pythonhosted.org/packages/bc/24/45b5c948c76c16ecf1e4ad1ca4a4a3fce55b3317ee172bc8230ff2956ae1/hypothesis-6.163.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:b4ad2134405d5345434c22dea96bbc12c85abcfc3c253a8063dbc9ff01164555", size = 765438, upload-time = "2026-07-28T07:16:04.69Z" }, + { url = "https://files.pythonhosted.org/packages/0f/72/7725039a75b3679dc445a169026b11860a0e67a600c4ffed49a42040a000/hypothesis-6.163.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50073f8e63c1e7d3403899755657a990d8bba7b5b5bff66b1c56796d4969bb28", size = 1094699, upload-time = "2026-07-28T07:15:56.668Z" }, + { url = "https://files.pythonhosted.org/packages/dd/fa/bcfa3879f303a302ec6e5f4d35b583924867a0821b42fa275f440f3b8dab/hypothesis-6.163.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8c5d1e6bad47edf6fb1d7406cf6d67314ac08325c63a49550d782a4596ea302b", size = 1123315, upload-time = "2026-07-28T07:16:40.381Z" }, + { url = "https://files.pythonhosted.org/packages/5f/7f/e7fe2f0658db5182bb1d4b266d17f8f81cf3db82eeba27677ddfea13ac09/hypothesis-6.163.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ec3b709508ccd835d8ded1db025b7800618f2289a22a6bfd4927da5f4eb33c", size = 1144220, upload-time = "2026-07-28T07:15:20.617Z" }, + { url = "https://files.pythonhosted.org/packages/90/14/c26f93a4693bfd83d7ba14b7043d986458026f6635d1b157bc2f1c56a59e/hypothesis-6.163.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:7cb3d927360fe73f9a06d646e6082237142ee39c24679c7133d22bf06dd03b45", size = 1099527, upload-time = "2026-07-28T07:16:17.955Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f5/4c1dde28d2e18e169dd6c471ef4bcf12abf3c6a3f4a1744bdabdbdf411ba/hypothesis-6.163.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3f3cceb4720a39127622fbf3bcebe1775b894372c53b5edddfdef10bbdeef9ec", size = 1136272, upload-time = "2026-07-28T07:16:38.182Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5b/54153509ed42cc17e1b65efe34a0c781f42fb04c902dc5f25486c537ec88/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59f5fdb8addb44c17520a60d50542d9db6ceba577bbf54efefa9c10ee20be140", size = 1268518, upload-time = "2026-07-28T07:15:42.991Z" }, + { url = "https://files.pythonhosted.org/packages/81/86/f86f0d15d91b9cbff3546c1b8891950b2b08c0527e7494133fb2180e1b8b/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f1fe222f50a1898e87a1e7323ab35f9e956278efabe4dd55a1342808206d05ad", size = 1396357, upload-time = "2026-07-28T07:15:26.696Z" }, + { url = "https://files.pythonhosted.org/packages/46/3a/c096b6b272f15e17e8e65c6ccfb2e1d167456ff5bbd9db33dca3185199f6/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:56ed585baab75cb98462c57ca88bbdc6a9d935a14118dd572fb476c3ecec2a06", size = 1269128, upload-time = "2026-07-28T07:16:11.902Z" }, + { url = "https://files.pythonhosted.org/packages/bb/9f/0807445874b3083a22c9a14a0ffc31bd0060ef3b408ce4cb31c20779cdf8/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2849c23b2e0fe2eef4c1ec336b01eac7ad7397c49fca43c264f59ec1e6046eac", size = 1311189, upload-time = "2026-07-28T07:16:08.545Z" }, + { url = "https://files.pythonhosted.org/packages/d6/de/3319aa8fcffd1641defc1c5eea1ad646a33d1b62528ef487a89752bc8079/hypothesis-6.163.0-cp310-abi3-win32.whl", hash = "sha256:b2ddcdaf6691101e06dc4a5add7b8c8fdf1e68daba599255a281f3f3550d3331", size = 655743, upload-time = "2026-07-28T07:16:30.966Z" }, + { url = "https://files.pythonhosted.org/packages/8e/48/36bc72910451e6e88b75e59a6ddbf0db34ff61a3b11c9439801dfd5fec20/hypothesis-6.163.0-cp310-abi3-win_amd64.whl", hash = "sha256:4ab0dadc09c537d4ac57e564039dfe7daf09c98375306d54bfc0fd6c218efcca", size = 661902, upload-time = "2026-07-28T07:16:34.407Z" }, + { url = "https://files.pythonhosted.org/packages/63/80/796ac61dddb3ede550ea127e28e027a57a4ea481581c0bb27701fefd655a/hypothesis-6.163.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:213527755f0fc2b1f3721e73fd60023e2752a48f914e3e2df8d35111956ae5c8", size = 770366, upload-time = "2026-07-28T07:15:22.003Z" }, + { url = "https://files.pythonhosted.org/packages/eb/87/53924f322922bcfc05e40c79d432978333827b335fa9efa5fd1e8cbf3c90/hypothesis-6.163.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ca1b48bde68c528a79dec2a2859e05035802e5b1c9c3579f388c9de6ed6d0148", size = 766150, upload-time = "2026-07-28T07:15:16.151Z" }, + { url = "https://files.pythonhosted.org/packages/f3/fb/b62480e6510052139d7b2a0219d4ae8bd52ccad0ed74db15dd61330a6962/hypothesis-6.163.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a3db868a943c814cc557104712d43bf609adfe5ea9f708f38377d366b4855f8", size = 1095046, upload-time = "2026-07-28T07:15:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/4d/d7/fa8aac7b47f41d0fe30fa270c5026f37a0c0c60d7b9a1a93dcc8fcfc50ce/hypothesis-6.163.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4159a1c2560e10de51b1c14956e277eb1b37526c9abef9e87c1e531760486448", size = 1144516, upload-time = "2026-07-28T07:16:32.662Z" }, + { url = "https://files.pythonhosted.org/packages/4f/26/c543c76d8a8b8f58f2d7adf0cb42e4928be3464e95f4fa9d7221b42ea9ce/hypothesis-6.163.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ffdda3006a383a48f71a23b4f2b3fae3fe1b09af67925d885985f7ec34d66bcb", size = 1268886, upload-time = "2026-07-28T07:16:36.102Z" }, + { url = "https://files.pythonhosted.org/packages/d9/54/3613ef980cfa60f5c6bfbc533989d6c67b6b5f4e9e638fc6d010a5dd852f/hypothesis-6.163.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3b6cee2afe6c67b31a4a64b63a876e0b020befdc61daabea80f7a0e14f19203a", size = 1311472, upload-time = "2026-07-28T07:16:13.707Z" }, + { url = "https://files.pythonhosted.org/packages/af/0c/cbaebc49fd5807a4b4286dea6e60d5330951e43115d002371bdfc99e70f8/hypothesis-6.163.0-cp311-cp311-win_amd64.whl", hash = "sha256:0a933aca9ebf9daf951d07cf01200c94c321b6ee0b42cc7b67675c9686d914c2", size = 661580, upload-time = "2026-07-28T07:15:32.939Z" }, + { url = "https://files.pythonhosted.org/packages/8f/2c/74e989557efc429b28282cbe754c17fc74495467a72a1c94b5eb734fe374/hypothesis-6.163.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a57352efa938889ea9992667a5014c0fc870d03945de71918574d1cf28276378", size = 771445, upload-time = "2026-07-28T07:16:44.2Z" }, + { url = "https://files.pythonhosted.org/packages/fd/08/3ed2089d8cbeae125ea92879e82b99ea3a8b676c837710019249ccab379f/hypothesis-6.163.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0a0c396244c13805edcb73ff467c4c8178ccefc41c4ef5ed00a68e612fd773e9", size = 763070, upload-time = "2026-07-28T07:15:34.318Z" }, + { url = "https://files.pythonhosted.org/packages/e5/1e/4e85a11f15e8ebd730f3b3e4a5d83653f7da482a4e81fa36958951d89b4a/hypothesis-6.163.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28a6cc1c25a6cc9b6ec079eaabd32ac769994831ecddd57123ce43c9056dcf34", size = 1093500, upload-time = "2026-07-28T07:15:49.539Z" }, + { url = "https://files.pythonhosted.org/packages/1f/29/c4790d2a5e6f48e6be5f868124a0d3c7e1d0102e2ca50503a0718e51289c/hypothesis-6.163.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd312b15044b1c1a0920a5827a830559b2d1fa380851cedf509f8b835309c5b9", size = 1143541, upload-time = "2026-07-28T07:15:44.393Z" }, + { url = "https://files.pythonhosted.org/packages/da/a8/01b72694e758449e9b530b72ecaf5f3625c80a3968de0d12ee6e784de22e/hypothesis-6.163.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b839dfd1342bb50570cb0c66b80322307cdb468abf14faf5df4dab022bc1b9ce", size = 1266326, upload-time = "2026-07-28T07:15:12.604Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a3/b3997add991e2fc6123b3c8dde3631f9f633db67ba23f5b2567736b50b9e/hypothesis-6.163.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c4f5be1482189c7b0a1dcac269fffe97a7d18cc04ac9a9a4d6613212dd87f38b", size = 1310536, upload-time = "2026-07-28T07:15:17.798Z" }, + { url = "https://files.pythonhosted.org/packages/5d/fe/36f576185d63ee0b4d94ed9564415de08baed4937e785b3695c8dd665c9c/hypothesis-6.163.0-cp312-cp312-win_amd64.whl", hash = "sha256:7ca7b20bf38d51e15f7808b0239791c4792b1709ce0c63093acaff56a09c31e6", size = 659021, upload-time = "2026-07-28T07:15:53.143Z" }, + { url = "https://files.pythonhosted.org/packages/58/69/c474a3fa1c33d9a6e059e820221275d9c60eb10085f6164212c291856157/hypothesis-6.163.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:a16ebce774755a7a652bd44c62101dc914372ed1a98935969624848c9627b4a4", size = 771335, upload-time = "2026-07-28T07:15:14.988Z" }, + { url = "https://files.pythonhosted.org/packages/54/27/8951688de58314780ba0af5bf2675709009dcc1fb568d244f2a03c4de8e9/hypothesis-6.163.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40dfab6fe6a02a80abef81aebf88e53cd529e3f2f6ba3486b674a67b1f4a3512", size = 763020, upload-time = "2026-07-28T07:15:25.218Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d7/9eb7451400507f4b5c972c81c2bc57d0009597ea37295519a9645963a50e/hypothesis-6.163.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f28ad27193c1fbcfb52ef2ee63d2b721563525089e80962b4268b306dac45507", size = 1093415, upload-time = "2026-07-28T07:16:27.408Z" }, + { url = "https://files.pythonhosted.org/packages/24/cc/c12c780676c7a4a4051d05e7b278a94bf1bb496ef31882881160f16866c9/hypothesis-6.163.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8aac96db8a6c7ee43aba2ee0d3c43893da1fb7c38ed54790c1be2b6d8fd87b96", size = 1143356, upload-time = "2026-07-28T07:16:01.571Z" }, + { url = "https://files.pythonhosted.org/packages/4b/c1/61c5ebdb77a3f259803e60a12f56de35f8b6d641a6219f798dcfa69dece3/hypothesis-6.163.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9d23f0f3a14bb6e6f99c793d340196dba4af95ba25bfcab624d1794f540f5e27", size = 1266375, upload-time = "2026-07-28T07:15:11.353Z" }, + { url = "https://files.pythonhosted.org/packages/a2/53/f3a89b4d21d89098d1dec749632aa0fece04f5d17a9e7e91c7f10596d55a/hypothesis-6.163.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b123b4995a7612f1130e2b2362c9a5d0568df887bf7e7bdb45c23af8cd5423c9", size = 1310257, upload-time = "2026-07-28T07:16:42.213Z" }, + { url = "https://files.pythonhosted.org/packages/06/e7/88e71c4cec0df68aa7a2fb251083c544f30697bd90fb4b1ed19de493ab81/hypothesis-6.163.0-cp313-cp313-win_amd64.whl", hash = "sha256:b268211e625cd550e361fc387bf1db5deb1e9cae0ce4041116f0a0aafeef7c06", size = 658983, upload-time = "2026-07-28T07:16:10.289Z" }, + { url = "https://files.pythonhosted.org/packages/71/df/e3b2f0419cebcc86bba96a357d7ef37790538ed6b09f3183ae01f1fc3d23/hypothesis-6.163.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9105c66ea8dbc108adc42058bb7b65bd953f53ee178bf63bf9ebb0cded6c8c96", size = 771564, upload-time = "2026-07-28T07:15:28.017Z" }, + { url = "https://files.pythonhosted.org/packages/f1/f8/a6f75e61ecd983029f8463bf007498155c4ed33114e6931e7aa3fbaf651e/hypothesis-6.163.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e165f6cc2075059b7c95dac1612bfb25494f72d90f56880e84c288b089f8a896", size = 763164, upload-time = "2026-07-28T07:15:29.973Z" }, + { url = "https://files.pythonhosted.org/packages/5e/07/5193812567f6ca46c1f0cd02dabfd47c85d7c9e3287b8a870fc8150ee462/hypothesis-6.163.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cba5202f74e7e4cdb676d86f26e8cc1b4fdc88f7f58ba73c8ac45b6b22f3070", size = 1093916, upload-time = "2026-07-28T07:16:25.626Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0c/9d61f0e306499d734dc63ebe374b01c5f50be7072fd5e76eed25cc0b86ac/hypothesis-6.163.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7f706df6839dcc53f20833f2933cbcd126fd2fdee7c312e053de49df4b64e44", size = 1143547, upload-time = "2026-07-28T07:15:07.311Z" }, + { url = "https://files.pythonhosted.org/packages/60/b4/2a9eb04c9847ddf7e6b30bba8bc27b7efe5511d368335b6a41657b3f02dd/hypothesis-6.163.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:487ab8ec2f01a225d6a1e2ceadc5290cde2c691952bd2e7f76199cf82e06fb25", size = 1266755, upload-time = "2026-07-28T07:15:41.457Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f3/8d1903fbfcbf48b90bb5590826821afc9b4fc494391ab1fdfab9df23e928/hypothesis-6.163.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ae63dec6d1d467b7f4737455f81a7a82f14a41c14510937fcfbc726a085b5f8", size = 1310560, upload-time = "2026-07-28T07:15:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/47/2b/1a8c0457b44775d0aad369f21cbae8026b772a71aa917d1b956b7349b2a4/hypothesis-6.163.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:31dc46c48aa53c3ec92d03120978ca7f19b9cf96d195ed3fc93503f1433c94a6", size = 603067, upload-time = "2026-07-28T07:16:06.697Z" }, + { url = "https://files.pythonhosted.org/packages/10/56/a9cd947064043035457dec0124ac2437ca72acf358b30cf203a229ad831b/hypothesis-6.163.0-cp314-cp314-win_amd64.whl", hash = "sha256:320b076bf6436f971f1c73ee651e60001226d1b4e341f2c4a1ca87248261ca03", size = 658931, upload-time = "2026-07-28T07:15:37.174Z" }, + { url = "https://files.pythonhosted.org/packages/77/37/2d16317fda0ecd915cb094be9a9e8911106e693ed03a4d680de314711854/hypothesis-6.163.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ab34c61d9249f1a8129cb4276062c04e3e47b5be8de6446e7c7fe11362d6fe43", size = 770142, upload-time = "2026-07-28T07:16:21.827Z" }, + { url = "https://files.pythonhosted.org/packages/3f/65/d80a9bfb7868f2c6c072a684993548391c56e0afa1972f79ee1fe168d91b/hypothesis-6.163.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f2f1b67a48da86d3e41c9445367b49a49f7efdb60fc8b5e3593f05e6afb2efbe", size = 761694, upload-time = "2026-07-28T07:15:31.36Z" }, + { url = "https://files.pythonhosted.org/packages/a7/01/c1f2515c638d2637300bb2fd6af129319eae75cdf2ed7804644535f64fb2/hypothesis-6.163.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c084749c115ea7918cf7efa144682783da17eec70d1276689182b871126e715", size = 1092511, upload-time = "2026-07-28T07:15:51.503Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f8/b3cd946308b5a09e52b075792610da310c0b7972bb1e8aa673400b86540c/hypothesis-6.163.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e72e8d5818e5ef8cd6a2191c386e3fd1a6d9e3739cf97289b4d9b5dbc8e38d", size = 1142425, upload-time = "2026-07-28T07:16:15.626Z" }, + { url = "https://files.pythonhosted.org/packages/85/96/b122859e6f7335b54aff759f573a813158d2249488450a75e76462ddaf61/hypothesis-6.163.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5a3ac6c62d49f7fe518dfe7fa924fa03aac839993702207802b0e45f9e1b0dab", size = 1264946, upload-time = "2026-07-28T07:15:23.848Z" }, + { url = "https://files.pythonhosted.org/packages/ba/2e/e0226e8c904b8b4788eb52dacd303c91acbd260904abf64e8f9bad03c88a/hypothesis-6.163.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e568a3d766b7ba8df00e0c33efc4c6530cde14fbc72daabe4824eed211ed7596", size = 1309320, upload-time = "2026-07-28T07:16:29.218Z" }, + { url = "https://files.pythonhosted.org/packages/8c/48/e8bd29fed17c9608524b6a39db4a27b6eec7ce85bda78bba3ee0deebd80e/hypothesis-6.163.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8f22fb8218ba6a452bf9000fc656e1ed57625d17cc8a3871a0fcea3b1b69ebf", size = 659064, upload-time = "2026-07-28T07:16:00.075Z" }, + { url = "https://files.pythonhosted.org/packages/39/db/d4e877b8639bbebedeff4a0511f5fc459c06a31d79a79bf75584eddda8da/hypothesis-6.163.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:002a9709345892279fb0e81b5a05b72d08cfe81f937339827be0d588607ca9b0", size = 771252, upload-time = "2026-07-28T07:15:08.694Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/3492490e997e5c8c9244a56728a623ca817577af3462fca5d1def060a066/hypothesis-6.163.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:52f16840add2eb02c2416f3b83cec4f527b6c19699f2d31eff4859233c715526", size = 767161, upload-time = "2026-07-28T07:15:54.972Z" }, + { url = "https://files.pythonhosted.org/packages/51/73/37a4d4a6f3f0789fb2fddc851da957075fbe223706d1148ccc4dda825171/hypothesis-6.163.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34fc895691a2420595506eb17f3a104f2fa9039f013c0770a6cc2743ccaf6fed", size = 1096016, upload-time = "2026-07-28T07:15:09.821Z" }, + { url = "https://files.pythonhosted.org/packages/48/46/20bc7801f8b539334dc0439c28753632a078c96d6732eccf1bd4880644d2/hypothesis-6.163.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ef8954e37c80e0c46e6161eef1c72c71059b95250e620a77bd646f6c7a52a2d", size = 1145797, upload-time = "2026-07-28T07:15:39.864Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/4d5c8feb78ed86e698d4e0665d3179eb657ae66d3606ffe096d8055aaa82/hypothesis-6.163.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d0838a28e9943d5b834ebae59b02adda76e2cd1e65caa808104c72102052057d", size = 662696, upload-time = "2026-07-28T07:15:38.519Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] From 2fc45bc167582b4d31e3312f97a4731180cf1dfd Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 00:15:47 +0500 Subject: [PATCH 02/25] Add the Ossie -> Cube export direction, completing the converter Both directions are now implemented and losslessly round-trip. Export emits one `model/cubes/.yml` per dataset plus a `model/views/.yml` for the model itself -- the view is always emitted, not optional, because it is the model boundary for view-first Cube users. A model imported from Cube restores its original file paths, view curation, segments, pre-aggregations, hierarchies, and every other Cube-only construct from the stash; a hand-authored Ossie model gets a view generated at the FK sink with each cube addressed by its join path. Because Cube has a `meta` field at every level, Ossie constructs Cube has no slot for (`unique_keys`, foreign-vendor `custom_extensions`, the structured form of `ai_context`) are parked under `meta.ossie` instead of being dropped. So Ossie -> Cube -> Ossie is lossless too, not just Cube -> Ossie -> Cube. Reference forms follow Cube's semantics rather than being uniform: `{CUBE.member}` when the dataset declares a field of that name (reusing the member's SQL, compile-time checked), `{CUBE}.column` for a raw physical column, and `{other_cube.member}` across cubes -- which is also what gives a cross-dataset metric its implicit join. The cube's own name is never spelled out, so the model survives `extends`. COUNT(DISTINCT ) converts back to Cube's bare `type: count`, closing the loop on the fan-out mapping in both directions. Tests (160): per-direction unit tests, fixture round-trips including a TPC-DS model generated from examples/tpcds_semantic_model.yaml as the guide asks, core-spec JSON Schema validation of every emitted Ossie document, and Hypothesis property-based round-trips over generated Cube models with a seeded fallback when hypothesis is unavailable. The property tests found two real defects: a `{{` anywhere in a file disqualifies it as Jinja (matching Cube's own file-level check), which could leave a join pointing at a cube that was never converted -- the error now names the skipped file instead of just reporting a missing cube. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 33 +- converters/cube/pyproject.toml | 3 + converters/cube/src/ossie_cube/__init__.py | 5 +- converters/cube/src/ossie_cube/_common.py | 4 - converters/cube/src/ossie_cube/cli.py | 37 +- converters/cube/src/ossie_cube/cube_to_osi.py | 58 +- converters/cube/src/ossie_cube/osi_to_cube.py | 711 ++++++++++++++++++ converters/cube/tests/_roundtrip_helpers.py | 210 ++++++ converters/cube/tests/_util.py | 14 + .../tpcds_cube/model/cubes/customer.yml | 83 ++ .../tpcds_cube/model/cubes/date_dim.yml | 84 +++ .../fixtures/tpcds_cube/model/cubes/item.yml | 98 +++ .../fixtures/tpcds_cube/model/cubes/store.yml | 97 +++ .../tpcds_cube/model/cubes/store_sales.yml | 211 ++++++ .../model/views/tpcds_retail_model.yml | 60 ++ converters/cube/tests/test_cube_to_osi.py | 21 + converters/cube/tests/test_osi_to_cube.py | 429 +++++++++++ converters/cube/tests/test_roundtrip.py | 194 +++++ .../cube/tests/test_roundtrip_properties.py | 84 +++ converters/cube/uv.lock | 184 +++++ 20 files changed, 2594 insertions(+), 26 deletions(-) create mode 100644 converters/cube/src/ossie_cube/osi_to_cube.py create mode 100644 converters/cube/tests/_roundtrip_helpers.py create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml create mode 100644 converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml create mode 100644 converters/cube/tests/test_osi_to_cube.py create mode 100644 converters/cube/tests/test_roundtrip.py create mode 100644 converters/cube/tests/test_roundtrip_properties.py diff --git a/converters/cube/README.md b/converters/cube/README.md index 4fe445fe..7a5c4a4a 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -23,10 +23,6 @@ Bidirectional, offline conversion between an [Apache Ossie](https://github.com/a semantic model and a [Cube](https://cube.dev/docs/product/data-modeling/overview) data model. No Cube deployment, API token, or network access required. -> **Status:** the **import** direction (Cube -> Ossie) is implemented. The -> **export** direction (Ossie -> Cube) is in progress; the mapping table below -> describes the agreed behavior for both. - A Cube data model is a *directory* of YAML files rather than a single document, so this converter maps one Ossie YAML document to/from the Cube model layout: @@ -68,18 +64,23 @@ The only runtime dependency is `PyYAML`. Python 3.11+. ```bash ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] [--no-strict-fanout] +ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] ``` -With no `-o` the Ossie YAML goes to stdout; issues always go to stderr. `--view` -picks which view's name/description/AI context map onto the Ossie model when the -directory holds several. `--name` overrides the model name. +`import` with no `-o` writes the Ossie YAML to stdout; `export` always needs `-o` +(a directory). Issues always go to stderr. `--view` picks which view's +name/description/AI context map onto the Ossie model when the directory holds +several; `--name` overrides the model name. `--base-cube` picks the cube a +*generated* view is rooted at, and is only consulted for a hand-authored Ossie +model with no stashed views. ### Python API ```python -from ossie_cube import convert_cube_to_ossie +from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube -ossie_yaml, issues = convert_cube_to_ossie(files) # {relative filename: YAML str} +ossie_yaml, issues = convert_cube_to_ossie(files) # {relative filename: YAML str} +files, issues = convert_ossie_to_cube(ossie_yaml) # -> {relative filename: YAML str} for issue in issues: print(issue) ``` @@ -228,10 +229,18 @@ uv sync uv run pytest ``` +Example-based unit tests per direction, fixture round-trip tests (including the +[TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks +for as a baseline), core-spec JSON Schema validation of every emitted Ossie +document, and Hypothesis property-based round-trip tests over generated Cube +models -- which fall back to a seeded sweep when `hypothesis` is unavailable, so +the properties still run. + ## Future effort Both the Apache Ossie specification and Cube's data model are still evolving. As either side adds or changes fields, this converter will be updated to track them. -Known next steps: the export direction, offline `extends` resolution, and a -first-class Ossie representation for measure additivity so the fan-out caveat can -be recorded in the model instead of an issue log. +Known next steps: offline `extends` resolution, `.js`/`.ts` model support (which +needs Cube's own transpiler, so most likely a Cube-side exporter feeding this +converter), and a first-class Ossie representation for measure additivity so the +fan-out caveat can be recorded in the model instead of an issue log. diff --git a/converters/cube/pyproject.toml b/converters/cube/pyproject.toml index 4b60853e..a78e12b8 100644 --- a/converters/cube/pyproject.toml +++ b/converters/cube/pyproject.toml @@ -44,6 +44,9 @@ dependencies = [ dev = [ "pytest>=8.0", "hypothesis>=6.0", + # So the core-spec schema validation in test_roundtrip.py runs rather than + # skipping; the converter itself needs neither. + "jsonschema>=4.0", ] [project.scripts] diff --git a/converters/cube/src/ossie_cube/__init__.py b/converters/cube/src/ossie_cube/__init__.py index fcf01fd5..037f4162 100644 --- a/converters/cube/src/ossie_cube/__init__.py +++ b/converters/cube/src/ossie_cube/__init__.py @@ -19,14 +19,16 @@ models. Pure offline transforms: Ossie YAML string <-> {relative filename: YAML string}. - from ossie_cube import convert_cube_to_ossie + from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube ossie_yaml, issues = convert_cube_to_ossie(files) + files, issues = convert_ossie_to_cube(ossie_yaml) """ from ._common import ConversionError from .converter_issues import ConverterIssue, IssueLog, IssueType from .cube_to_osi import convert_cube_to_ossie +from .osi_to_cube import convert_ossie_to_cube __all__ = [ "ConversionError", @@ -34,4 +36,5 @@ "IssueLog", "IssueType", "convert_cube_to_ossie", + "convert_ossie_to_cube", ] diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 9bfaee02..1468ab4c 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -496,10 +496,6 @@ def join_source(cube, cube_name): "Opaque": "string", } -# Ossie datatypes whose temporal role makes `is_time` default to true (spec.md, -# "DataType and is_time"). -TEMPORAL_DATATYPES = frozenset({"Date", "Time", "DateTime", "DateTimeTz"}) - # Cube measure `type` -> the Ossie aggregate function that reproduces it. # `count` is absent: it maps through the cube's primary key, see # primary_key_count_expression(). diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py index 51bc336c..8616a2a7 100644 --- a/converters/cube/src/ossie_cube/cli.py +++ b/converters/cube/src/ossie_cube/cli.py @@ -18,15 +18,17 @@ """Command-line interface for the Apache Ossie <-> Cube converter. ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] + ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] `import` converts a Cube data model directory (any `.yml` holding `cubes:` / `views:`) into an Apache Ossie semantic model; with no `-o` the Ossie YAML goes to -stdout. Conversions that could not carry something across print an issue list to -stderr. +stdout. `export` does the reverse and always needs `-o` (a directory). +Conversions that could not carry something across print an issue list to stderr. By default a metric whose value a static Ossie expression cannot keep correct -under row multiplication is refused, mirroring Cube's own refusal to answer such -a query; pass `--no-strict-fanout` to emit it with a recorded issue instead. +under row multiplication is refused on import, mirroring Cube's own refusal to +answer such a query; pass `--no-strict-fanout` to emit it with a recorded issue +instead. """ import argparse @@ -35,6 +37,7 @@ from ._common import ConversionError from .cube_to_osi import convert_cube_to_ossie +from .osi_to_cube import convert_ossie_to_cube def _build_parser(): @@ -58,6 +61,18 @@ def _build_parser(): action="store_false", default=True, help="record fan-out-unsafe metrics as issues instead of " "refusing the conversion") + + exp = sub.add_parser( + "export", help="Apache Ossie semantic model -> Cube data model directory") + exp.add_argument("-i", "--input", required=True, help="Ossie YAML file") + exp.add_argument("-o", "--output", required=True, + help="output directory for the Cube model files") + exp.add_argument("-d", "--dialect", + help="preferred Ossie expression dialect (e.g. SNOWFLAKE); " + "ANSI_SQL is always the fallback") + exp.add_argument("-b", "--base-cube", + help="dataset a generated view is rooted at (only used for a " + "model with no stashed views; default: the FK-sink dataset)") return parser @@ -97,6 +112,20 @@ def _report(issues): def main(argv=None): args = _build_parser().parse_args(argv) try: + if args.command == "export": + with open(args.input) as fh: + ossie_yaml = fh.read() + files, issues = convert_ossie_to_cube( + ossie_yaml, dialect=args.dialect, base_cube=args.base_cube) + for rel, text in files.items(): + dest = os.path.join(args.output, *rel.split("/")) + os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) + with open(dest, "w") as fh: + fh.write(text) + print(f"Wrote {len(files)} file(s) to {args.output}", file=sys.stderr) + _report(issues) + return 0 + files = _read_model_dir(args.input) out, issues = convert_cube_to_ossie( files, model_name=args.name, view=args.view, diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 6e165e8e..35800a2f 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -121,6 +121,7 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True) # individual members -- so the view, not any cube, is the model boundary. mapped_name = _pick_view(views, view, issues) mapped_view = views.get(mapped_name) or {} + cubes = _order_by_view(cubes, mapped_view) model = {"name": model_name or mapped_name or "cube_model"} if mapped_view.get("description"): @@ -131,7 +132,7 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True) # Joins are decomposed first: a join with no Ossie form is parked on its # declaring cube's stash, which has to be known before the dataset is built. - relationships, extra_joins = _convert_joins(cubes, issues) + relationships, extra_joins = _convert_joins(cubes, sorted(extra_files), issues) datasets = [] pk_by_cube = {} @@ -177,6 +178,13 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True) stash["extra_files"] = extra_files write_stash(model, stash) + # Foreign-vendor extensions a previous export parked on the mapped view are + # restored after the stash is written, so the CUBE entry stays first. + parked_exts = ((mapped_view.get("meta") or {}).get("ossie") or {}).get( + "custom_extensions") + if parked_exts: + model.setdefault("custom_extensions", []).extend(parked_exts) + return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}), issues @@ -264,6 +272,29 @@ def _as_named_list(value, what): f"{what}: expected a list or mapping, got {type(value).__name__}") +def _order_by_view(cubes, mapped_view): + """Order the datasets the way the mapped view presents them. + + The view is the model boundary, so its `cubes:` order is the order a Cube user + sees -- and carrying it over means the Ossie dataset order is meaningful rather + than an artifact of how the files happened to be named. A cube the view does + not include keeps its file position, after the ones it does. + """ + ranks = {} + for entry in mapped_view.get("cubes") or []: + if not isinstance(entry, dict): + continue + path = entry.get("join_path") + if not isinstance(path, str) or not path: + continue + leaf = path.split(".")[-1] + ranks.setdefault(leaf, len(ranks)) + if not ranks: + return cubes + order = sorted(cubes, key=lambda name: (ranks.get(name, len(ranks)),)) + return {name: cubes[name] for name in order} + + def _pick_view(views, requested, issues): if requested is not None: if requested not in views: @@ -296,7 +327,10 @@ def _ai_context_from_meta(meta): return parked text = meta.get("ai_context") if isinstance(text, str) and text.strip(): - return {"instructions": text.strip()} + # Kept verbatim rather than stripped: a folded block scalar carries a + # trailing newline, and normalizing it away here would make the round trip + # lossy for the sake of cosmetics. + return {"instructions": text} return None @@ -471,13 +505,17 @@ def _convert_geo_dimension(cname, dname, dim, issues): # --- joins ---------------------------------------------------------------------- -def _convert_joins(cubes, issues): +def _convert_joins(cubes, skipped_files, issues): """Turn every cube's `joins` into Ossie relationships. Ossie's `from` is always the many side. A `many_to_one` join declared on cube A points A(many) -> B(one) directly; a `one_to_many` join is flipped, and the declared side and type are stashed so export restores the original. + `skipped_files` names the input files that held no convertible cube, so a join + pointing into one of them explains itself rather than just reporting a missing + cube. + Returns (relationships, {cube name: [unconvertible join, ...]}). """ relationships = [] @@ -489,8 +527,13 @@ def _convert_joins(cubes, issues): target = require_str(join, "name", f"cube '{cname}': join") what = f"join '{cname}' -> '{target}'" if target not in cubes: + hint = "" + if skipped_files: + hint = (f"; note that no cube was converted from " + f"{', '.join(repr(f) for f in skipped_files)} -- if " + f"'{target}' is defined there, that is why") raise ConversionError( - f"{what}: '{target}' is not a cube in this model") + f"{what}: '{target}' is not a cube in this model{hint}") raw_rel = snake(require_str(join, "relationship", what)) rel_type = _RELATIONSHIP_ALIASES.get(raw_rel) if rel_type is None: @@ -754,7 +797,7 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): resolver = _MeasureResolver(cubes, pk_by_cube, issues) counts = {} - for (_cname, mname) in resolver.measures(): + for (_, mname) in resolver.measures(): counts[mname] = counts.get(mname, 0) + 1 metrics = [] @@ -828,6 +871,11 @@ def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, snake(k): v for k, v in measure.items() if snake(k) not in ("description", "meta") } + elif sql is not None: + # The operand's exact Cube spelling: `{CUBE}.city` and `{CUBE.city}` are + # equivalent but not interchangeable byte-for-byte, and export cannot tell + # which one the author wrote from the Ossie expression alone. + stash["sql"] = sql if metric_name != mname: stash["name"] = mname if measure.get("title"): diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py new file mode 100644 index 00000000..0dc4d4a5 --- /dev/null +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -0,0 +1,711 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an Apache Ossie semantic model to a Cube data model. + +Pure offline conversion. Produces the Cube model-directory layout: one +`model/cubes/.yml` per dataset and a `model/views/.yml` for the model +itself, plus -- when a prior import stashed them -- the original file paths and +every Cube-only construct restored verbatim. + +Ossie features Cube has no field for (`unique_keys`, foreign-vendor +`custom_extensions`, the structured form of `ai_context`) are parked under +`meta.ossie` rather than dropped, since Cube has a `meta` field at every level. +That keeps `Ossie -> Cube -> Ossie` lossless as well. + +Usage (CLI): + ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] +""" + +import re + +from ._common import ( + DATATYPE_TO_DIM_TYPE, + OSSIE_FUNC_TO_AGG, + OSSIE_VERSION, + ConversionError, + cube_file, + dump_yaml, + examples_of, + foreign_vendor_extensions, + instructions_of, + is_simple_identifier, + load_yaml, + ossie_expr_to_cube_sql, + parse_source, + pick_expression, + primary_key_operand, + read_stash, + require_str, + sanitize_name, + synonyms_of, + view_file, +) +from .converter_issues import IssueLog, IssueType + +# An aggregate call the exporter can turn back into a structured Cube measure. +_AGG_CALL_RE = re.compile( + r"^\s*(SUM|AVG|MIN|MAX|COUNT|APPROX_COUNT_DISTINCT)\s*\((.*)\)\s*$", + re.IGNORECASE | re.DOTALL, +) +_DISTINCT_RE = re.compile(r"^DISTINCT\s+(.+)$", re.IGNORECASE | re.DOTALL) + +# The order Cube's own YAML documentation and generators use, so exported files +# read the way a hand-authored model does. +_CUBE_KEY_ORDER = [ + "name", "sql_table", "sql", "title", "description", "meta", "joins", + "dimensions", "measures", "segments", +] +_DIM_KEY_ORDER = [ + "name", "sql", "type", "primary_key", "title", "description", "meta", +] +_MEASURE_KEY_ORDER = [ + "name", "sql", "type", "filters", "title", "description", "meta", +] + + +def convert_ossie_to_cube(ossie_yaml_str, dialect=None, base_cube=None): + """Parse Ossie YAML and return Cube model files as {relative filename: YAML str}. + + Returns (files, IssueLog). `dialect` prepends a warehouse dialect (e.g. + SNOWFLAKE) to the expression preference order; ANSI_SQL is always the fallback. + `base_cube` names the dataset a generated view is rooted at, and is only + consulted for a hand-authored Ossie model with no stashed views. + """ + root = load_yaml(ossie_yaml_str, "Ossie model") + if not isinstance(root, dict): + raise ConversionError("Invalid Ossie YAML: expected a mapping at the root") + version = str(root.get("version", "")) + if version != OSSIE_VERSION: + raise ConversionError( + f"Unsupported Ossie version '{version}'. Supported: {OSSIE_VERSION}") + models = root.get("semantic_model") + if not isinstance(models, list) or not models: + raise ConversionError("'semantic_model' must be a non-empty list") + + issues = IssueLog() + if len(models) > 1: + issues.add(IssueType.PARKED_IN_META, "model", + f"{len(models)} semantic models found; converting only the first") + return _convert_model(models[0], dialect, base_cube, issues) + + +def _convert_model(model, dialect, base_cube, issues): + name = model.get("name", "") + dataset_list = model.get("datasets") or [] + if not dataset_list: + raise ConversionError(f"Model '{name}' has no datasets") + + # Dataset -> cube names. A collision (including a case-insensitive duplicate, + # which sanitizes identically) fails loudly rather than merging. + cube_names = {} + taken = set() + for ds in dataset_list: + ds_name = require_str(ds, "name", f"Model '{name}': dataset") + cube_names[ds_name] = sanitize_name( + ds_name, f"Model '{name}': dataset", taken) + taken.add(cube_names[ds_name].lower()) + datasets = {ds["name"]: ds for ds in dataset_list} + + relationships = model.get("relationships") or [] + for rel in relationships: + scope = f"Model '{name}': relationship '{rel.get('name', '')}'" + if (require_str(rel, "from", scope) not in datasets + or require_str(rel, "to", scope) not in datasets): + raise ConversionError(f"{scope} references an unknown dataset") + + model_stash = read_stash(model) + + # Per-cube facts the join and measure stages need. + members_by_cube = {} + pk_by_cube = {} + for ds_name, ds in datasets.items(): + cname = cube_names[ds_name] + members_by_cube[cname] = { + sanitize_name(f["name"], f"dataset '{ds_name}': field", set()) + for f in (ds.get("fields") or []) + } + pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] + + joins_by_cube = _build_joins(relationships, cube_names, issues) + measures_by_cube = _build_measures( + model, cube_names, members_by_cube, pk_by_cube, datasets, relationships, + base_cube, dialect, issues) + + # Cubes, grouped by the file they belong in: several datasets can share one + # stashed original path, in which case they go back into the same file. + stashed_paths = model_stash.get("cube_files") or {} + files_content = {} + for ds_name, ds in datasets.items(): + cname = cube_names[ds_name] + cube = _build_cube(ds, cname, members_by_cube[cname], + joins_by_cube.get(cname), measures_by_cube.get(cname), + dialect, issues) + path = stashed_paths.get(cname) or cube_file(cname) + files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) + + for vpath, view in _build_views(model, model_stash, cube_names, relationships, + datasets, base_cube, issues).items(): + files_content.setdefault(vpath, {}).setdefault("views", []).append(view) + + files = {path: dump_yaml(content) for path, content in files_content.items()} + + # Files a prior import could not convert (`.js` models, Jinja-templated YAML, + # non-model YAML) restore verbatim. + for fname, text in (model_stash.get("extra_files") or {}).items(): + files[fname] = text + return files, issues + + +# --- ai_context ----------------------------------------------------------------- + +def _ai_context_to_meta(ai_context): + """Split an Ossie `ai_context` into (Cube prose, parked original). + + Cube's `meta.ai_context` is free text, so the instructions go there verbatim + and any synonyms are appended as prose -- which is how Cube's own + documentation expresses them ("Common acronyms: LC = Lucky Charms"). The + structured original is parked under `meta.ossie.ai_context` whenever the prose + alone would not restore it, so the Ossie round trip stays exact. + """ + if not ai_context: + return None, None + instructions = instructions_of(ai_context) + synonyms = synonyms_of(ai_context) + examples = examples_of(ai_context) + + parts = [instructions] if instructions else [] + if synonyms: + parts.append("Also known as: " + ", ".join(str(s) for s in synonyms) + ".") + if examples: + parts.append("Example questions: " + + " ".join(str(e) for e in examples)) + prose = "\n".join(parts) if parts else None + + # Import reads a bare prose value back as {"instructions": prose}. Anything + # else -- a plain string, synonyms, examples, extra keys -- needs the original. + round_trips = (isinstance(ai_context, dict) + and set(ai_context) == {"instructions"} + and ai_context.get("instructions") == prose) + return prose, (None if round_trips else ai_context) + + +def _build_meta(ai_context, stashed_meta, parked_extra): + """Assemble a Cube `meta` from the Ossie AI context, a stashed original meta, + and anything Ossie-only that needs parking.""" + prose, parked_ai = _ai_context_to_meta(ai_context) + meta = {} + if prose: + meta["ai_context"] = prose + for key, value in (stashed_meta or {}).items(): + meta[key] = value + parked = dict(parked_extra or {}) + if parked_ai is not None: + parked["ai_context"] = parked_ai + if parked: + meta["ossie"] = parked + return meta + + +def _ordered(obj, order): + """Re-key a dict so the well-known Cube keys come first, in their documented + order, with anything restored from the stash following.""" + out = {k: obj[k] for k in order if k in obj} + for key, value in obj.items(): + if key not in out: + out[key] = value + return out + + +# --- cubes ---------------------------------------------------------------------- + +def _build_cube(ds, cname, members, joins, measures, dialect, issues): + ds_name = ds["name"] + scope = f"dataset '{ds_name}'" + stash = read_stash(ds) + cube = {"name": cname} + + kind, value = parse_source(ds.get("source"), ds_name) + cube[kind] = value + if ds.get("description"): + cube["description"] = ds["description"] + + parked = {} + if ds.get("unique_keys"): + parked["unique_keys"] = [list(k) for k in ds["unique_keys"]] + issues.add(IssueType.PARKED_IN_META, scope, + "unique_keys have no Cube field; parked under meta.ossie") + foreign = foreign_vendor_extensions(ds) + if foreign: + parked["custom_extensions"] = foreign + cube_extras = dict(stash.get("cube_extras") or {}) + stashed_meta = cube_extras.pop("meta", None) + meta = _build_meta(ds.get("ai_context"), stashed_meta, parked) + if meta: + cube["meta"] = meta + if "ai_context" in meta: + issues.add(IssueType.CUBE_LEVEL_AI_CONTEXT_INERT, scope, + "Cube's agent reads ai_context only on views and members, " + "so this cube-level value has no effect in Cube") + + dimensions, covered = _build_dimensions(ds, cname, members, dialect, issues) + # A primary-key column no field covers still has to exist as a dimension for + # Cube to join or roll up the cube. + pk_names = [] + for col in (ds.get("primary_key") or []): + col = str(col) + if col in covered: + pk_names.append(covered[col]) + continue + issues.add(IssueType.PARKED_IN_META, scope, + f"primary key column '{col}' has no field; emitted as a " + f"non-public dimension with type 'string' (Cube requires a type " + f"and Ossie carries none here)") + synth = {"name": col, "sql": col, "type": "string", + "primary_key": True, "public": False} + dimensions.append(synth) + covered[col] = col + pk_names.append(col) + for dim in dimensions: + if dim["name"] in pk_names: + dim["primary_key"] = True + + dimensions.extend( + _ordered(dict(d, name=n), _DIM_KEY_ORDER) + for n, d in (stash.get("extra_dimensions") or {}).items() + ) + if dimensions: + cube["dimensions"] = [_ordered(d, _DIM_KEY_ORDER) for d in dimensions] + + joins = list(joins or []) + # Joins a prior import could not represent go back at their original indices. + for item in sorted(stash.get("extra_joins") or [], key=lambda x: x.get("index", 0)): + joins.insert(min(item.get("index", 0), len(joins)), item["join"]) + if joins: + cube["joins"] = joins + if measures: + cube["measures"] = [_ordered(m, _MEASURE_KEY_ORDER) for m in measures] + + for key, value in cube_extras.items(): + cube[key] = value + return _ordered(cube, _CUBE_KEY_ORDER) + + +def _build_dimensions(ds, cname, members, dialect, issues): + """Build a cube's dimensions from an Ossie dataset's fields. + + Returns (dimensions, {column or field name: dimension name}) -- the second + value is what primary-key resolution matches against. Fields carrying a `geo` + stash are re-merged into the single Cube dimension they were split from. + """ + ds_name = ds["name"] + dimensions = [] + covered = {} + taken = set() + geo_parts = {} + for field in (ds.get("fields") or []): + fname = require_str(field, "name", f"dataset '{ds_name}': field") + stash = read_stash(field) + if "geo" in stash: + geo = stash["geo"] + slot = geo_parts.setdefault(geo["of"], {"index": len(dimensions)}) + slot[geo["part"]] = geo["sql"] + if "host" in geo: + slot["host"] = geo["host"] + if geo["part"] == "latitude": + dimensions.append(None) # placeholder, filled in below + continue + + dname = sanitize_name(fname, f"dataset '{ds_name}': field", taken) + taken.add(dname.lower()) + expr = pick_expression(field.get("expression"), dialect) + if expr is None: + issues.add(IssueType.NO_USABLE_DIALECT, f"{ds_name}.{fname}", + "no ANSI_SQL or preferred-dialect expression; field dropped") + continue + + dim = {"name": dname} + if "sql" in stash: + # The exact Cube spelling a prior import saw. + dim["sql"] = stash["sql"] + else: + dim["sql"] = ossie_expr_to_cube_sql(expr, cname, members, ()) + dim["type"] = _dimension_type(field, stash, f"{ds_name}.{fname}", issues) + if field.get("label"): + dim["title"] = field["label"] + if field.get("description"): + dim["description"] = field["description"] + parked = {} + foreign = foreign_vendor_extensions(field) + if foreign: + parked["custom_extensions"] = foreign + extras = {k: v for k, v in stash.items() if k not in ("sql", "type", "meta")} + meta = _build_meta(field.get("ai_context"), stash.get("meta"), parked) + if meta: + dim["meta"] = meta + for key, value in extras.items(): + dim[key] = value + + dimensions.append(dim) + covered[dname] = dname + if is_simple_identifier(expr): + covered[expr.strip()] = dname + + for of, slot in geo_parts.items(): + if "latitude" not in slot or "longitude" not in slot: + raise ConversionError( + f"dataset '{ds_name}': geo dimension '{of}' is missing its " + f"{'longitude' if 'latitude' in slot else 'latitude'} half") + dim = {"name": of, "type": "geo", + "latitude": {"sql": slot["latitude"]}, + "longitude": {"sql": slot["longitude"]}} + for key, value in (slot.get("host") or {}).items(): + dim[key] = value + dimensions[slot["index"]] = dim + covered[of] = of + return [d for d in dimensions if d is not None], covered + + +def _dimension_type(field, stash, scope, issues): + """Choose the Cube `type`, which every dimension must declare.""" + if "type" in stash: + # Cube collapses Integer/Decimal/Float into `number`, so import parks the + # original rather than asserting an Ossie datatype; restore it here. + return stash["type"] + datatype = field.get("datatype") + explicit_is_time = (field.get("dimension") or {}).get("is_time") + if datatype: + ctype = DATATYPE_TO_DIM_TYPE.get(datatype) + if ctype is None: + raise ConversionError(f"{scope}: unknown datatype '{datatype}'") + if explicit_is_time is True and ctype != "time": + issues.add(IssueType.PARKED_IN_META, scope, + f"is_time is true but datatype '{datatype}' maps to Cube " + f"type '{ctype}'; Cube marks time dimensions by type, so " + f"the temporal role is not carried") + elif explicit_is_time is False and ctype == "time": + issues.add(IssueType.PARKED_IN_META, scope, + f"is_time is false but datatype '{datatype}' maps to Cube " + f"type 'time', which Cube always treats as a time dimension") + return ctype + if explicit_is_time: + return "time" + issues.add(IssueType.PARKED_IN_META, scope, + "no datatype; emitted as Cube type 'string', which Cube requires") + return "string" + + +# --- joins ---------------------------------------------------------------------- + +def _build_joins(relationships, cube_names, issues): + """Group Ossie relationships into per-cube `joins` lists. + + A stashed `declared_on`/`relationship` restores the original declaring side and + type. A hand-authored relationship is declared on its `from` (many) cube as + `many_to_one`, which is the orientation Ossie already guarantees. + """ + joins_by_cube = {} + for rel in relationships: + rname = rel.get("name", "") + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + if not isinstance(from_cols, list) or not isinstance(to_cols, list) \ + or not from_cols or not to_cols: + raise ConversionError( + f"Relationship '{rname}': from_columns and to_columns are required " + f"lists") + if len(from_cols) != len(to_cols): + raise ConversionError( + f"Relationship '{rname}': from_columns ({len(from_cols)}) and " + f"to_columns ({len(to_cols)}) must have the same length") + + stash = read_stash(rel) + from_cube = cube_names[rel["from"]] + to_cube = cube_names[rel["to"]] + declared_on = stash.get("declared_on") + relationship = stash.get("relationship", "many_to_one") + + if declared_on == to_cube: + # The import flipped a one_to_many (or kept a one_to_one) declared on + # the other side; flip back to the original orientation. + own, other = to_cube, from_cube + own_cols, other_cols = to_cols, from_cols + else: + own, other = from_cube, to_cube + own_cols, other_cols = from_cols, to_cols + + join = {"name": other, "relationship": relationship} + if "sql" in stash: + join["sql"] = stash["sql"] + else: + join["sql"] = " AND ".join( + "{CUBE}." + str(a) + " = {" + other + "." + str(b) + "}" + for a, b in zip(own_cols, other_cols)) + for key, value in stash.items(): + if key not in ("declared_on", "relationship", "sql"): + join[key] = value + if rel.get("ai_context"): + issues.add(IssueType.PARKED_IN_META, f"relationship '{rname}'", + "Cube joins carry no metadata, so relationship ai_context " + "has no home; dropped") + joins_by_cube.setdefault(own, []).append( + _ordered(join, ["name", "sql", "relationship"])) + return joins_by_cube + + +# --- measures ------------------------------------------------------------------- + +def _build_measures(model, cube_names, members_by_cube, pk_by_cube, datasets, + relationships, base_cube, dialect, issues): + """Group Ossie metrics into per-cube `measures` lists.""" + name = model.get("name", "") + sanitized = set(cube_names.values()) + base_cache = [] + + def resolve_base(): + if not base_cache: + base_cache.append(cube_names[_pick_base_cube( + name, datasets, relationships, base_cube)]) + return base_cache[0] + + measures_by_cube = {} + for metric in (model.get("metrics") or []): + mname_raw = require_str(metric, "name", "metric") + scope = f"metric '{mname_raw}'" + stash = read_stash(metric) + mname = stash.get("name") or sanitize_name(mname_raw, scope, set()) + + if "measure" in stash: + # A prior import stashed the original measure (a filtered, calculated, + # or otherwise non-reconstructible one); restore it verbatim and + # re-inject the natively mapped metadata. + measure = dict(stash["measure"]) + measure["name"] = mname + _apply_measure_metadata(metric, measure, stash) + target = stash.get("cube") or resolve_base() + _place(measures_by_cube, target, measure, name) + continue + + expr = pick_expression(metric.get("expression"), dialect) + if expr is None: + issues.add(IssueType.NO_USABLE_DIALECT, scope, + "no ANSI_SQL or preferred-dialect expression; metric dropped") + continue + + referenced = { + m.group(1) for m in re.finditer( + r"(?)` is Cube's bare `type: count` -- + which is how import renders it, precisely because that form stays correct + whether or not the cube is fanned out. A recognized aggregate over a single + operand becomes the matching `type` plus `sql`; anything else becomes a + calculated `type: number` measure carrying the whole expression. + """ + measure = {"name": mname} + m = _AGG_CALL_RE.match(expr) + if m and _balanced(m.group(2)): + func, inner = m.group(1).upper(), m.group(2).strip() + distinct = _DISTINCT_RE.match(inner) + if func == "COUNT" and distinct: + inner = distinct.group(1).strip() + if primary_key and inner == primary_key_operand(target, primary_key): + measure["type"] = "count" + return measure + func = "COUNT_DISTINCT" + if func == "COUNT" and inner == "*": + measure["type"] = "count" + return measure + agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) + if agg is not None: + measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( + inner, target, members, sanitized) + measure["type"] = agg + return measure + + # A ratio, a window expression, or a multi-dataset aggregate: Cube expresses + # these as a calculated measure whose sql carries the aggregation. + measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( + expr, target, members, sanitized) + measure["type"] = "number" + if len({ + ref for ref in re.findall( + r"(? 1: + issues.add(IssueType.PARKED_IN_META, scope, + f"expression spans several datasets; emitted as a calculated " + f"measure on cube '{target}' -- verify the join path") + return measure + + +def _apply_measure_metadata(metric, measure, stash): + if stash.get("title"): + measure["title"] = stash["title"] + if metric.get("description"): + measure["description"] = metric["description"] + parked = {} + foreign = foreign_vendor_extensions(metric) + if foreign: + parked["custom_extensions"] = foreign + meta = _build_meta(metric.get("ai_context"), stash.get("meta"), parked) + if meta: + measure["meta"] = meta + + +def _balanced(s): + depth = 0 + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth < 0: + return False + return depth == 0 + + +# --- views ---------------------------------------------------------------------- + +def _build_views(model, model_stash, cube_names, relationships, datasets, + base_cube, issues): + """Return {file path: view dict}. + + Stashed views restore verbatim, with the natively mapped description and AI + context re-injected on the mapped one. The `views` stash key being *present* -- + even empty -- means the original Cube model's view set is known, so a view is + only generated for hand-authored Ossie. + """ + # The model's foreign-vendor extensions have no Cube field, so they ride on the + # view that represents the model -- the mapped one, or the generated one. + parked = {} + foreign = foreign_vendor_extensions(model) + if foreign: + parked["custom_extensions"] = foreign + + out = {} + if "views" in model_stash: + mapped = model_stash.get("mapped_view") + paths = model_stash.get("view_files") or {} + if foreign and mapped is None: + issues.add(IssueType.PARKED_IN_META, "model", + "no mapped view to park foreign-vendor custom_extensions on; " + "they have no Cube home and are dropped") + for vname, view in (model_stash["views"] or {}).items(): + view = dict(view) + if vname == mapped: + if model.get("description"): + view["description"] = model["description"] + meta = _build_meta(model.get("ai_context"), view.get("meta"), parked) + if meta: + view["meta"] = meta + out[paths.get(vname) or view_file(vname)] = view + return out + + vname = sanitize_name(model.get("name", "model"), "Model", set()) + view = {"name": vname} + if model.get("description"): + view["description"] = model["description"] + meta = _build_meta(model.get("ai_context"), None, parked) + if meta: + view["meta"] = meta + view["cubes"] = _view_cubes( + cube_names, relationships, + cube_names[_pick_base_cube(model.get("name", ""), datasets, + relationships, base_cube)]) + out[view_file(vname)] = view + return out + + +def _view_cubes(cube_names, relationships, base): + """Build a generated view's `cubes:` list: the base cube plus every cube + reachable from it, each addressed by its full `join_path`.""" + adjacency = {} + for rel in relationships: + a, b = cube_names[rel["from"]], cube_names[rel["to"]] + adjacency.setdefault(a, []).append(b) + adjacency.setdefault(b, []).append(a) + + entries = [{"join_path": base, "includes": "*"}] + paths = {base: base} + queue = [base] + while queue: + current = queue.pop(0) + for neighbor in adjacency.get(current, []): + if neighbor in paths: + continue + paths[neighbor] = f"{paths[current]}.{neighbor}" + entries.append({"join_path": paths[neighbor], "includes": "*"}) + queue.append(neighbor) + # A cube no relationship reaches cannot be addressed by a join path, so it is + # simply not part of the generated view; it is still exported and joinable. + return entries + + +def _pick_base_cube(model_name, datasets, relationships, hint): + """Choose the cube a generated view is rooted at: an explicit hint, else the + dataset that is never a relationship `to` (the FK sink of a many-to-one star).""" + if hint is not None: + if hint not in datasets: + raise ConversionError( + f"Model '{model_name}': requested base cube '{hint}' is not a dataset") + return hint + if len(datasets) == 1: + return next(iter(datasets)) + if not relationships: + raise ConversionError( + f"Model '{model_name}': {len(datasets)} datasets but no relationships; " + f"name the view's base cube with --base-cube.") + incoming = {name: 0 for name in datasets} + for rel in relationships: + incoming[rel["to"]] += 1 + roots = [n for n in datasets if incoming[n] == 0] + if not roots: + raise ConversionError( + f"Model '{model_name}': every dataset is a relationship target (the " + f"graph has a cycle); name the view's base cube with --base-cube.") + if len(roots) > 1: + raise ConversionError( + f"Model '{model_name}': multiple candidate base cubes {sorted(roots)}; " + f"name the view's base cube with --base-cube.") + return roots[0] diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py new file mode 100644 index 00000000..f7d20f16 --- /dev/null +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -0,0 +1,210 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared model builders and round-trip assertions for property-based tests. + +This module is deliberately free of any third-party test dependency (no +hypothesis, no pytest) so the generation and assertion logic can run two ways: + + - driven by Hypothesis strategies (see test_roundtrip_properties.py), and + - driven by a plain seeded `random.Random` (RandomRnd below), which is how the + logic is exercised when hypothesis is not installed. + +Both drivers implement the small `Rnd` interface (chance/count/pick/text); the +builders depend only on that interface, so the generated model space is identical +either way. + +The builders generate within the *round-trippable subset* -- the shapes the +converter reproduces exactly. Known normalizations are avoided by construction: + + - names are generated already valid as Cube identifiers, so the sanitizer never + renames anything; + - the topology is a star with a single fact, so there is one unambiguous FK sink + and no cycle; + - every cube declares a primary key, which a bare `type: count` needs; + - `sum`/`avg` measures are only placed on the fact cube, which is never the + `to` side of a join -- a non-idempotent aggregate on a fanned-out cube is + refused by design, and that refusal has its own targeted tests; + - a view lists every cube, so dataset ordering is pinned by the view rather + than by file names. + +Name fuzzing (collisions, reserved words) and the fan-out refusal are left to the +targeted unit tests, which assert the converter *rejects* or *reports* those. +""" + +import random +import string + +from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube +from ossie_cube._common import dump_yaml, load_yaml + +# Aggregates whose value survives duplicate rows, so they are safe on any cube. +IDEMPOTENT_AGGS = ["count_distinct", "count_distinct_approx", "min", "max"] +# Aggregates only placed on the fact cube; see the module docstring. +FACT_ONLY_AGGS = ["sum", "avg"] + +DIM_TYPES = ["string", "number", "boolean", "time"] + + +class RandomRnd: + """The `Rnd` interface backed by a seeded `random.Random`.""" + + def __init__(self, seed): + self.r = random.Random(seed) + + def chance(self, p=0.5): + return self.r.random() < p + + def count(self, lo, hi): + return self.r.randint(lo, hi) + + def pick(self, seq): + return self.r.choice(list(seq)) + + def text(self): + # Alphanumeric with optional interior spaces; no leading/trailing space and + # no YAML-special characters, so the value survives a dump/load cycle + # verbatim. + alnum = string.ascii_letters + string.digits + words = [] + for _ in range(self.r.randint(1, 3)): + words.append("".join( + self.r.choice(alnum) for _ in range(self.r.randint(1, 6)))) + return " ".join(words) + + +def build_cube_model(rnd): + """Generate a Cube model as {relative filename: YAML str}.""" + dim_count = rnd.count(1, 3) + dim_names = [f"dim_{i}" for i in range(dim_count)] + fact = "fact" + + cubes = {} + cubes[fact] = _build_cube(rnd, fact, is_fact=True, dim_names=dim_names) + for name in dim_names: + cubes[name] = _build_cube(rnd, name, is_fact=False, dim_names=()) + + files = {} + for name, cube in cubes.items(): + files[f"model/cubes/{name}.yml"] = dump_yaml({"cubes": [cube]}) + + view = {"name": "main"} + if rnd.chance(0.6): + view["description"] = rnd.text() + if rnd.chance(0.6): + view["meta"] = {"ai_context": rnd.text()} + view["cubes"] = ( + [{"join_path": fact, "includes": "*"}] + + [{"join_path": f"{fact}.{d}", "includes": "*"} for d in dim_names] + ) + files["model/views/main.yml"] = dump_yaml({"views": [view]}) + return files + + +def _build_cube(rnd, name, is_fact, dim_names): + cube = {"name": name} + if rnd.chance(0.3): + cube["sql"] = f"SELECT * FROM raw.{name}" + else: + cube["sql_table"] = f"public.{name}" + if rnd.chance(0.5): + cube["description"] = rnd.text() + + if is_fact and dim_names: + cube["joins"] = [ + {"name": d, "sql": "{CUBE}." + f"{d}_id" + " = {" + f"{d}.id" + "}", + "relationship": "many_to_one"} + for d in dim_names + ] + + dimensions = [{"name": "id", "sql": "id", "type": "number", + "primary_key": True}] + for d in dim_names: + dimensions.append({"name": f"{d}_id", "sql": f"{d}_id", "type": "number"}) + for i in range(rnd.count(0, 3)): + dimensions.append(_build_dimension(rnd, f"attr_{i}")) + if rnd.chance(0.25): + dimensions.append({ + "name": "place", "type": "geo", + "latitude": {"sql": "{CUBE}.lat"}, + "longitude": {"sql": "{CUBE}.lon"}, + }) + cube["dimensions"] = dimensions + + # Every cube carries a bare `count`, which collides across cubes and so + # exercises the `__` qualification on import. + measures = [{"name": "count", "type": "count"}] + aggs = IDEMPOTENT_AGGS + (FACT_ONLY_AGGS if is_fact else []) + for i in range(rnd.count(0, 2)): + measure = {"name": f"m_{i}", "sql": "{CUBE}.value", "type": rnd.pick(aggs)} + if rnd.chance(0.4): + measure["description"] = rnd.text() + if rnd.chance(0.3): + measure["meta"] = {"ai_context": rnd.text()} + if rnd.chance(0.25): + measure["format"] = "currency" + measures.append(measure) + cube["measures"] = measures + return cube + + +def _build_dimension(rnd, name): + dtype = rnd.pick(DIM_TYPES) + dim = {"name": name, "type": dtype} + if rnd.chance(0.3): + # A computed expression, which import translates and stashes verbatim. + dim["sql"] = "LOWER({CUBE}." + name + ")" if dtype == "string" \ + else "{CUBE}." + name + else: + dim["sql"] = name + if rnd.chance(0.4): + dim["title"] = rnd.text() + if rnd.chance(0.4): + dim["description"] = rnd.text() + if rnd.chance(0.3): + dim["meta"] = {"ai_context": rnd.text()} + if rnd.chance(0.2): + dim["format"] = "percent" if dtype == "number" else None + if dim["format"] is None: + del dim["format"] + return dim + + +def _parse_files(files): + return {name: load_yaml(text, name) for name, text in files.items()} + + +def assert_cube_roundtrip_is_lossless(files): + """Cube -> Ossie -> Cube reproduces the model structurally.""" + ossie, _ = convert_cube_to_ossie(files) + files2, _ = convert_ossie_to_cube(ossie) + assert _parse_files(files2) == _parse_files(files), ( + "Cube -> Ossie -> Cube changed the model") + + +def assert_ossie_roundtrip_is_lossless(files): + """Ossie -> Cube -> Ossie reproduces the model too.""" + ossie, _ = convert_cube_to_ossie(files) + files2, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files2) + assert load_yaml(ossie2) == load_yaml(ossie), ( + "Ossie -> Cube -> Ossie changed the model") + + +def check_model(files): + assert_cube_roundtrip_is_lossless(files) + assert_ossie_roundtrip_is_lossless(files) diff --git a/converters/cube/tests/_util.py b/converters/cube/tests/_util.py index feb7e253..bf11b481 100644 --- a/converters/cube/tests/_util.py +++ b/converters/cube/tests/_util.py @@ -46,6 +46,20 @@ def parse(yaml_str): return load_yaml(yaml_str) +def parse_files(files): + """Parse every file of a Cube model dict for structural comparison. + + Comments and key order are not part of the data model, so round-trip fidelity + is asserted on the parsed structures. A non-YAML file (a `.js` model preserved + verbatim) is compared as text. + """ + out = {} + for name, text in files.items(): + out[name] = (load_yaml(text, name) if name.lower().endswith((".yml", ".yaml")) + else text) + return out + + def model_of(ossie_yaml): """The sole semantic model of an Ossie document.""" doc = parse(ossie_yaml) diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml new file mode 100644 index 00000000..206ea8d8 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: customer + sql_table: tpcds.public.customer + description: Customer dimension with demographic information + meta: + ai_context: 'Also known as: customers, shoppers, buyers.' + ossie: + unique_keys: + - - c_customer_sk + ai_context: + synonyms: + - customers + - shoppers + - buyers + dimensions: + - name: c_customer_sk + sql: c_customer_sk + type: number + primary_key: true + description: Surrogate key for customer + - name: c_customer_id + sql: c_customer_id + type: string + description: Business key for customer + meta: + ai_context: 'Also known as: customer ID, customer number.' + ossie: + ai_context: + synonyms: + - customer ID + - customer number + - name: c_first_name + sql: c_first_name + type: string + description: Customer first name + - name: c_last_name + sql: c_last_name + type: string + description: Customer last name + - name: customer_full_name + sql: c_first_name || ' ' || c_last_name + type: string + description: Customer full name (computed field) + meta: + ai_context: 'Also known as: full name, customer name.' + ossie: + ai_context: + synonyms: + - full name + - customer name + - name: c_email_address + sql: c_email_address + type: string + description: Customer email address + meta: + ai_context: 'Also known as: email, contact.' + ossie: + ai_context: + synonyms: + - email + - contact diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml new file mode 100644 index 00000000..8855e811 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: date_dim + sql_table: tpcds.public.date_dim + description: Date dimension with calendar attributes + meta: + ai_context: 'Also known as: calendar, dates, time periods.' + ossie: + unique_keys: + - - d_date_sk + ai_context: + synonyms: + - calendar + - dates + - time periods + dimensions: + - name: d_date_sk + sql: d_date_sk + type: number + primary_key: true + description: Surrogate key for date + - name: d_date + sql: d_date + type: time + description: Actual date value + meta: + ai_context: 'Also known as: date, calendar date.' + ossie: + ai_context: + synonyms: + - date + - calendar date + - name: d_year + sql: d_year + type: number + description: Year + meta: + ai_context: 'Also known as: year.' + ossie: + ai_context: + synonyms: + - year + - name: d_quarter_name + sql: d_quarter_name + type: time + description: Quarter name (e.g., 2024Q1) + meta: + ai_context: 'Also known as: quarter, fiscal quarter.' + ossie: + ai_context: + synonyms: + - quarter + - fiscal quarter + - name: d_month_name + sql: d_month_name + type: time + description: Month name + meta: + ai_context: 'Also known as: month.' + ossie: + ai_context: + synonyms: + - month diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml new file mode 100644 index 00000000..79cf11de --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: item + sql_table: tpcds.public.item + description: Item/Product dimension with product attributes + meta: + ai_context: 'Also known as: products, items, merchandise.' + ossie: + unique_keys: + - - i_item_sk + ai_context: + synonyms: + - products + - items + - merchandise + dimensions: + - name: i_item_sk + sql: i_item_sk + type: number + primary_key: true + description: Surrogate key for item + - name: i_item_id + sql: i_item_id + type: string + description: Business key for item + meta: + ai_context: 'Also known as: item ID, product ID, SKU.' + ossie: + ai_context: + synonyms: + - item ID + - product ID + - SKU + - name: i_item_desc + sql: i_item_desc + type: string + description: Item description + meta: + ai_context: 'Also known as: product description, item name.' + ossie: + ai_context: + synonyms: + - product description + - item name + - name: i_brand + sql: i_brand + type: string + description: Brand name + meta: + ai_context: 'Also known as: brand, manufacturer.' + ossie: + ai_context: + synonyms: + - brand + - manufacturer + - name: i_category + sql: i_category + type: string + description: Item category + meta: + ai_context: 'Also known as: product category, department.' + ossie: + ai_context: + synonyms: + - product category + - department + - name: i_current_price + sql: i_current_price + type: number + description: Current price of the item + meta: + ai_context: 'Also known as: price, list price.' + ossie: + ai_context: + synonyms: + - price + - list price diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml new file mode 100644 index 00000000..c3d59832 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: store + sql_table: tpcds.public.store + description: Store dimension with location and store attributes + meta: + ai_context: 'Also known as: stores, retail locations, branches.' + ossie: + unique_keys: + - - s_store_id + ai_context: + synonyms: + - stores + - retail locations + - branches + dimensions: + - name: s_store_sk + sql: s_store_sk + type: number + primary_key: true + description: Surrogate key for store + - name: s_store_id + sql: s_store_id + type: string + description: Business key for store + meta: + ai_context: 'Also known as: store ID, store number.' + ossie: + ai_context: + synonyms: + - store ID + - store number + - name: s_store_name + sql: s_store_name + type: string + description: Store name + meta: + ai_context: 'Also known as: store name, location name.' + ossie: + ai_context: + synonyms: + - store name + - location name + - name: s_city + sql: s_city + type: string + description: City where store is located + meta: + ai_context: 'Also known as: city, location.' + ossie: + ai_context: + synonyms: + - city + - location + - name: s_state + sql: s_state + type: string + description: State where store is located + meta: + ai_context: 'Also known as: state, region.' + ossie: + ai_context: + synonyms: + - state + - region + - name: s_number_employees + sql: s_number_employees + type: number + description: Number of employees at the store + meta: + ai_context: 'Also known as: employee count, staff size.' + ossie: + ai_context: + synonyms: + - employee count + - staff size diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml new file mode 100644 index 00000000..48f2b70f --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: store_sales + sql_table: tpcds.public.store_sales + description: Fact table containing all store sales transactions + meta: + ai_context: 'Also known as: sales transactions, store purchases, retail sales, + POS data.' + ossie: + unique_keys: + - - ss_item_sk + - ss_ticket_number + ai_context: + synonyms: + - sales transactions + - store purchases + - retail sales + - POS data + joins: + - name: date_dim + sql: '{CUBE}.ss_sold_date_sk = {date_dim.d_date_sk}' + relationship: many_to_one + - name: customer + sql: '{CUBE}.ss_customer_sk = {customer.c_customer_sk}' + relationship: many_to_one + - name: item + sql: '{CUBE}.ss_item_sk = {item.i_item_sk}' + relationship: many_to_one + - name: store + sql: '{CUBE}.ss_store_sk = {store.s_store_sk}' + relationship: many_to_one + dimensions: + - name: ss_sold_date_sk + sql: ss_sold_date_sk + type: number + description: Foreign key to date dimension + meta: + ai_context: 'Also known as: sale date, transaction date.' + ossie: + ai_context: + synonyms: + - sale date + - transaction date + - name: ss_item_sk + sql: ss_item_sk + type: number + primary_key: true + description: Foreign key to item dimension + meta: + ai_context: 'Also known as: product, item.' + ossie: + ai_context: + synonyms: + - product + - item + - name: ss_customer_sk + sql: ss_customer_sk + type: number + description: Foreign key to customer dimension + meta: + ai_context: 'Also known as: customer, buyer.' + ossie: + ai_context: + synonyms: + - customer + - buyer + - name: ss_store_sk + sql: ss_store_sk + type: number + description: Foreign key to store dimension + meta: + ai_context: 'Also known as: store, location.' + ossie: + ai_context: + synonyms: + - store + - location + - name: ss_quantity + sql: ss_quantity + type: number + description: Quantity of items sold + meta: + ai_context: 'Also known as: units sold, quantity.' + ossie: + ai_context: + synonyms: + - units sold + - quantity + - name: ss_sales_price + sql: ss_sales_price + type: number + description: Sales price per unit + meta: + ai_context: 'Also known as: unit price, price.' + ossie: + ai_context: + synonyms: + - unit price + - price + - name: ss_ext_sales_price + sql: ss_ext_sales_price + type: number + description: Extended sales price (quantity * price) + meta: + ai_context: 'Also known as: total price, line total.' + ossie: + ai_context: + synonyms: + - total price + - line total + - name: ss_net_profit + sql: ss_net_profit + type: number + description: Net profit from the sale + meta: + ai_context: 'Also known as: profit, margin.' + ossie: + ai_context: + synonyms: + - profit + - margin + - name: ss_ticket_number + sql: ss_ticket_number + type: string + primary_key: true + public: false + measures: + - name: total_sales + sql: '{CUBE.ss_ext_sales_price}' + type: sum + description: Total sales revenue across all transactions + meta: + ai_context: 'Also known as: total revenue, gross sales, sales amount.' + ossie: + ai_context: + synonyms: + - total revenue + - gross sales + - sales amount + - name: total_profit + sql: '{CUBE.ss_net_profit}' + type: sum + description: Total net profit from store sales + meta: + ai_context: 'Also known as: net profit, total earnings, profit.' + ossie: + ai_context: + synonyms: + - net profit + - total earnings + - profit + - name: customer_lifetime_value + sql: SUM({CUBE.ss_ext_sales_price}) / COUNT(DISTINCT {customer.c_customer_sk}) + type: number + description: Average lifetime sales value per customer + meta: + ai_context: 'Also known as: CLV, LTV, customer value, lifetime revenue.' + ossie: + ai_context: + synonyms: + - CLV + - LTV + - customer value + - lifetime revenue + - name: sales_by_brand + sql: '{CUBE.ss_ext_sales_price}' + type: sum + description: Total sales by brand (requires grouping by item.i_brand) + meta: + ai_context: 'Also known as: brand sales, brand performance, brand revenue.' + ossie: + ai_context: + synonyms: + - brand sales + - brand performance + - brand revenue + - name: store_productivity + sql: SUM({CUBE.ss_ext_sales_price}) / NULLIF(SUM({store.s_number_employees}), + 0) + type: number + description: Sales per employee across stores + meta: + ai_context: 'Also known as: sales per employee, employee productivity, revenue + per employee.' + ossie: + ai_context: + synonyms: + - sales per employee + - employee productivity + - revenue per employee diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml b/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml new file mode 100644 index 00000000..b4694965 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +views: +- name: tpcds_retail_model + description: TPC-DS retail semantic model for sales and customer analytics + meta: + ai_context: Use this semantic model for retail analytics. It provides comprehensive + sales, customer, product, and store data from the TPC-DS benchmark. The model + supports time-based analysis, customer segmentation, product performance, and + store operations metrics. + ossie: + custom_extensions: + - vendor_name: SALESFORCE + data: | + { + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "0.1.1" + } + } + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' + cubes: + - join_path: store_sales + includes: '*' + - join_path: store_sales.date_dim + includes: '*' + - join_path: store_sales.customer + includes: '*' + - join_path: store_sales.item + includes: '*' + - join_path: store_sales.store + includes: '*' diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index 15e07a95..44fdb14a 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -434,6 +434,27 @@ def test_jinja_templated_file_is_preserved_not_parsed(): assert issues.of_type(IssueType.TEMPLATED_MEMBER_DROPPED) +def test_join_into_a_skipped_file_explains_itself(): + """A file the converter had to skip whole (Jinja, `.js`) can leave a join + pointing at a cube that is no longer there. The error says so, rather than just + reporting a missing cube.""" + files = { + "model/cubes/dyn.yml": ( + "cubes:\n - name: users\n sql_table: t{{ suffix }}\n"), + "model/cubes/orders.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + ), + } + with pytest.raises(ConversionError, match="model/cubes/dyn.yml"): + convert_cube_to_ossie(files) + + def test_javascript_model_is_preserved_not_parsed(): files = { "model/cubes/orders.js": "cube(`orders`, { sql_table: `public.orders` });", diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py new file mode 100644 index 00000000..87099837 --- /dev/null +++ b/converters/cube/tests/test_osi_to_cube.py @@ -0,0 +1,429 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Apache Ossie semantic model -> Cube data model.""" + +import pytest +from _util import by_name, parse + +from ossie_cube import ConversionError, IssueType, convert_ossie_to_cube +from ossie_cube._common import OSSIE_VERSION + + +def _ossie(datasets, relationships="", metrics="", model_extra=""): + return (f"version: {OSSIE_VERSION}\n" + "semantic_model:\n" + "- name: shop\n" + f"{model_extra}" + " datasets:\n" + f"{datasets}" + f"{relationships}" + f"{metrics}") + + +_ORDERS = ( + " - name: orders\n" + " source: sales.public.orders\n" + " primary_key:\n" + " - id\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" +) + + +def _cubes(files, path="model/cubes/orders.yml"): + return by_name(parse(files[path])["cubes"]) + + +# --- layout --------------------------------------------------------------------- + +def test_emits_one_file_per_cube_plus_a_view(): + files, _ = convert_ossie_to_cube(_ossie(_ORDERS)) + assert set(files) == {"model/cubes/orders.yml", "model/views/shop.yml"} + + +def test_version_is_enforced(): + with pytest.raises(ConversionError, match="Unsupported Ossie version"): + convert_ossie_to_cube("version: 9.9.9\nsemantic_model: []\n") + + +def test_model_without_datasets_is_rejected(): + with pytest.raises(ConversionError, match="no datasets"): + convert_ossie_to_cube( + f"version: {OSSIE_VERSION}\nsemantic_model:\n- name: shop\n datasets: []\n") + + +def test_relationship_to_unknown_dataset_is_rejected(): + rel = (" relationships:\n" + " - name: r\n from: orders\n to: ghosts\n" + " from_columns: [x]\n to_columns: [y]\n") + with pytest.raises(ConversionError, match="unknown dataset"): + convert_ossie_to_cube(_ossie(_ORDERS, rel)) + + +def test_mismatched_relationship_columns_are_rejected(): + rel = (" relationships:\n" + " - name: r\n from: orders\n to: orders\n" + " from_columns: [a, b]\n to_columns: [c]\n") + with pytest.raises(ConversionError, match="same length"): + convert_ossie_to_cube(_ossie(_ORDERS, rel)) + + +# --- datasets and fields -------------------------------------------------------- + +def test_source_becomes_sql_table_or_sql(): + files, _ = convert_ossie_to_cube(_ossie(_ORDERS)) + assert _cubes(files)["orders"]["sql_table"] == "sales.public.orders" + + query = _ORDERS.replace("source: sales.public.orders", + "source: SELECT * FROM raw.orders") + files, _ = convert_ossie_to_cube(_ossie(query)) + cube = _cubes(files)["orders"] + assert cube["sql"] == "SELECT * FROM raw.orders" + assert "sql_table" not in cube + + +def test_every_dimension_declares_a_type(): + """Cube's schema requires `type` on every dimension, so the converter always + emits one -- falling back to `string` with an issue when Ossie carries none.""" + no_type = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: note\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: note\n" + ) + files, issues = convert_ossie_to_cube(_ossie(no_type)) + assert _cubes(files)["orders"]["dimensions"][0]["type"] == "string" + assert issues.of_type(IssueType.PARKED_IN_META) + + +@pytest.mark.parametrize("datatype,expected", [ + ("String", "string"), + ("Integer", "number"), + ("Decimal", "number"), + ("Float", "number"), + ("Boolean", "boolean"), + ("Date", "time"), + ("DateTime", "time"), + ("DateTimeTz", "time"), + ("Opaque", "string"), +]) +def test_datatype_maps_to_cube_type(datatype, expected): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: f\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: f\n" + f" datatype: {datatype}\n" + ) + files, _ = convert_ossie_to_cube(_ossie(ds)) + assert _cubes(files)["orders"]["dimensions"][0]["type"] == expected + + +def test_is_time_on_a_non_temporal_datatype_is_reported(): + """Cube marks time dimensions by `type`, so an Integer year grain cannot carry + the temporal role -- that is a real loss and it is reported, not hidden.""" + ds = ( + " - name: date_dim\n" + " source: t\n" + " fields:\n" + " - name: d_year\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: d_year\n" + " datatype: Integer\n" + " dimension:\n" + " is_time: true\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds)) + dim = parse(files["model/cubes/date_dim.yml"])["cubes"][0]["dimensions"][0] + assert dim["type"] == "number" + detail = issues.of_type(IssueType.PARKED_IN_META)[0].detail + assert "temporal role is not carried" in detail + + +def test_primary_key_column_without_a_field_is_synthesized(): + ds = ( + " - name: orders\n" + " source: t\n" + " primary_key:\n" + " - ticket_no\n" + " fields:\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds)) + dims = by_name(_cubes(files)["orders"]["dimensions"]) + assert dims["ticket_no"] == { + "name": "ticket_no", "sql": "ticket_no", "type": "string", + "primary_key": True, "public": False} + assert issues.of_type(IssueType.PARKED_IN_META) + + +def test_field_name_is_sanitized_and_collisions_are_rejected(): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: Order Status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status\n" + " - name: order status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status2\n" + ) + with pytest.raises(ConversionError, match="collides"): + convert_ossie_to_cube(_ossie(ds)) + + +def test_missing_dialect_drops_the_field_with_an_issue(): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: f\n" + " expression:\n" + " dialects:\n" + " - dialect: MDX\n" + " expression: '[f]'\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds)) + assert "dimensions" not in _cubes(files)["orders"] + assert issues.of_type(IssueType.NO_USABLE_DIALECT) + + +def test_preferred_dialect_wins_over_ansi(): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: email\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: LOWER(email)\n" + " - dialect: SNOWFLAKE\n" + " expression: LOWER(email)::VARCHAR\n" + " datatype: String\n" + ) + files, _ = convert_ossie_to_cube(_ossie(ds), dialect="SNOWFLAKE") + assert _cubes(files)["orders"]["dimensions"][0]["sql"] == "LOWER(email)::VARCHAR" + + +# --- joins ---------------------------------------------------------------------- + +_TWO_DATASETS = _ORDERS + ( + " - name: users\n" + " source: sales.public.users\n" + " primary_key:\n" + " - id\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" +) +_REL = (" relationships:\n" + " - name: orders_to_users\n" + " from: orders\n" + " to: users\n" + " from_columns: [user_id]\n" + " to_columns: [id]\n") + + +def test_relationship_lands_on_the_many_side_as_many_to_one(): + files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL)) + join = _cubes(files)["orders"]["joins"][0] + assert join == {"name": "users", "sql": "{CUBE}.user_id = {users.id}", + "relationship": "many_to_one"} + # The one side declares nothing; Cube needs the join on one side only. + assert "joins" not in _cubes(files, "model/cubes/users.yml")["users"] + + +def test_composite_relationship_becomes_an_and_chain(): + rel = (" relationships:\n" + " - name: r\n from: orders\n to: users\n" + " from_columns: [user_id, region]\n" + " to_columns: [id, region]\n") + files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) + assert _cubes(files)["orders"]["joins"][0]["sql"] == ( + "{CUBE}.user_id = {users.id} AND {CUBE}.region = {users.region}") + + +def test_relationship_ai_context_has_no_cube_home(): + rel = _REL + " ai_context:\n instructions: Join carefully.\n" + _, issues = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) + assert any("ai_context" in i.detail for i in issues.of_type( + IssueType.PARKED_IN_META)) + + +# --- metrics -------------------------------------------------------------------- + +def _metric(name, expr): + return (" metrics:\n" + f" - name: {name}\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + f" expression: {expr}\n") + + +@pytest.mark.parametrize("expr,expected", [ + ("SUM(orders.amount)", {"type": "sum", "sql": "{CUBE.amount}"}), + ("AVG(orders.amount)", {"type": "avg", "sql": "{CUBE.amount}"}), + ("MIN(orders.amount)", {"type": "min", "sql": "{CUBE.amount}"}), + ("MAX(orders.amount)", {"type": "max", "sql": "{CUBE.amount}"}), + ("COUNT(DISTINCT orders.amount)", + {"type": "count_distinct", "sql": "{CUBE.amount}"}), + ("APPROX_COUNT_DISTINCT(orders.amount)", + {"type": "count_distinct_approx", "sql": "{CUBE.amount}"}), + ("COUNT(*)", {"type": "count"}), +]) +def test_aggregate_expressions_become_structured_measures(expr, expected): + files, _ = convert_ossie_to_cube(_ossie(_ORDERS, metrics=_metric("m", expr))) + measure = _cubes(files)["orders"]["measures"][0] + assert {k: v for k, v in measure.items() if k != "name"} == expected + + +def test_count_distinct_over_the_primary_key_becomes_a_bare_count(): + """The inverse of the import rule: COUNT(DISTINCT ) is exactly Cube's + fan-out-safe `type: count`, so it round-trips back to the idiomatic form.""" + files, _ = convert_ossie_to_cube( + _ossie(_ORDERS, metrics=_metric("m", "COUNT(DISTINCT orders.id)"))) + measure = _cubes(files)["orders"]["measures"][0] + assert measure == {"name": "m", "type": "count"} + + +def test_declared_member_gets_a_member_reference_and_a_raw_column_does_not(): + """`{CUBE.member}` reuses a declared member's SQL and is compile-time checked; + `{CUBE}.column` passes a raw column through. The choice follows from whether + the dataset declares a field of that name.""" + files, _ = convert_ossie_to_cube( + _ossie(_ORDERS, metrics=_metric("m", "SUM(orders.shipping_fee)"))) + # `shipping_fee` is not a declared field, so it stays a raw column. + assert _cubes(files)["orders"]["measures"][0]["sql"] == "{CUBE}.shipping_fee" + + +def test_ratio_becomes_a_calculated_measure(): + files, issues = convert_ossie_to_cube(_ossie( + _TWO_DATASETS, _REL, + _metric("aov", "SUM(orders.amount) / COUNT(DISTINCT users.id)"))) + measure = _cubes(files)["orders"]["measures"][0] + assert measure["type"] == "number" + assert measure["sql"] == "SUM({CUBE.amount}) / COUNT(DISTINCT {users.id})" + assert any("spans several datasets" in i.detail + for i in issues.of_type(IssueType.PARKED_IN_META)) + + +def test_metric_lands_on_the_dataset_its_expression_references(): + files, _ = convert_ossie_to_cube(_ossie( + _TWO_DATASETS, _REL, _metric("users_seen", "COUNT(DISTINCT users.id)"))) + assert "measures" not in _cubes(files)["orders"] + assert _cubes(files, "model/cubes/users.yml")["users"]["measures"][0]["name"] == ( + "users_seen") + + +def test_two_metrics_colliding_on_one_cube_are_rejected(): + metrics = (" metrics:\n" + " - name: Total Amount\n" + " expression:\n dialects:\n - dialect: ANSI_SQL\n" + " expression: SUM(orders.amount)\n" + " - name: total amount\n" + " expression:\n dialects:\n - dialect: ANSI_SQL\n" + " expression: SUM(orders.id)\n") + with pytest.raises(ConversionError, match="two metrics map to measure"): + convert_ossie_to_cube(_ossie(_ORDERS, metrics=metrics)) + + +# --- views ---------------------------------------------------------------------- + +def test_generated_view_is_rooted_at_the_fk_sink(): + files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL)) + view = parse(files["model/views/shop.yml"])["views"][0] + assert view["cubes"] == [ + {"join_path": "orders", "includes": "*"}, + {"join_path": "orders.users", "includes": "*"}, + ] + + +def test_ambiguous_base_cube_is_rejected_and_the_hint_resolves_it(): + two_facts = _TWO_DATASETS # no relationships at all + with pytest.raises(ConversionError, match="no relationships"): + convert_ossie_to_cube(_ossie(two_facts)) + files, _ = convert_ossie_to_cube(_ossie(two_facts), base_cube="orders") + assert parse(files["model/views/shop.yml"])["views"][0]["cubes"][0][ + "join_path"] == "orders" + + +def test_unknown_base_cube_is_rejected(): + with pytest.raises(ConversionError, match="not a dataset"): + convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL), base_cube="nope") + + +def test_synonyms_reach_cube_as_prose_and_are_parked_structurally(): + """Cube has no synonyms field; its docs express them as ai_context prose. The + structured list is parked so the Ossie round trip stays exact.""" + ds = ( + " - name: orders\n" + " source: t\n" + " ai_context:\n" + " instructions: Order facts.\n" + " synonyms:\n" + " - purchases\n" + " - sales\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" + ) + files, _ = convert_ossie_to_cube(_ossie(ds)) + meta = _cubes(files)["orders"]["meta"] + assert meta["ai_context"] == "Order facts.\nAlso known as: purchases, sales." + assert meta["ossie"]["ai_context"]["synonyms"] == ["purchases", "sales"] diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py new file mode 100644 index 00000000..760fc22a --- /dev/null +++ b/converters/cube/tests/test_roundtrip.py @@ -0,0 +1,194 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Fixture-based round-trip tests. + +- Cube -> Ossie -> Cube must be lossless (the stash carries everything). +- Ossie -> Cube -> Ossie must be identical up to the documented normalizations. +- Every Ossie document the importer emits must validate against the core-spec + JSON schema (skipped when jsonschema is not installed). +""" + +import json + +import pytest +from _util import REPO_ROOT, load_fixture_dir, parse, parse_files + +from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube + +FIXTURES = ["fixtureA_cube", "tpcds_cube"] + + +@pytest.mark.parametrize("fixture", FIXTURES) +def test_cube_roundtrip_is_lossless(fixture): + """Cube -> Ossie -> Cube reproduces the original model, structurally. + + Compared parsed rather than byte-for-byte: YAML comments (including the + licence headers on the fixtures) are not part of the data model, and key order + within a mapping is not semantic. + """ + files = load_fixture_dir(fixture) + ossie, _ = convert_cube_to_ossie(files) + files2, _ = convert_ossie_to_cube(ossie) + assert parse_files(files2) == parse_files(files) + + +@pytest.mark.parametrize("fixture", FIXTURES) +def test_imported_ossie_validates_against_core_spec_schema(fixture): + jsonschema = pytest.importorskip("jsonschema") + with open(REPO_ROOT / "core-spec" / "osi-schema.json") as fh: + schema = json.load(fh) + ossie, _ = convert_cube_to_ossie(load_fixture_dir(fixture)) + jsonschema.validate(parse(ossie), schema) + + +@pytest.mark.parametrize("fixture", FIXTURES) +def test_ossie_roundtrip_is_lossless(fixture): + """Ossie -> Cube -> Ossie reproduces the model too. + + Cube has a `meta` field at every level, so the export direction parks what + Cube has no slot for under `meta.ossie` instead of dropping it -- which makes + this direction lossless as well, unlike converters whose target format has + nowhere to put the leftovers. + """ + ossie, _ = convert_cube_to_ossie(load_fixture_dir(fixture)) + files, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files) + assert parse(ossie2) == parse(ossie) + + +def test_hand_authored_ossie_gets_a_generated_view(): + """A model with no stashed views is not from Cube, so export has to invent the + view -- the model boundary Cube users work with.""" + ossie = _HAND_AUTHORED + files, _ = convert_ossie_to_cube(ossie) + assert set(files) == { + "model/cubes/orders.yml", "model/cubes/customers.yml", + "model/views/ecommerce.yml", + } + view = parse(files["model/views/ecommerce.yml"])["views"][0] + assert view["name"] == "ecommerce" + assert view["description"] == "Orders and customers" + # Rooted at the FK sink, with the joined cube addressed by its join path. + assert view["cubes"] == [ + {"join_path": "orders", "includes": "*"}, + {"join_path": "orders.customers", "includes": "*"}, + ] + + +def test_hand_authored_ossie_survives_the_round_trip(): + files, _ = convert_ossie_to_cube(_HAND_AUTHORED) + ossie2, _ = convert_cube_to_ossie(files) + model = parse(ossie2)["semantic_model"][0] + assert model["name"] == "ecommerce" + assert model["description"] == "Orders and customers" + assert [d["name"] for d in model["datasets"]] == ["orders", "customers"] + assert model["relationships"][0]["from_columns"] == ["customer_id"] + metrics = {m["name"]: m for m in model["metrics"]} + assert metrics["total_revenue"]["expression"]["dialects"][0]["expression"] == ( + "SUM(orders.amount)") + + +def test_ossie_only_constructs_are_parked_not_dropped(): + """`unique_keys` and a foreign vendor's extensions have no Cube field, so they + ride under `meta.ossie` and come back intact.""" + files, _ = convert_ossie_to_cube(_HAND_AUTHORED) + orders = parse(files["model/cubes/orders.yml"])["cubes"][0] + parked = orders["meta"]["ossie"] + assert parked["unique_keys"] == [["order_number"]] + assert parked["custom_extensions"][0]["vendor_name"] == "SNOWFLAKE" + + ossie2, _ = convert_cube_to_ossie(files) + ds = {d["name"]: d for d in parse(ossie2)["semantic_model"][0]["datasets"]} + assert ds["orders"]["unique_keys"] == [["order_number"]] + vendors = {e["vendor_name"] for e in ds["orders"]["custom_extensions"]} + assert "SNOWFLAKE" in vendors + + +_HAND_AUTHORED = """ +version: 0.2.0.dev0 +semantic_model: +- name: ecommerce + description: Orders and customers + ai_context: + instructions: Use for sales analysis. + synonyms: + - sales + - purchases + datasets: + - name: orders + source: sales.public.orders + primary_key: + - id + unique_keys: + - - order_number + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Integer + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + datatype: Integer + - name: ordered_at + expression: + dialects: + - dialect: ANSI_SQL + expression: ordered_at + datatype: Date + custom_extensions: + - vendor_name: SNOWFLAKE + data: '{"warehouse": "ANALYTICS_WH"}' + - name: customers + source: sales.public.customers + primary_key: + - id + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Integer + - name: email + expression: + dialects: + - dialect: ANSI_SQL + expression: LOWER(email) + datatype: String + relationships: + - name: orders_to_customers + from: orders + to: customers + from_columns: + - customer_id + to_columns: + - id + metrics: + - name: total_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + description: Total revenue + datatype: Decimal +""" diff --git a/converters/cube/tests/test_roundtrip_properties.py b/converters/cube/tests/test_roundtrip_properties.py new file mode 100644 index 00000000..0a7e4106 --- /dev/null +++ b/converters/cube/tests/test_roundtrip_properties.py @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Property-based round-trip tests over generated Cube models. + +The generators live in `_roundtrip_helpers` and depend only on a tiny +chance/count/pick/text interface, so the same model space is explored whether +Hypothesis is installed or not. Without it, a seeded sweep runs instead -- the +properties are still checked in CI on a Python where hypothesis fails to build. +""" + +import pytest +from _roundtrip_helpers import RandomRnd, build_cube_model, check_model + +try: + from hypothesis import HealthCheck, given, settings + from hypothesis import strategies as st + + HAVE_HYPOTHESIS = True +except ImportError: # pragma: no cover - exercised only without hypothesis + HAVE_HYPOTHESIS = False + + +SEEDS = list(range(60)) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_seeded_models_roundtrip(seed): + """A deterministic sweep, so a failure names a reproducible seed.""" + check_model(build_cube_model(RandomRnd(seed))) + + +if HAVE_HYPOTHESIS: + class _HypothesisRnd: + """The `Rnd` interface backed by a Hypothesis data strategy.""" + + def __init__(self, data): + self.data = data + + def chance(self, p=0.5): + return self.data.draw(st.booleans()) + + def count(self, lo, hi): + return self.data.draw(st.integers(min_value=lo, max_value=hi)) + + def pick(self, seq): + return self.data.draw(st.sampled_from(list(seq))) + + def text(self): + # Printable, no leading/trailing whitespace and no newlines, so the + # value survives a YAML dump/load cycle verbatim. Round-tripping + # arbitrary Unicode is a PyYAML property, not a converter one. + # + # Jinja delimiters are excluded because they are out of the + # round-trippable subset by design: the converter treats a file + # containing them as templated and preserves it whole, exactly as + # Cube's own CubeSchemaConverter does. That behavior has its own + # targeted test. + return self.data.draw(st.text( + alphabet=st.characters(min_codepoint=32, max_codepoint=126), + min_size=1, max_size=24, + ).map(str.strip).filter( + lambda s: s and not s.startswith("#") + and not any(t in s for t in ("{{", "}}", "{%", "%}")))) + + @settings(max_examples=150, deadline=None, + suppress_health_check=[HealthCheck.too_slow]) + @given(st.data()) + def test_generated_models_roundtrip(data): + check_model(build_cube_model(_HypothesisRnd(data))) diff --git a/converters/cube/uv.lock b/converters/cube/uv.lock index c213eb6b..6b158087 100644 --- a/converters/cube/uv.lock +++ b/converters/cube/uv.lock @@ -13,6 +13,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "hypothesis" }, + { name = "jsonschema" }, { name = "pytest" }, ] @@ -22,9 +23,19 @@ requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] [package.metadata.requires-dev] dev = [ { name = "hypothesis", specifier = ">=6.0" }, + { name = "jsonschema", specifier = ">=4.0" }, { name = "pytest", specifier = ">=8.0" }, ] +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + [[package]] name = "colorama" version = "0.4.6" @@ -108,6 +119,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + [[package]] name = "packaging" version = "26.2" @@ -206,6 +244,143 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + [[package]] name = "sortedcontainers" version = "2.4.0" @@ -214,3 +389,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f233 wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] From aa011ed8352b144d040f5aedc6a40305fc25513f Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 00:40:39 +0500 Subject: [PATCH 03/25] Close the test gaps coverage exposed; drop two dead code paths Coverage was 91% with several load-bearing branches never executed. Adds test_edge_cases.py (45 tests) for the paths the fixtures and property tests cannot reach, since those generate inside the round-trippable subset by design: - composite primary keys -- the COUNT(DISTINCT CONCAT(CAST(...))) form is central to the fan-out mapping and was never run in either direction; - `count` with `sql` (COUNT(x)), including that it is fan-out-unsafe where a bare count is not; - the export side of the one_to_many flip -- import flipping it was tested, export flipping it back was not; - off-layout file grouping: several cubes in one oddly-named file have to return to that same file, not be split into the canonical layout; - the JavaScript-style mapping form of dimensions/measures/joins; - legacy belongsTo/hasMany/hasOne spellings; - unconvertible joins restored at their original positions; - geo dimension extras, measure `title`, `ai_context.examples`, a bare string `ai_context`, multiple views, and the malformed-input errors. Two dead paths removed, both found by the same pass: - member-level Jinja handling was unreachable. JINJA_RE is checked per file (as Cube's own CubeSchemaConverter does), so a templated member's file never reaches the per-member branch. Renamed the issue type TEMPLATED_MEMBER_DROPPED -> TEMPLATED_FILE_SKIPPED to match what it actually reports, and dropped the matching `extra_dimensions` restore. - `is_cube_name` was never called. Coverage 91% -> 96%; the remaining 40 lines are defensive ConversionError branches on malformed input. 201 tests. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 2 +- converters/cube/src/ossie_cube/_common.py | 5 - .../cube/src/ossie_cube/converter_issues.py | 8 +- converters/cube/src/ossie_cube/cube_to_osi.py | 20 +- converters/cube/src/ossie_cube/osi_to_cube.py | 4 - converters/cube/tests/test_cube_to_osi.py | 4 +- converters/cube/tests/test_edge_cases.py | 712 ++++++++++++++++++ 7 files changed, 722 insertions(+), 33 deletions(-) create mode 100644 converters/cube/tests/test_edge_cases.py diff --git a/converters/cube/README.md b/converters/cube/README.md index 7a5c4a4a..83609f2c 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -185,7 +185,7 @@ element it concerns, and a detail string. | `MULTI_STAGE_MEASURE_DROPPED` | A `multi_stage` measure (`group_by`/`reduce_by`/`time_shift`/`rank`) renders as a window function over another grain | | `CUBE_LEVEL_AI_CONTEXT_INERT` | Cube's agent ignores cube-level `meta.ai_context` | | `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | -| `TEMPLATED_MEMBER_DROPPED` | Jinja templating, or a `.js`/`.ts` model file | +| `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | | `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | | `PARKED_IN_META` | An element preserved in the stash with no native mapping | diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 1468ab4c..806d0a4d 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -173,11 +173,6 @@ def is_simple_identifier(expr): return isinstance(expr, str) and bool(_IDENTIFIER_RE.match(expr.strip())) -def is_cube_name(name): - """True if `name` is already a valid Cube identifier.""" - return isinstance(name, str) and bool(_CUBE_NAME_RE.match(name)) - - def sanitize_name(name, what, taken): """Coerce an Ossie name into a valid Cube identifier. diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index 64dfe4b1..de29c586 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -53,9 +53,11 @@ class IssueType(Enum): # because an Ossie field holds a single expression. GEO_DIMENSION_SPLIT = "GEO_DIMENSION_SPLIT" - # A dimension or measure whose `sql` uses Jinja templating, or a cube using - # `extends`: no static form, so it is preserved in the stash only. - TEMPLATED_MEMBER_DROPPED = "TEMPLATED_MEMBER_DROPPED" + # A file with no static form -- Jinja templating anywhere in it, or a `.js` / + # `.ts` data model needing Cube's transpiler. Detected per file (as Cube's own + # CubeSchemaConverter does), so the whole file is preserved verbatim in the + # stash rather than half-converted. + TEMPLATED_FILE_SKIPPED = "TEMPLATED_FILE_SKIPPED" # An Ossie field or metric with no usable expression dialect (export). NO_USABLE_DIALECT = "NO_USABLE_DIALECT" diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 35800a2f..bbcd76c6 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -201,12 +201,12 @@ def _collect(files, issues): # A `.js`/`.ts` data model needs Cube's own transpiler and a `.py` one # is Jinja-driven. Preserved verbatim so the round trip keeps the file, # but no cube inside it is converted. - issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, fname, + issues.add(IssueType.TEMPLATED_FILE_SKIPPED, fname, "not a YAML data model; preserved in custom_extensions only") extra_files[fname] = text continue if JINJA_RE.search(text): - issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, fname, + issues.add(IssueType.TEMPLATED_FILE_SKIPPED, fname, "uses Jinja templating, which has no static form; " "preserved in custom_extensions only") extra_files[fname] = text @@ -371,15 +371,8 @@ def _convert_cube(cname, cube, extra_joins, issues): fields = [] primary_key = [] - templated = {} for dim in _as_named_list(cube.get("dimensions"), f"{scope} dimensions"): dname = require_str(dim, "name", f"{scope}: dimension") - if JINJA_RE.search(str(dim.get("sql", ""))): - issues.add(IssueType.TEMPLATED_MEMBER_DROPPED, f"{cname}.{dname}", - "dimension sql uses Jinja templating; preserved in " - "custom_extensions only") - templated[dname] = dim - continue if dim.get("primary_key"): primary_key.append(dname) fields.extend(_convert_dimension(cname, dname, dim, issues)) @@ -387,8 +380,6 @@ def _convert_cube(cname, cube, extra_joins, issues): ds["fields"] = fields if primary_key: ds["primary_key"] = primary_key - if templated: - stash["extra_dimensions"] = templated if extra_joins: stash["extra_joins"] = extra_joins @@ -699,13 +690,6 @@ def expression(self, cname, mname, stack=()): f"multi_stage measure (type '{mtype}'); preserved in " f"custom_extensions only") return None - if JINJA_RE.search(str(measure.get("sql", ""))): - self._issues.add( - IssueType.TEMPLATED_MEMBER_DROPPED, scope, - "measure sql uses Jinja templating; preserved in " - "custom_extensions only") - return None - sql = measure.get("sql") filter_exprs = [ self._translate(f["sql"], cname, stack + (key,)) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 0dc4d4a5..5d163d54 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -284,10 +284,6 @@ def _build_cube(ds, cname, members, joins, measures, dialect, issues): if dim["name"] in pk_names: dim["primary_key"] = True - dimensions.extend( - _ordered(dict(d, name=n), _DIM_KEY_ORDER) - for n, d in (stash.get("extra_dimensions") or {}).items() - ) if dimensions: cube["dimensions"] = [_ordered(d, _DIM_KEY_ORDER) for d in dimensions] diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index 44fdb14a..faedcbf7 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -431,7 +431,7 @@ def test_jinja_templated_file_is_preserved_not_parsed(): model = model_of(out) assert by_name(model["datasets"]).keys() == {"orders"} assert "model/cubes/dyn.yml" in stash_of(model)["extra_files"] - assert issues.of_type(IssueType.TEMPLATED_MEMBER_DROPPED) + assert issues.of_type(IssueType.TEMPLATED_FILE_SKIPPED) def test_join_into_a_skipped_file_explains_itself(): @@ -463,7 +463,7 @@ def test_javascript_model_is_preserved_not_parsed(): } out, issues = convert_cube_to_ossie(files) assert "model/cubes/orders.js" in stash_of(model_of(out))["extra_files"] - assert issues.of_type(IssueType.TEMPLATED_MEMBER_DROPPED) + assert issues.of_type(IssueType.TEMPLATED_FILE_SKIPPED) def test_extends_is_refused_rather_than_half_resolved(): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py new file mode 100644 index 00000000..84874100 --- /dev/null +++ b/converters/cube/tests/test_edge_cases.py @@ -0,0 +1,712 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Coverage-driven tests for paths the fixtures and property tests do not reach. + +The fixture and Hypothesis suites cover the common shapes well, but they generate +inside the round-trippable subset and so never exercise several load-bearing +branches: composite primary keys (central to the fan-out mapping), `count` over an +expression, the export side of the one_to_many flip, off-layout file grouping, the +JavaScript-style mapping form of a collection, and Jinja detection. Each of those +is pinned here, along with the error paths for malformed input. +""" + +import pytest +from _util import by_name, expr_of, model_of, parse, parse_files, stash_of + +from ossie_cube import ( + ConversionError, + IssueType, + convert_cube_to_ossie, + convert_ossie_to_cube, +) + + +def _files(**named): + return {f"model/cubes/{n}.yml": t for n, t in named.items()} + + +def _roundtrip(files): + ossie, issues = convert_cube_to_ossie(files, strict_fanout=False) + back, _ = convert_ossie_to_cube(ossie) + return ossie, back, issues + + +# --- composite primary keys ----------------------------------------------------- + +_COMPOSITE = _files(order_lines=( + "cubes:\n" + " - name: order_lines\n" + " sql_table: public.order_lines\n" + " dimensions:\n" + " - name: order_id\n" + " sql: order_id\n" + " type: number\n" + " primary_key: true\n" + " - name: line_no\n" + " sql: line_no\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: count\n" + " type: count\n" +)) + + +def test_composite_primary_key_becomes_a_concatenated_distinct_count(): + """Cube concatenates a composite key with CAST + CONCAT in `primaryKeyCount`; + the Ossie expression mirrors that so the count stays correct under fan-out and + stays portable (both functions are REQUIRED in the expression language).""" + ossie, _ = convert_cube_to_ossie(_COMPOSITE) + model = model_of(ossie) + assert by_name(model["datasets"])["order_lines"]["primary_key"] == [ + "order_id", "line_no"] + assert expr_of(model["metrics"][0]) == ( + "COUNT(DISTINCT CONCAT(CAST(order_lines.order_id AS VARCHAR), " + "CAST(order_lines.line_no AS VARCHAR)))") + + +def test_composite_key_count_converts_back_to_a_bare_count(): + _, back, _ = _roundtrip(_COMPOSITE) + cube = parse(back["model/cubes/order_lines.yml"])["cubes"][0] + assert cube["measures"] == [{"name": "count", "type": "count"}] + assert [d["name"] for d in cube["dimensions"] if d.get("primary_key")] == [ + "order_id", "line_no"] + + +def test_composite_key_roundtrips(): + _, back, _ = _roundtrip(_COMPOSITE) + assert parse_files(back) == parse_files(_COMPOSITE) + + +# --- count over an expression --------------------------------------------------- + +_COUNT_SQL = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: statuses\n" + " sql: \"{CUBE}.status\"\n" + " type: count\n" +)) + + +def test_count_over_an_expression_keeps_its_operand(): + """`type: count` with `sql` is COUNT(x), not COUNT(*) -- Cube only routes + through the primary key when no sql is given.""" + ossie, _ = convert_cube_to_ossie(_COUNT_SQL) + assert expr_of(model_of(ossie)["metrics"][0]) == "COUNT(orders.status)" + + +def test_count_over_an_expression_roundtrips(): + _, back, _ = _roundtrip(_COUNT_SQL) + assert parse_files(back) == parse_files(_COUNT_SQL) + + +def test_count_over_an_expression_is_fanout_unsafe(): + """Unlike a bare count, COUNT(x) over a fanned-out dataset over-counts.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: emails\n" + " sql: \"{CUBE}.email\"\n" + " type: count\n" + )) + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files) + _, issues = convert_cube_to_ossie(files, strict_fanout=False) + assert issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + + +# --- join orientation, both ways ------------------------------------------------ + +_ONE_TO_MANY = _files(m=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: orders\n" + " sql: \"{CUBE}.id = {orders}.user_id\"\n" + " relationship: one_to_many\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" +)) + + +def test_one_to_many_is_flipped_back_onto_its_original_cube(): + """Ossie's `from` is always the many side, so import flips a one_to_many. Export + has to flip it back -- onto `users`, not `orders`.""" + _, back, _ = _roundtrip(_ONE_TO_MANY) + cubes = by_name(parse(back["model/cubes/m.yml"])["cubes"]) + assert cubes["users"]["joins"] == [{ + "name": "orders", "sql": "{CUBE}.id = {orders}.user_id", + "relationship": "one_to_many"}] + assert "joins" not in cubes["orders"] + + +def test_one_to_one_keeps_its_declared_orientation(): + files = _files(m=_ONE_TO_MANY["model/cubes/m.yml"].replace( + "one_to_many", "one_to_one")) + ossie, issues = convert_cube_to_ossie(files) + rel = model_of(ossie)["relationships"][0] + assert (rel["from"], rel["to"]) == ("users", "orders") + assert any("one_to_one" in i.detail for i in issues.of_type( + IssueType.PARKED_IN_META)) + _, back, _ = _roundtrip(files) + assert parse_files(back) == parse_files(files) + + +@pytest.mark.parametrize("alias,emitted", [ + ("belongsTo", "belongs_to"), + ("belongs_to", "belongs_to"), + ("hasMany", "has_many"), + ("hasOne", "has_one"), +]) +def test_legacy_relationship_spellings_are_accepted_and_kept_semantically( + alias, emitted): + """Cube still accepts belongsTo/hasMany/hasOne. The *kind* of relationship is + preserved rather than modernized to many_to_one, but the spelling is normalized + to snake_case along with every other key -- the documented normalization.""" + files = _files(m=_ONE_TO_MANY["model/cubes/m.yml"].replace( + "one_to_many", alias)) + _, back, _ = _roundtrip(files) + joins = [c.get("joins") for c in parse(back["model/cubes/m.yml"])["cubes"] + if c.get("joins")] + assert joins[0][0]["relationship"] == emitted + + +def test_two_joins_between_one_pair_get_distinct_relationship_names(): + """Ossie relationship names are unique per model, so a second join between the + same two cubes is suffixed rather than colliding.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.buyer_id = {users}.id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: orders\n" + " sql: \"{CUBE}.id = {orders}.seller_id\"\n" + " relationship: one_to_many\n" + )) + ossie, _ = convert_cube_to_ossie(files) + names = [r["name"] for r in model_of(ossie)["relationships"]] + assert names == ["orders_to_users", "orders_to_users_2"] + + +def test_unconvertible_join_is_restored_at_its_original_position(): + """A non-equi join has no Ossie form, so it rides in the stash -- and export has + to put it back among the converted joins, in order.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: rates\n" + " sql: \"{CUBE}.day >= {rates}.valid_from\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " - name: rates\n" + " sql_table: public.rates\n" + " - name: users\n" + " sql_table: public.users\n" + )) + _, back, issues = _roundtrip(files) + assert parse_files(back) == parse_files(files) + orders = by_name(parse(back["model/cubes/m.yml"])["cubes"])["orders"] + assert [j["name"] for j in orders["joins"]] == ["rates", "users"] + assert issues.of_type(IssueType.PARKED_IN_META) + + +def test_join_clause_written_target_side_first_still_decomposes(): + """Either side of the equality may name either cube.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{users.id} = {CUBE}.user_id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + )) + ossie, back, _ = _roundtrip(files) + rel = model_of(ossie)["relationships"][0] + assert (rel["from_columns"], rel["to_columns"]) == (["user_id"], ["id"]) + assert parse_files(back) == parse_files(files) + + +def test_join_clause_not_spanning_both_cubes_is_preserved(): + """A clause has to relate the two joined cubes. One comparing a cube to itself + (or reaching a third cube) is a valid Cube join with no Ossie relationship form, + so it is preserved verbatim instead of guessed at.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.a = {CUBE}.b\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + )) + ossie, back, issues = _roundtrip(files) + assert "relationships" not in model_of(ossie) + assert any("references cubes other than" in i.detail for i in issues) + assert parse_files(back) == parse_files(files) + + +def test_join_clause_reaching_an_unrelated_cube_is_preserved(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {regions}.id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " - name: regions\n" + " sql_table: public.regions\n" + )) + ossie, back, issues = _roundtrip(files) + assert "relationships" not in model_of(ossie) + assert any("not between two member references" in i.detail for i in issues) + assert parse_files(back) == parse_files(files) + + +def test_join_clause_that_is_not_a_single_equality_is_preserved(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.a = {users}.b = 1\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + )) + ossie, back, issues = _roundtrip(files) + assert "relationships" not in model_of(ossie) + assert any("not a single equality" in i.detail for i in issues) + assert parse_files(back) == parse_files(files) + + +def test_metric_without_a_usable_dialect_is_dropped_with_an_issue(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " metrics:\n" + " - name: m\n" + " expression:\n" + " dialects:\n" + " - dialect: MAQL\n" + " expression: SELECT SUM(x)\n" + ) + files, issues = convert_ossie_to_cube(ossie) + assert "measures" not in parse(files["model/cubes/orders.yml"])["cubes"][0] + assert issues.of_type(IssueType.NO_USABLE_DIALECT) + + +# --- file layout ---------------------------------------------------------------- + +def test_off_layout_files_are_restored_with_their_grouping(): + """Import accepts any layout. Several cubes in one oddly-named file have to go + back into that same file, not be split into the canonical per-cube layout.""" + files = { + "schema/warehouse/everything.yaml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " - name: users\n" + " sql_table: public.users\n" + "views:\n" + " - name: main\n" + " description: All of it\n" + ), + } + ossie, back, _ = _roundtrip(files) + assert set(back) == {"schema/warehouse/everything.yaml"} + assert parse_files(back) == parse_files(files) + stash = stash_of(model_of(ossie)) + assert stash["cube_files"]["orders"] == "schema/warehouse/everything.yaml" + assert stash["view_files"]["main"] == "schema/warehouse/everything.yaml" + + +def test_non_model_yaml_is_preserved_verbatim(): + files = { + "model/cubes/orders.yml": ( + "cubes:\n - name: orders\n sql_table: public.orders\n"), + "model/notes.yaml": "just: some data\n", + } + ossie, back, issues = _roundtrip(files) + assert back["model/notes.yaml"] == "just: some data\n" + assert issues.of_type(IssueType.PARKED_IN_META) + + +# --- the JavaScript-style mapping form ------------------------------------------ + +def test_collections_may_be_mappings_keyed_by_name(): + """Cube's post-transpile schema keys dimensions/measures/joins by name, and a + model converted from JavaScript can carry that shape. Both forms are accepted; + export always emits the list form YAML models use.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " id:\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " status:\n" + " sql: status\n" + " type: string\n" + " measures:\n" + " count:\n" + " type: count\n" + )) + ossie, _ = convert_cube_to_ossie(files) + model = model_of(ossie) + fields = by_name(by_name(model["datasets"])["orders"]["fields"]) + assert set(fields) == {"id", "status"} + assert expr_of(model["metrics"][0]) == "COUNT(DISTINCT orders.id)" + + back, _ = convert_ossie_to_cube(ossie) + cube = parse(back["model/cubes/m.yml"])["cubes"][0] + assert isinstance(cube["dimensions"], list) + assert isinstance(cube["measures"], list) + + +def test_a_collection_of_the_wrong_shape_is_rejected(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions: not-a-collection\n" + )) + with pytest.raises(ConversionError, match="expected a list or mapping"): + convert_cube_to_ossie(files) + + +# --- Jinja ---------------------------------------------------------------------- + +def test_jinja_anywhere_disqualifies_the_whole_file(): + """Jinja is detected per *file*, not per member -- Cube's own CubeSchemaConverter + uses the same file-level rule. So templating inside a single dimension's `sql` + still costs the whole file, which is preserved verbatim rather than + half-converted. There is deliberately no member-level Jinja path.""" + templated = ( + "cubes:\n" + " - name: templated\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: dyn\n" + " sql: \"{{ 'x' }}\"\n" + " type: string\n" + ) + files = { + "model/cubes/templated.yml": templated, + "model/cubes/plain.yml": ( + "cubes:\n - name: plain\n sql_table: public.plain\n"), + } + ossie, issues = convert_cube_to_ossie(files) + model = model_of(ossie) + assert [d["name"] for d in model["datasets"]] == ["plain"] + assert stash_of(model)["extra_files"]["model/cubes/templated.yml"] == templated + assert issues.of_type(IssueType.TEMPLATED_FILE_SKIPPED) + + # And it comes back byte-for-byte, since it was never parsed. + back, _ = convert_ossie_to_cube(ossie) + assert back["model/cubes/templated.yml"] == templated + + +# --- metadata corners ----------------------------------------------------------- + +def test_measure_title_survives_the_round_trip(): + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " measures:\n" + " - name: revenue\n" + " sql: \"{CUBE}.amount\"\n" + " type: sum\n" + " title: Total Revenue\n" + )) + ossie, back, _ = _roundtrip(files) + assert stash_of(model_of(ossie)["metrics"][0])["title"] == "Total Revenue" + assert parse_files(back) == parse_files(files) + + +def test_geo_dimension_extras_survive_the_split_and_merge(): + files = _files(users=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: home\n" + " type: geo\n" + " title: Home Location\n" + " description: Where they live\n" + " latitude:\n" + " sql: \"{CUBE}.lat\"\n" + " longitude:\n" + " sql: \"{CUBE}.lon\"\n" + )) + _, back, issues = _roundtrip(files) + assert parse_files(back) == parse_files(files) + assert issues.of_type(IssueType.GEO_DIMENSION_SPLIT) + + +def test_geo_dimension_missing_a_half_is_rejected(): + files = _files(users=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: home\n" + " type: geo\n" + " latitude:\n" + " sql: lat\n" + )) + with pytest.raises(ConversionError, match="missing 'longitude.sql'"): + convert_cube_to_ossie(files) + + +def test_ai_context_examples_reach_cube_as_prose_and_park_structurally(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " ai_context:\n" + " instructions: Sales model.\n" + " examples:\n" + " - What were sales last month?\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + ) + files, _ = convert_ossie_to_cube(ossie) + meta = parse(files["model/views/shop.yml"])["views"][0]["meta"] + assert meta["ai_context"] == ( + "Sales model.\nExample questions: What were sales last month?") + assert meta["ossie"]["ai_context"]["examples"] == [ + "What were sales last month?"] + # And the structured form is what comes back, not the flattened prose. + ossie2, _ = convert_cube_to_ossie(files) + assert model_of(ossie2)["ai_context"]["examples"] == [ + "What were sales last month?"] + + +def test_a_plain_string_ai_context_survives_as_a_string(): + """Ossie allows `ai_context` to be a bare string. Import reads Cube's prose back + as {'instructions': ...}, so the original scalar has to be parked to survive.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " ai_context: orders, purchases, sales\n" + ) + files, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files) + ds = by_name(model_of(ossie2)["datasets"])["orders"] + assert ds["ai_context"] == "orders, purchases, sales" + + +# --- multiple views ------------------------------------------------------------- + +_TWO_VIEWS = { + "model/cubes/orders.yml": ( + "cubes:\n - name: orders\n sql_table: public.orders\n"), + "model/views/a.yml": "views:\n - name: a\n description: View A\n", + "model/views/b.yml": "views:\n - name: b\n description: View B\n", +} + + +def test_several_views_need_an_explicit_choice(): + _, issues = convert_cube_to_ossie(_TWO_VIEWS) + assert any("none chosen with --view" in i.detail + for i in issues.of_type(IssueType.PARKED_IN_META)) + + +def test_choosing_a_view_maps_its_metadata_onto_the_model(): + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") + model = model_of(ossie) + assert model["name"] == "b" + assert model["description"] == "View B" + # The unchosen view is still preserved whole. + assert set(stash_of(model)["views"]) == {"a", "b"} + + +def test_both_views_are_restored_on_export(): + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") + back, _ = convert_ossie_to_cube(ossie) + assert parse_files(back) == parse_files(_TWO_VIEWS) + + +# --- malformed input ------------------------------------------------------------ + +def test_malformed_yaml_is_reported_cleanly(): + with pytest.raises(ConversionError, match="Invalid YAML"): + convert_cube_to_ossie({"model/cubes/m.yml": "cubes: [oops\n"}) + + +def test_empty_input_is_rejected(): + with pytest.raises(ConversionError, match="non-empty mapping"): + convert_cube_to_ossie({}) + + +def test_a_non_string_name_is_rejected_cleanly(): + files = _files(m="cubes:\n - name: 42\n sql_table: t\n") + with pytest.raises(ConversionError, match="must be a string"): + convert_cube_to_ossie(files) + + +def test_ossie_root_must_be_a_mapping(): + with pytest.raises(ConversionError, match="expected a mapping at the root"): + convert_ossie_to_cube("- just\n- a\n- list\n") + + +def test_measure_without_a_type_is_rejected(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: t\n" + " measures:\n" + " - name: m\n" + " sql: amount\n" + )) + with pytest.raises(ConversionError, match="missing required 'type'"): + convert_cube_to_ossie(files) + + +def test_unknown_dimension_type_is_rejected(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: t\n" + " dimensions:\n" + " - name: d\n" + " sql: d\n" + " type: quaternion\n" + )) + with pytest.raises(ConversionError, match="unknown type 'quaternion'"): + convert_cube_to_ossie(files) + + +def test_unknown_ossie_datatype_is_rejected(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: f\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: f\n" + " datatype: Quaternion\n" + ) + with pytest.raises(ConversionError, match="unknown datatype"): + convert_ossie_to_cube(ossie) + + +def test_dataset_without_a_source_is_rejected_on_export(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + ) + with pytest.raises(ConversionError, match="missing/empty 'source'"): + convert_ossie_to_cube(ossie) + + +def test_several_semantic_models_convert_the_first_with_an_issue(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: first\n" + " datasets:\n" + " - name: orders\n" + " source: t\n" + "- name: second\n" + " datasets:\n" + " - name: users\n" + " source: t\n" + ) + files, issues = convert_ossie_to_cube(ossie) + assert set(files) == {"model/cubes/orders.yml", "model/views/first.yml"} + assert any("converting only the first" in i.detail for i in issues) From 42c561f26ca49fdc7d6d609ba4f86baaafbc26eb Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 00:48:48 +0500 Subject: [PATCH 04/25] Explain a view-only input, accept a single file, and test the CLI Answering "what happens if I pass only a view file?" turned up three things. The behavior was right -- a Cube view projects members from cubes and defines none, so it cannot become an Ossie model on its own and the conversion is refused. But the message said "no convertible cubes found", which reads as if the file was not recognized at all. It now says a view was found, explains why that is not enough, and names the cubes its join_paths reference so the user knows which files to add. Pointing `-i` at a single `.yml` was refused as "not a directory". There is nothing ambiguous about a single model file, so it is now accepted. And cli.py had no tests at all -- 0% coverage. Adds test_cli.py (15 tests) covering the input shapes people reach for first (directory, single file, view-only), that stdout stays pipeable while issues go to stderr, that a fan-out refusal exits non-zero and --no-strict-fanout downgrades it, that node_modules and dotfiles are skipped, and a CLI round trip that reproduces the TPC-DS fixture. 216 tests; coverage 96% -> 97%, cli.py 0% -> 99%. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 21 +- converters/cube/src/ossie_cube/cli.py | 24 +- converters/cube/src/ossie_cube/cube_to_osi.py | 54 +++- converters/cube/tests/test_cli.py | 238 ++++++++++++++++++ 4 files changed, 319 insertions(+), 18 deletions(-) create mode 100644 converters/cube/tests/test_cli.py diff --git a/converters/cube/README.md b/converters/cube/README.md index 83609f2c..edce0ef3 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -67,12 +67,18 @@ ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] ``` -`import` with no `-o` writes the Ossie YAML to stdout; `export` always needs `-o` -(a directory). Issues always go to stderr. `--view` picks which view's -name/description/AI context map onto the Ossie model when the directory holds -several; `--name` overrides the model name. `--base-cube` picks the cube a -*generated* view is rooted at, and is only consulted for a hand-authored Ossie -model with no stashed views. +`import` takes a model directory *or* a single model file, and with no `-o` writes +the Ossie YAML to stdout; `export` always needs `-o` (a directory). Issues always go +to stderr, so stdout stays pipeable. `--view` picks which view's +name/description/AI context map onto the Ossie model when the input holds several; +`--name` overrides the model name. `--base-cube` picks the cube a *generated* view +is rooted at, and is only consulted for a hand-authored Ossie model with no stashed +views. + +**A view on its own is not a model.** A Cube view projects members from cubes and +defines none of its own, so passing only `views/sales.yml` is refused -- with an +error naming the cubes it references, so you know which files to add. Include the +cube files (or point `-i` at the model directory). ### Python API @@ -229,7 +235,8 @@ uv sync uv run pytest ``` -Example-based unit tests per direction, fixture round-trip tests (including the +216 tests at 97% line coverage: example-based unit tests per direction, CLI +behavior tests, fixture round-trip tests (including the [TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks for as a baseline), core-spec JSON Schema validation of every emitted Ossie document, and Hypothesis property-based round-trip tests over generated Cube diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py index 8616a2a7..1f2c2e4b 100644 --- a/converters/cube/src/ossie_cube/cli.py +++ b/converters/cube/src/ossie_cube/cli.py @@ -49,7 +49,8 @@ def _build_parser(): imp = sub.add_parser( "import", help="Cube data model directory -> Apache Ossie semantic model YAML") - imp.add_argument("-i", "--input", required=True, help="Cube model directory") + imp.add_argument("-i", "--input", required=True, + help="Cube model directory, or a single model file") imp.add_argument("-o", "--output", help="output Ossie YAML file (default: stdout)") imp.add_argument("--name", @@ -76,15 +77,22 @@ def _build_parser(): return parser -def _read_model_dir(path): - """Collect every file under a Cube model directory as {relative path: text}. +def _read_model_input(path): + """Collect a Cube model as {relative path: text}. - Everything is collected, not just YAML: a `.js` data model has no Ossie form, - but the converter preserves it so a round trip does not lose the file. Hidden - files and directories (including `node_modules`) are skipped. + `path` is normally a model directory, but a single file is accepted too -- + pointing at one `.yml` is a natural thing to try and there is nothing ambiguous + about it. + + Under a directory everything is collected, not just YAML: a `.js` data model has + no Ossie form, but the converter preserves it so a round trip does not lose the + file. Hidden files and directories (including `node_modules`) are skipped. """ + if os.path.isfile(path): + with open(path) as fh: + return {os.path.basename(path): fh.read()} if not os.path.isdir(path): - raise ConversionError(f"'{path}' is not a directory") + raise ConversionError(f"'{path}' is not a file or directory") files = {} for dirpath, dirnames, filenames in os.walk(path): dirnames[:] = [d for d in sorted(dirnames) @@ -126,7 +134,7 @@ def main(argv=None): _report(issues) return 0 - files = _read_model_dir(args.input) + files = _read_model_input(args.input) out, issues = convert_cube_to_ossie( files, model_name=args.name, view=args.view, strict_fanout=args.strict_fanout) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index bbcd76c6..8ae1bd48 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -112,9 +112,7 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True) cubes, cube_paths, views, view_paths, extra_files = _collect(files, issues) if not cubes: - raise ConversionError( - "no convertible cubes found (a `.yml` file with a top-level `cubes:` " - "list); nothing to convert") + raise ConversionError(_no_cubes_message(views)) # The mapped view supplies the Ossie model's identity. Cube users are # view-first, and Cube's own agent reads `meta.ai_context` only from views and @@ -272,6 +270,56 @@ def _as_named_list(value, what): f"{what}: expected a list or mapping, got {type(value).__name__}") +def _cubes_referenced_by(view): + """The cube names a view's `cubes:` entries address, in order. + + Every segment of a `join_path` names a cube (`orders.users.addresses` reaches + three), so all of them count as referenced. + """ + names = [] + for entry in view.get("cubes") or []: + if not isinstance(entry, dict): + continue + path = entry.get("join_path") + if not isinstance(path, str) or not path: + continue + for segment in path.split("."): + if segment and segment not in names: + names.append(segment) + return names + + +def _no_cubes_message(views): + """Explain *why* there is nothing to convert. + + Being handed only view files is an easy mistake -- a Cube view looks like a + complete model, and it is what a view-first user thinks of as "the model". But a + view only projects members from cubes and defines none of its own, so it cannot + become an Ossie semantic model on its own. Naming the cubes it references turns + the error into instructions. + """ + if not views: + return ("no convertible cubes found (a `.yml` file with a top-level " + "`cubes:` list); nothing to convert") + referenced = [] + for view in views.values(): + for name in _cubes_referenced_by(view): + if name not in referenced: + referenced.append(name) + which = ", ".join(f"'{v}'" for v in sorted(views)) + needed = ( + f" It references {', '.join(repr(c) for c in referenced)}, so include the " + f"file(s) defining those cubes." + if referenced else + " Include the files defining the cubes it draws from." + ) + return ( + f"found only view(s) {which} and no cubes. A Cube view projects members " + f"from cubes rather than defining any, so it has no Ossie dataset to " + f"convert on its own.{needed}" + ) + + def _order_by_view(cubes, mapped_view): """Order the datasets the way the mapped view presents them. diff --git a/converters/cube/tests/test_cli.py b/converters/cube/tests/test_cli.py new file mode 100644 index 00000000..d4c5f761 --- /dev/null +++ b/converters/cube/tests/test_cli.py @@ -0,0 +1,238 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line behavior: what the user actually types, and what they get back. + +Covers the input shapes people reach for first -- a whole model directory, a single +file, and (a common mistake) just the view -- plus the exit codes and where output +goes, since those are the converter's contract with a shell script. +""" + +import pytest +from _util import REPO_ROOT, load_fixture_dir, parse + +from ossie_cube.cli import main + +_ORDERS = ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: count\n" + " type: count\n" +) +_VIEW = ( + "views:\n" + " - name: sales\n" + " description: Sales overview\n" + " cubes:\n" + " - join_path: orders\n" + " includes: '*'\n" +) + + +def _write(root, **files): + for rel, text in files.items(): + path = root / rel.replace("|", "/") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return root + + +# --- input shapes --------------------------------------------------------------- + +def test_a_model_directory_converts(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, "views|sales.yml": _VIEW}) + assert main(["import", "-i", str(model)]) == 0 + doc = parse(capsys.readouterr().out) + assert doc["semantic_model"][0]["name"] == "sales" + + +def test_a_single_file_converts(tmp_path, capsys): + """Pointing at one `.yml` is a natural thing to try and there is nothing + ambiguous about it, so it is accepted rather than refused on a technicality.""" + path = tmp_path / "orders.yml" + path.write_text(_ORDERS) + assert main(["import", "-i", str(path)]) == 0 + doc = parse(capsys.readouterr().out) + assert [d["name"] for d in doc["semantic_model"][0]["datasets"]] == ["orders"] + + +def test_only_a_view_file_is_refused_with_an_actionable_message(tmp_path, capsys): + """The likeliest mistake for a view-first user: a Cube view looks like the whole + model, but it projects members from cubes and defines none, so the error names + the cubes whose files are missing rather than claiming nothing was recognized.""" + path = tmp_path / "sales.yml" + path.write_text(_VIEW) + assert main(["import", "-i", str(path)]) == 1 + err = capsys.readouterr().err + assert "found only view(s) 'sales' and no cubes" in err + assert "projects members from cubes" in err + assert "'orders'" in err # named from the view's join_path + + +def test_a_view_with_no_cube_references_still_explains_itself(tmp_path, capsys): + path = tmp_path / "bare.yml" + path.write_text("views:\n - name: sales\n description: Sales\n") + assert main(["import", "-i", str(path)]) == 1 + assert "Include the files defining the cubes it draws from" in \ + capsys.readouterr().err + + +def test_a_missing_path_is_reported_not_traced(tmp_path, capsys): + assert main(["import", "-i", str(tmp_path / "nope")]) == 1 + assert "is not a file or directory" in capsys.readouterr().err + + +def test_an_empty_directory_is_reported(tmp_path, capsys): + empty = tmp_path / "empty" + empty.mkdir() + assert main(["import", "-i", str(empty)]) == 1 + assert "holds no files" in capsys.readouterr().err + + +def test_node_modules_and_dotfiles_are_skipped(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, + "node_modules|junk.yml": "cubes:\n - name: junk\n sql_table: t\n", + ".hidden.yml": "cubes:\n - name: hidden\n sql_table: t\n", + }) + assert main(["import", "-i", str(model)]) == 0 + doc = parse(capsys.readouterr().out) + assert [d["name"] for d in doc["semantic_model"][0]["datasets"]] == ["orders"] + + +# --- output and exit codes ------------------------------------------------------ + +def test_output_goes_to_a_file_when_asked(tmp_path, capsys): + model = _write(tmp_path / "model", **{"cubes|orders.yml": _ORDERS}) + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(model), "-o", str(out)]) == 0 + assert capsys.readouterr().out == "" + assert parse(out.read_text())["semantic_model"][0]["datasets"] + + +def test_issues_go_to_stderr_so_stdout_stays_pipeable(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|users.yml": ( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: home\n" + " type: geo\n" + " latitude:\n" + " sql: lat\n" + " longitude:\n" + " sql: lon\n" + )}) + assert main(["import", "-i", str(model)]) == 0 + captured = capsys.readouterr() + assert "GEO_DIMENSION_SPLIT" in captured.err + assert "conversion issue" in captured.err + parse(captured.out) # stdout is still clean YAML + + +def test_fanout_refusal_exits_nonzero_and_the_flag_downgrades_it(tmp_path, capsys): + model = _write(tmp_path / "model", **{"cubes|m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: ltv\n" + " sql: \"{CUBE}.ltv\"\n" + " type: sum\n" + )}) + assert main(["import", "-i", str(model)]) == 1 + assert "FANOUT_UNSAFE_METRIC" in capsys.readouterr().err + + assert main(["import", "-i", str(model), "--no-strict-fanout"]) == 0 + captured = capsys.readouterr() + assert "FANOUT_UNSAFE_METRIC" in captured.err + assert parse(captured.out)["semantic_model"][0]["metrics"] + + +def test_view_and_name_flags_take_effect(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, + "views|a.yml": "views:\n - name: a\n description: A\n", + "views|b.yml": "views:\n - name: b\n description: B\n", + }) + assert main(["import", "-i", str(model), "--view", "b"]) == 0 + assert parse(capsys.readouterr().out)["semantic_model"][0]["description"] == "B" + + assert main(["import", "-i", str(model), "--view", "b", + "--name", "custom"]) == 0 + assert parse(capsys.readouterr().out)["semantic_model"][0]["name"] == "custom" + + assert main(["import", "-i", str(model), "--view", "ghost"]) == 1 + assert "not found" in capsys.readouterr().err + + +# --- export --------------------------------------------------------------------- + +def test_export_writes_the_model_directory(tmp_path, capsys): + out = tmp_path / "out" + assert main(["export", "-i", + str(REPO_ROOT / "examples" / "tpcds_semantic_model.yaml"), + "-o", str(out)]) == 0 + assert (out / "model" / "cubes" / "store_sales.yml").is_file() + assert (out / "model" / "views" / "tpcds_retail_model.yml").is_file() + assert "Wrote 6 file(s)" in capsys.readouterr().err + + +def test_export_of_a_missing_input_is_reported(tmp_path, capsys): + assert main(["export", "-i", str(tmp_path / "nope.yaml"), + "-o", str(tmp_path / "out")]) == 1 + assert "Error:" in capsys.readouterr().err + + +def test_a_cli_round_trip_reproduces_the_fixture(tmp_path, capsys): + fixture = load_fixture_dir("tpcds_cube") + src = _write(tmp_path / "src", **{k.replace("/", "|"): v + for k, v in fixture.items()}) + ossie = tmp_path / "model.yaml" + back = tmp_path / "back" + assert main(["import", "-i", str(src), "-o", str(ossie)]) == 0 + assert main(["export", "-i", str(ossie), "-o", str(back)]) == 0 + capsys.readouterr() + for rel in fixture: + assert (back / rel.replace("/", "/")).is_file(), rel + assert parse((back / rel).read_text()) == parse(fixture[rel]) + + +def test_no_subcommand_is_a_usage_error(): + with pytest.raises(SystemExit) as excinfo: + main([]) + assert excinfo.value.code == 2 From f39803e6c2bc5c4b448a3f9726a40b176c94b874 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 01:02:45 +0500 Subject: [PATCH 05/25] tests: add more edge cases --- converters/cube/tests/test_edge_cases.py | 80 ++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 84874100..22b650dd 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -397,6 +397,86 @@ def test_off_layout_files_are_restored_with_their_grouping(): assert stash["view_files"]["main"] == "schema/warehouse/everything.yaml" +_MIXED_VIEW_FILE = ( + "views:\n" + " - name: sales\n" + " description: Sales overview\n" + " meta:\n" + " ai_context: Use for revenue questions.\n" + " cubes:\n" + " - join_path: orders\n" + " includes: '*'\n" + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: revenue\n" + " sql: \"{CUBE}.amount\"\n" + " type: sum\n" +) + + +def test_a_view_file_may_also_define_cubes(): + """`cubes:` and `views:` are independent top-level keys, so one file can hold + both -- a self-contained model. Note the view's own nested `cubes:` (its + include list) is a different key at a different level and is not confused with + cube definitions.""" + files = {"model/views/sales.yml": _MIXED_VIEW_FILE} + ossie, _ = convert_cube_to_ossie(files) + model = model_of(ossie) + # The view supplied the model identity... + assert model["name"] == "sales" + assert model["description"] == "Sales overview" + assert model["ai_context"]["instructions"] == "Use for revenue questions." + # ...and the cube in the same file became the dataset. + assert [d["name"] for d in model["datasets"]] == ["orders"] + assert expr_of(model["metrics"][0]) == "SUM(orders.amount)" + # The view's include list round-trips as curation, not as a dataset. + assert stash_of(model)["views"]["sales"]["cubes"] == [ + {"join_path": "orders", "includes": "*"}] + + +def test_a_mixed_file_is_rebuilt_as_one_file(): + """Both halves have to go back into the single file they came from, rather than + being split into the canonical per-cube and per-view layout.""" + files = {"model/views/sales.yml": _MIXED_VIEW_FILE} + _, back, _ = _roundtrip(files) + assert set(back) == {"model/views/sales.yml"} + assert parse_files(back) == parse_files(files) + rebuilt = parse(back["model/views/sales.yml"]) + assert [c["name"] for c in rebuilt["cubes"]] == ["orders"] + assert [v["name"] for v in rebuilt["views"]] == ["sales"] + + +def test_a_cube_file_may_also_define_views(): + """The mirror image: the canonical cube path holding the view. The view's path is + the off-layout one here, so it is the one that gets stashed.""" + files = {"model/cubes/orders.yml": _MIXED_VIEW_FILE} + ossie, back, _ = _roundtrip(files) + assert stash_of(model_of(ossie))["view_files"]["sales"] == ( + "model/cubes/orders.yml") + assert "cube_files" not in stash_of(model_of(ossie)) + assert set(back) == {"model/cubes/orders.yml"} + assert parse_files(back) == parse_files(files) + + +def test_a_single_monolithic_file_round_trips(): + """Neither path is canonical, so both are stashed and both return to the one + file -- the shape you get from `-i model.yml`.""" + files = {"model.yml": _MIXED_VIEW_FILE} + ossie, back, _ = _roundtrip(files) + stash = stash_of(model_of(ossie)) + assert stash["cube_files"]["orders"] == "model.yml" + assert stash["view_files"]["sales"] == "model.yml" + assert set(back) == {"model.yml"} + assert parse_files(back) == parse_files(files) + + def test_non_model_yaml_is_preserved_verbatim(): files = { "model/cubes/orders.yml": ( From c1e778ecba2f8259da5994e08800d5d462a9fb82 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 01:24:32 +0500 Subject: [PATCH 06/25] Accept several input paths on import, not just one root Cube itself has a single model root (`CUBEJS_SCHEMA_PATH` is one string, default `model`), so pointing at that root is the idiomatic whole-project case and the recursive walk already handles files spread across subdirectories under it. But *requiring* one path was friction: converting two cubes out of fifty, or files that live in separate trees, meant assembling a directory first just to satisfy the CLI. `-i` now takes any number of files and/or directories, so globs work too. Files are keyed relative to the deepest directory containing every input, because those keys decide where export writes them back. That generalization is deliberately behavior-preserving: one directory anchors to itself and one file to its own directory, so existing round trips key exactly as before. Two files from `cubes/` and `views/` key as `cubes/orders.yml` and `views/sales.yml`, and export reproduces that tree. Overlapping inputs (a directory plus a file inside it) are now an error rather than reading the same file twice. Verified on a real Cube model passed as two explicit file paths: round trip is content-identical with the original filenames preserved, the Ossie output passes validation/validate.py, and the fan-out guard correctly refuses both measures on the joined cube's `one` side. 226 tests, 96% line coverage. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 31 ++++++--- converters/cube/src/ossie_cube/cli.py | 96 ++++++++++++++++++--------- converters/cube/tests/test_cli.py | 74 +++++++++++++++++++++ 3 files changed, 162 insertions(+), 39 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index edce0ef3..118f2884 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -67,13 +67,28 @@ ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] ``` -`import` takes a model directory *or* a single model file, and with no `-o` writes -the Ossie YAML to stdout; `export` always needs `-o` (a directory). Issues always go -to stderr, so stdout stays pipeable. `--view` picks which view's -name/description/AI context map onto the Ossie model when the input holds several; -`--name` overrides the model name. `--base-cube` picks the cube a *generated* view -is rooted at, and is only consulted for a hand-authored Ossie model with no stashed -views. +`import` accepts a model directory (walked recursively), individual files, or any +mix of several — so converting part of a model does not mean assembling a directory +first: + +```bash +ossie-cube import -i model/ # the whole model +ossie-cube import -i model/cubes/orders.yml # one file +ossie-cube import -i model/cubes/orders.yml model/views/*.yml # a subset +``` + +Cube itself has a single model root (`CUBEJS_SCHEMA_PATH` is one path), so pointing +at that root is the idiomatic whole-project case. With several paths, files are keyed +relative to their common parent directory — which is what decides where `export` +writes them back — and the single-directory and single-file cases are keyed exactly +as they would be alone. + +With no `-o`, `import` writes the Ossie YAML to stdout; `export` always needs `-o` (a +directory). Issues always go to stderr, so stdout stays pipeable. `--view` picks +which view's name/description/AI context map onto the Ossie model when the input +holds several; `--name` overrides the model name. `--base-cube` picks the cube a +*generated* view is rooted at, and is only consulted for a hand-authored Ossie model +with no stashed views. **A view on its own is not a model.** A Cube view projects members from cubes and defines none of its own, so passing only `views/sales.yml` is refused -- with an @@ -235,7 +250,7 @@ uv sync uv run pytest ``` -216 tests at 97% line coverage: example-based unit tests per direction, CLI +226 tests at 96% line coverage: example-based unit tests per direction, CLI behavior tests, fixture round-trip tests (including the [TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks for as a baseline), core-spec JSON Schema validation of every emitted Ossie diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py index 1f2c2e4b..ce4804b1 100644 --- a/converters/cube/src/ossie_cube/cli.py +++ b/converters/cube/src/ossie_cube/cli.py @@ -18,12 +18,15 @@ """Command-line interface for the Apache Ossie <-> Cube converter. ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] + ossie-cube import -i cubes/orders.yml cubes/users.yml views/sales.yml ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] -`import` converts a Cube data model directory (any `.yml` holding `cubes:` / -`views:`) into an Apache Ossie semantic model; with no `-o` the Ossie YAML goes to -stdout. `export` does the reverse and always needs `-o` (a directory). -Conversions that could not carry something across print an issue list to stderr. +`import` converts a Cube data model (any `.yml` holding `cubes:` / `views:`) into an +Apache Ossie semantic model; with no `-o` the Ossie YAML goes to stdout. It accepts +a model directory, individual files, or a mix of several -- so converting part of a +model does not mean assembling a directory first. `export` does the reverse and +always needs `-o` (a directory). Conversions that could not carry something across +print an issue list to stderr. By default a metric whose value a static Ossie expression cannot keep correct under row multiplication is refused on import, mirroring Cube's own refusal to @@ -49,8 +52,11 @@ def _build_parser(): imp = sub.add_parser( "import", help="Cube data model directory -> Apache Ossie semantic model YAML") - imp.add_argument("-i", "--input", required=True, - help="Cube model directory, or a single model file") + imp.add_argument("-i", "--input", required=True, nargs="+", + metavar="PATH", + help="Cube model directories and/or files. A directory is " + "walked recursively; several paths are merged, keyed " + "relative to their common parent (globs work)") imp.add_argument("-o", "--output", help="output Ossie YAML file (default: stdout)") imp.add_argument("--name", @@ -77,36 +83,64 @@ def _build_parser(): return parser -def _read_model_input(path): - """Collect a Cube model as {relative path: text}. +def _read_model_input(paths): + """Collect a Cube model as {relative path: text} from one or more paths. - `path` is normally a model directory, but a single file is accepted too -- - pointing at one `.yml` is a natural thing to try and there is nothing ambiguous - about it. + Cube itself has a single model root (`CUBEJS_SCHEMA_PATH`, one string), so + pointing at a model directory is the idiomatic whole-project case. But + converting part of a model -- two cubes out of fifty, or files that live in + different trees -- is a real workflow, so several paths merge into one model + rather than forcing the caller to assemble a directory first. - Under a directory everything is collected, not just YAML: a `.js` data model has - no Ossie form, but the converter preserves it so a round trip does not lose the - file. Hidden files and directories (including `node_modules`) are skipped. + Keys are relative to the deepest directory containing every input, which + leaves the single-directory and single-file cases keyed exactly as before. + Directories are walked recursively, collecting everything rather than only + YAML: a `.js` data model has no Ossie form, but the converter preserves it so a + round trip does not lose the file. Hidden files and directories (including + `node_modules`) are skipped. """ - if os.path.isfile(path): - with open(path) as fh: - return {os.path.basename(path): fh.read()} - if not os.path.isdir(path): - raise ConversionError(f"'{path}' is not a file or directory") + resolved = [os.path.abspath(p) for p in paths] + for path, original in zip(resolved, paths): + if not os.path.exists(path): + raise ConversionError(f"'{original}' is not a file or directory") + + # The anchor keys every file. Using the inputs' common parent means one + # directory anchors to itself and one file to its own directory, so those + # cases are unchanged; several inputs stay distinguishable from each other. + containers = [p if os.path.isdir(p) else os.path.dirname(p) for p in resolved] + try: + anchor = os.path.commonpath(containers) + except ValueError: + # No shared prefix at all (different drives on Windows); fall back to bare + # file names, which are still unique or else reported as a collision below. + anchor = None + files = {} - for dirpath, dirnames, filenames in os.walk(path): - dirnames[:] = [d for d in sorted(dirnames) - if not d.startswith(".") and d != "node_modules"] - for fname in sorted(filenames): - if fname.startswith("."): - continue - rel = os.path.relpath(os.path.join(dirpath, fname), path) - rel = rel.replace(os.sep, "/") - with open(os.path.join(dirpath, fname)) as fh: - files[rel] = fh.read() + for path in resolved: + if os.path.isfile(path): + _collect_file(files, path, anchor) + continue + for dirpath, dirnames, filenames in os.walk(path): + dirnames[:] = [d for d in sorted(dirnames) + if not d.startswith(".") and d != "node_modules"] + for fname in sorted(filenames): + if not fname.startswith("."): + _collect_file(files, os.path.join(dirpath, fname), anchor) if not files: - raise ConversionError(f"'{path}' holds no files") - return files + raise ConversionError( + f"{', '.join(repr(p) for p in paths)} holds no files") + return dict(sorted(files.items())) + + +def _collect_file(files, path, anchor): + rel = (os.path.basename(path) if anchor is None + else os.path.relpath(path, anchor)).replace(os.sep, "/") + if rel in files: + raise ConversionError( + f"two inputs both resolve to '{rel}'; pass their common parent " + f"directory instead, or rename one") + with open(path) as fh: + files[rel] = fh.read() def _report(issues): diff --git a/converters/cube/tests/test_cli.py b/converters/cube/tests/test_cli.py index d4c5f761..e608bd4f 100644 --- a/converters/cube/tests/test_cli.py +++ b/converters/cube/tests/test_cli.py @@ -78,6 +78,80 @@ def test_a_single_file_converts(tmp_path, capsys): assert [d["name"] for d in doc["semantic_model"][0]["datasets"]] == ["orders"] +def test_several_paths_merge_into_one_model(tmp_path, capsys): + """Cube has a single model root, but converting part of a model -- or files from + different trees -- should not require assembling a directory first.""" + a = tmp_path / "cubes" / "orders.yml" + b = tmp_path / "views" / "sales.yml" + for path, text in ((a, _ORDERS), (b, _VIEW)): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + assert main(["import", "-i", str(a), str(b)]) == 0 + model = parse(capsys.readouterr().out)["semantic_model"][0] + assert model["name"] == "sales" # the view was picked up + assert [d["name"] for d in model["datasets"]] == ["orders"] + + +def test_several_paths_are_keyed_relative_to_their_common_parent(tmp_path, capsys): + """The keys decide where export writes the files back, so two inputs from + different subtrees have to stay distinguishable.""" + a = tmp_path / "cubes" / "orders.yml" + b = tmp_path / "views" / "sales.yml" + for path, text in ((a, _ORDERS), (b, _VIEW)): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(a), str(b), "-o", str(out)]) == 0 + back = tmp_path / "back" + assert main(["export", "-i", str(out), "-o", str(back)]) == 0 + capsys.readouterr() + assert (back / "cubes" / "orders.yml").is_file() + assert (back / "views" / "sales.yml").is_file() + + +def test_mixing_a_directory_and_a_file_works(tmp_path, capsys): + model = _write(tmp_path / "model", **{"cubes|orders.yml": _ORDERS}) + extra = tmp_path / "extra.yml" + extra.write_text(_VIEW) + assert main(["import", "-i", str(model), str(extra)]) == 0 + assert parse(capsys.readouterr().out)["semantic_model"][0]["name"] == "sales" + + +def test_overlapping_inputs_are_reported(tmp_path, capsys): + """Passing a directory and a file inside it is an easy mistake (an overlapping + glob), and it would otherwise read the same file twice.""" + model = _write(tmp_path / "model", **{"cubes|orders.yml": _ORDERS}) + assert main(["import", "-i", str(model), + str(model / "cubes" / "orders.yml")]) == 1 + err = capsys.readouterr().err + assert "both resolve to 'cubes/orders.yml'" in err + + +def test_the_same_cube_in_two_inputs_is_reported(tmp_path, capsys): + a = tmp_path / "one" / "orders.yml" + b = tmp_path / "two" / "orders.yml" + for path in (a, b): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(_ORDERS) + # Distinct keys ('one/orders.yml', 'two/orders.yml'), but the same cube name. + assert main(["import", "-i", str(a), str(b)]) == 1 + assert "defined twice" in capsys.readouterr().err + + +def test_a_single_path_is_keyed_exactly_as_before(tmp_path, capsys): + """The multi-path anchor must not change the one-directory case, since the keys + are what export writes back.""" + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, "views|sales.yml": _VIEW}) + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(model), "-o", str(out)]) == 0 + back = tmp_path / "back" + assert main(["export", "-i", str(out), "-o", str(back)]) == 0 + capsys.readouterr() + assert (back / "cubes" / "orders.yml").is_file() + assert (back / "views" / "sales.yml").is_file() + + def test_only_a_view_file_is_refused_with_an_actionable_message(tmp_path, capsys): """The likeliest mistake for a view-first user: a Cube view looks like the whole model, but it projects members from cubes and defines none, so the error names From c24f7bf9c0f3657f6cc8a7a05c2996011766d7c7 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 01:34:42 +0500 Subject: [PATCH 07/25] Register CUBE in the converters supported-vendors table The converter emits vendor_name: CUBE in custom_extensions, so it belongs in the table converters/README.md keeps of vendors with defined extensions. Deliberately not touching the parallel list in core-spec/spec.md: vendor_name is a free-form string, so no spec change is needed, and edits under core-spec/ carry the heavier review process. Co-Authored-By: Claude Opus 5 --- converters/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/converters/README.md b/converters/README.md index 5c9a4d54..ee0b6099 100644 --- a/converters/README.md +++ b/converters/README.md @@ -76,6 +76,7 @@ The Ossie specification currently defines extensions for the following vendors: | `OMNI` | Omni semantic model | | `WISDOM` | WisdomAI domain | | `NVIDIA_GSF` | NVIDIA Generative Semantic Fabric standalone YAML | +| `CUBE` | Cube data model | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. From dd00190968578a4b3255d30c1431ea8fe75d8cca Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 12:12:08 +0500 Subject: [PATCH 08/25] Add Ossie-side test fixtures and snapshot both directions Every other bidirectional converter in the repo pairs a vendor fixture with an Ossie one (databricks, omni, gooddata, orionbelt); this converter was the only one keeping its Ossie inputs as inline Python strings. Adds fixtureA_ossie.yaml and tpcds_ossie.yaml, and moves the hand-authored Ossie model out of test_roundtrip.py into hand_authored_ossie.yaml. The point is not just tidiness. Following the databricks pattern, the fixtures are asserted as whole-document snapshots in both directions: import must reproduce the Ossie fixture, and exporting that fixture must reproduce the Cube fixture. Field-level assertions cannot see an unintended change elsewhere in the document; a snapshot shows it as a readable diff. Each fixture carries the command to regenerate it. Co-Authored-By: Claude Opus 5 --- .../cube/tests/fixtures/fixtureA_ossie.yaml | 203 ++++++ .../tests/fixtures/hand_authored_ossie.yaml | 95 +++ .../cube/tests/fixtures/tpcds_ossie.yaml | 645 ++++++++++++++++++ converters/cube/tests/test_roundtrip.py | 110 +-- 4 files changed, 974 insertions(+), 79 deletions(-) create mode 100644 converters/cube/tests/fixtures/fixtureA_ossie.yaml create mode 100644 converters/cube/tests/fixtures/hand_authored_ossie.yaml create mode 100644 converters/cube/tests/fixtures/tpcds_ossie.yaml diff --git a/converters/cube/tests/fixtures/fixtureA_ossie.yaml b/converters/cube/tests/fixtures/fixtureA_ossie.yaml new file mode 100644 index 00000000..d78abf3d --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_ossie.yaml @@ -0,0 +1,203 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# The Ossie form of fixtureA_cube/, asserted as a whole-document snapshot so an +# unintended change anywhere in the output shows up as a readable diff. +# Regenerate with: uv run ossie-cube import -i tests/fixtures/fixtureA_cube + +version: 0.2.0.dev0 +semantic_model: +- name: sales + description: Sales overview + ai_context: + instructions: | + Primary view for revenue analysis. Use it for any question about sales, orders, or customer spend. + datasets: + - name: orders + source: public.orders + description: Customer orders + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "id", "type": "number"}' + - name: user_id + expression: + dialects: + - dialect: ANSI_SQL + expression: user_id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "user_id", "type": "number"}' + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: status + datatype: String + label: Order Status + description: Current order status + ai_context: + instructions: Values are pending, shipped, and completed. + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "status"}' + - name: created_at + expression: + dialects: + - dialect: ANSI_SQL + expression: created_at + datatype: DateTime + dimension: + is_time: true + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "created_at"}' + - name: is_large + expression: + dialects: + - dialect: ANSI_SQL + expression: amount > 500 + datatype: Boolean + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "{CUBE}.amount > 500"}' + primary_key: + - id + - name: users + source: SELECT * FROM public.users WHERE deleted_at IS NULL + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "id", "type": "number"}' + - name: city + expression: + dialects: + - dialect: ANSI_SQL + expression: city + datatype: String + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "city"}' + - name: location_latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: lat + datatype: Float + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "geo": {"of": "location", "part": "latitude", "sql": "{CUBE}.lat"}}' + - name: location_longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: lon + datatype: Float + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "geo": {"of": "location", "part": "longitude", "sql": "{CUBE}.lon"}}' + primary_key: + - id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube_extras": {"segments": [{"name": "active", "sql": "{CUBE}.status + = ''active''"}]}}' + relationships: + - name: orders_to_users + from: orders + to: users + from_columns: + - user_id + to_columns: + - id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "declared_on": "orders", "relationship": "many_to_one", "sql": + "{CUBE}.user_id = {users}.id"}' + metrics: + - name: orders__count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT orders.id) + datatype: Integer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "name": "count"}' + - name: total_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + description: Total order amount + ai_context: + instructions: Use this for revenue questions. + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "measure": {"name": "total_amount", "sql": + "{CUBE}.amount", "type": "sum", "format": "currency"}}' + - name: completed_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(CASE WHEN (orders.status = 'completed') THEN orders.amount + END) + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "measure": {"name": "completed_amount", "sql": + "{CUBE}.amount", "type": "sum", "filters": [{"sql": "{CUBE}.status = ''completed''"}]}}' + - name: avg_order_value + expression: + dialects: + - dialect: ANSI_SQL + expression: (SUM(orders.amount)) / (COUNT(DISTINCT orders.id)) + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "measure": {"name": "avg_order_value", "sql": + "{total_amount} / {count}", "type": "number"}}' + - name: users__count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT users.id) + datatype: Integer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "users", "name": "count"}' + - name: cities + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT users.city) + datatype: Integer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "users", "sql": "{CUBE}.city"}' + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "views": {"sales": {"name": "sales", "cubes": [{"join_path": + "orders", "includes": "*"}, {"join_path": "orders.users", "includes": ["city"]}]}}, + "mapped_view": "sales"}' diff --git a/converters/cube/tests/fixtures/hand_authored_ossie.yaml b/converters/cube/tests/fixtures/hand_authored_ossie.yaml new file mode 100644 index 00000000..e62f18f4 --- /dev/null +++ b/converters/cube/tests/fixtures/hand_authored_ossie.yaml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# An Ossie model authored by hand rather than imported from Cube: no +# custom_extensions[CUBE] stash anywhere. Export therefore has to derive +# everything -- the cube layout, the join orientation, and a generated view -- +# instead of restoring it. Also carries the constructs Cube has no field for +# (unique_keys, a foreign vendor's extensions, structured ai_context), which +# export parks under meta.ossie. + +version: 0.2.0.dev0 +semantic_model: +- name: ecommerce + description: Orders and customers + ai_context: + instructions: Use for sales analysis. + synonyms: + - sales + - purchases + datasets: + - name: orders + source: sales.public.orders + primary_key: + - id + unique_keys: + - - order_number + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Integer + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + datatype: Integer + - name: ordered_at + expression: + dialects: + - dialect: ANSI_SQL + expression: ordered_at + datatype: Date + custom_extensions: + - vendor_name: SNOWFLAKE + data: '{"warehouse": "ANALYTICS_WH"}' + - name: customers + source: sales.public.customers + primary_key: + - id + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Integer + - name: email + expression: + dialects: + - dialect: ANSI_SQL + expression: LOWER(email) + datatype: String + relationships: + - name: orders_to_customers + from: orders + to: customers + from_columns: + - customer_id + to_columns: + - id + metrics: + - name: total_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + description: Total revenue + datatype: Decimal diff --git a/converters/cube/tests/fixtures/tpcds_ossie.yaml b/converters/cube/tests/fixtures/tpcds_ossie.yaml new file mode 100644 index 00000000..5ee7e1ae --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_ossie.yaml @@ -0,0 +1,645 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# The Ossie form of tpcds_cube/, asserted as a whole-document snapshot. +# Regenerate with: uv run ossie-cube import -i tests/fixtures/tpcds_cube + +version: 0.2.0.dev0 +semantic_model: +- name: tpcds_retail_model + description: TPC-DS retail semantic model for sales and customer analytics + ai_context: + instructions: Use this semantic model for retail analytics. It provides comprehensive + sales, customer, product, and store data from the TPC-DS benchmark. The model + supports time-based analysis, customer segmentation, product performance, and + store operations metrics. + datasets: + - name: store_sales + source: tpcds.public.store_sales + description: Fact table containing all store sales transactions + ai_context: + synonyms: + - sales transactions + - store purchases + - retail sales + - POS data + unique_keys: + - - ss_item_sk + - ss_ticket_number + fields: + - name: ss_sold_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sold_date_sk + description: Foreign key to date dimension + ai_context: + synonyms: + - sale date + - transaction date + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_sold_date_sk", "type": "number"}' + - name: ss_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_item_sk + description: Foreign key to item dimension + ai_context: + synonyms: + - product + - item + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_item_sk", "type": "number"}' + - name: ss_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_customer_sk + description: Foreign key to customer dimension + ai_context: + synonyms: + - customer + - buyer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_customer_sk", "type": "number"}' + - name: ss_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_store_sk + description: Foreign key to store dimension + ai_context: + synonyms: + - store + - location + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_store_sk", "type": "number"}' + - name: ss_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_quantity + description: Quantity of items sold + ai_context: + synonyms: + - units sold + - quantity + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_quantity", "type": "number"}' + - name: ss_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sales_price + description: Sales price per unit + ai_context: + synonyms: + - unit price + - price + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_sales_price", "type": "number"}' + - name: ss_ext_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ext_sales_price + description: Extended sales price (quantity * price) + ai_context: + synonyms: + - total price + - line total + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_ext_sales_price", "type": "number"}' + - name: ss_net_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_net_profit + description: Net profit from the sale + ai_context: + synonyms: + - profit + - margin + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_net_profit", "type": "number"}' + - name: ss_ticket_number + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ticket_number + datatype: String + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "ss_ticket_number", "public": false}' + primary_key: + - ss_item_sk + - ss_ticket_number + - name: date_dim + source: tpcds.public.date_dim + description: Date dimension with calendar attributes + ai_context: + synonyms: + - calendar + - dates + - time periods + unique_keys: + - - d_date_sk + fields: + - name: d_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date_sk + description: Surrogate key for date + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "d_date_sk", "type": "number"}' + - name: d_date + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date + datatype: DateTime + dimension: + is_time: true + description: Actual date value + ai_context: + synonyms: + - date + - calendar date + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "d_date"}' + - name: d_year + expression: + dialects: + - dialect: ANSI_SQL + expression: d_year + description: Year + ai_context: + synonyms: + - year + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "d_year", "type": "number"}' + - name: d_quarter_name + expression: + dialects: + - dialect: ANSI_SQL + expression: d_quarter_name + datatype: DateTime + dimension: + is_time: true + description: Quarter name (e.g., 2024Q1) + ai_context: + synonyms: + - quarter + - fiscal quarter + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "d_quarter_name"}' + - name: d_month_name + expression: + dialects: + - dialect: ANSI_SQL + expression: d_month_name + datatype: DateTime + dimension: + is_time: true + description: Month name + ai_context: + synonyms: + - month + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "d_month_name"}' + primary_key: + - d_date_sk + - name: customer + source: tpcds.public.customer + description: Customer dimension with demographic information + ai_context: + synonyms: + - customers + - shoppers + - buyers + unique_keys: + - - c_customer_sk + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_sk + description: Surrogate key for customer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "c_customer_sk", "type": "number"}' + - name: c_customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_id + datatype: String + description: Business key for customer + ai_context: + synonyms: + - customer ID + - customer number + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "c_customer_id"}' + - name: c_first_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name + datatype: String + description: Customer first name + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "c_first_name"}' + - name: c_last_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_last_name + datatype: String + description: Customer last name + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "c_last_name"}' + - name: customer_full_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name || ' ' || c_last_name + datatype: String + description: Customer full name (computed field) + ai_context: + synonyms: + - full name + - customer name + - name: c_email_address + expression: + dialects: + - dialect: ANSI_SQL + expression: c_email_address + datatype: String + description: Customer email address + ai_context: + synonyms: + - email + - contact + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "c_email_address"}' + primary_key: + - c_customer_sk + - name: item + source: tpcds.public.item + description: Item/Product dimension with product attributes + ai_context: + synonyms: + - products + - items + - merchandise + unique_keys: + - - i_item_sk + fields: + - name: i_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_sk + description: Surrogate key for item + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_item_sk", "type": "number"}' + - name: i_item_id + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_id + datatype: String + description: Business key for item + ai_context: + synonyms: + - item ID + - product ID + - SKU + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_item_id"}' + - name: i_item_desc + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_desc + datatype: String + description: Item description + ai_context: + synonyms: + - product description + - item name + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_item_desc"}' + - name: i_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: i_brand + datatype: String + description: Brand name + ai_context: + synonyms: + - brand + - manufacturer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_brand"}' + - name: i_category + expression: + dialects: + - dialect: ANSI_SQL + expression: i_category + datatype: String + description: Item category + ai_context: + synonyms: + - product category + - department + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_category"}' + - name: i_current_price + expression: + dialects: + - dialect: ANSI_SQL + expression: i_current_price + description: Current price of the item + ai_context: + synonyms: + - price + - list price + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "i_current_price", "type": "number"}' + primary_key: + - i_item_sk + - name: store + source: tpcds.public.store + description: Store dimension with location and store attributes + ai_context: + synonyms: + - stores + - retail locations + - branches + unique_keys: + - - s_store_id + fields: + - name: s_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_sk + description: Surrogate key for store + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_store_sk", "type": "number"}' + - name: s_store_id + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_id + datatype: String + description: Business key for store + ai_context: + synonyms: + - store ID + - store number + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_store_id"}' + - name: s_store_name + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_name + datatype: String + description: Store name + ai_context: + synonyms: + - store name + - location name + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_store_name"}' + - name: s_city + expression: + dialects: + - dialect: ANSI_SQL + expression: s_city + datatype: String + description: City where store is located + ai_context: + synonyms: + - city + - location + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_city"}' + - name: s_state + expression: + dialects: + - dialect: ANSI_SQL + expression: s_state + datatype: String + description: State where store is located + ai_context: + synonyms: + - state + - region + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_state"}' + - name: s_number_employees + expression: + dialects: + - dialect: ANSI_SQL + expression: s_number_employees + description: Number of employees at the store + ai_context: + synonyms: + - employee count + - staff size + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "s_number_employees", "type": "number"}' + primary_key: + - s_store_sk + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' + metrics: + - name: total_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales revenue across all transactions + ai_context: + synonyms: + - total revenue + - gross sales + - sales amount + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_ext_sales_price}"}' + - name: total_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_net_profit) + description: Total net profit from store sales + ai_context: + synonyms: + - net profit + - total earnings + - profit + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_net_profit}"}' + - name: customer_lifetime_value + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) + description: Average lifetime sales value per customer + ai_context: + synonyms: + - CLV + - LTV + - customer value + - lifetime revenue + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "measure": {"name": "customer_lifetime_value", + "sql": "SUM({CUBE.ss_ext_sales_price}) / COUNT(DISTINCT {customer.c_customer_sk})", + "type": "number"}}' + - name: sales_by_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales by brand (requires grouping by item.i_brand) + ai_context: + synonyms: + - brand sales + - brand performance + - brand revenue + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_ext_sales_price}"}' + - name: store_productivity + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), + 0) + description: Sales per employee across stores + ai_context: + synonyms: + - sales per employee + - employee productivity + - revenue per employee + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "measure": {"name": "store_productivity", + "sql": "SUM({CUBE.ss_ext_sales_price}) / NULLIF(SUM({store.s_number_employees}), + 0)", "type": "number"}}' + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "views": {"tpcds_retail_model": {"name": "tpcds_retail_model", + "cubes": [{"join_path": "store_sales", "includes": "*"}, {"join_path": "store_sales.date_dim", + "includes": "*"}, {"join_path": "store_sales.customer", "includes": "*"}, {"join_path": + "store_sales.item", "includes": "*"}, {"join_path": "store_sales.store", "includes": + "*"}]}}, "mapped_view": "tpcds_retail_model"}' + - vendor_name: SALESFORCE + data: | + { + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "0.1.1" + } + } + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index 760fc22a..d5914c8a 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -26,7 +26,8 @@ import json import pytest -from _util import REPO_ROOT, load_fixture_dir, parse, parse_files +from _util import (REPO_ROOT, canon, load_fixture, load_fixture_dir, parse, + parse_files) from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube @@ -47,6 +48,32 @@ def test_cube_roundtrip_is_lossless(fixture): assert parse_files(files2) == parse_files(files) +@pytest.mark.parametrize("cube_dir,ossie_file", [ + ("fixtureA_cube", "fixtureA_ossie.yaml"), + ("tpcds_cube", "tpcds_ossie.yaml"), +]) +def test_import_matches_the_committed_ossie_fixture(cube_dir, ossie_file): + """Whole-document snapshot, so an unintended change anywhere in the output shows + up as a readable diff rather than slipping past field-level assertions. + + Regenerate with `ossie-cube import -i tests/fixtures/` when a change + to the output is intended. + """ + ossie, _ = convert_cube_to_ossie(load_fixture_dir(cube_dir)) + assert canon(parse(ossie)) == canon(parse(load_fixture(ossie_file))) + + +@pytest.mark.parametrize("cube_dir,ossie_file", [ + ("fixtureA_cube", "fixtureA_ossie.yaml"), + ("tpcds_cube", "tpcds_ossie.yaml"), +]) +def test_export_of_the_ossie_fixture_matches_the_cube_fixture(cube_dir, ossie_file): + """The same snapshot in the other direction: the committed Ossie fixture has to + export back to the committed Cube fixture.""" + files, _ = convert_ossie_to_cube(load_fixture(ossie_file)) + assert parse_files(files) == parse_files(load_fixture_dir(cube_dir)) + + @pytest.mark.parametrize("fixture", FIXTURES) def test_imported_ossie_validates_against_core_spec_schema(fixture): jsonschema = pytest.importorskip("jsonschema") @@ -74,7 +101,7 @@ def test_ossie_roundtrip_is_lossless(fixture): def test_hand_authored_ossie_gets_a_generated_view(): """A model with no stashed views is not from Cube, so export has to invent the view -- the model boundary Cube users work with.""" - ossie = _HAND_AUTHORED + ossie = load_fixture("hand_authored_ossie.yaml") files, _ = convert_ossie_to_cube(ossie) assert set(files) == { "model/cubes/orders.yml", "model/cubes/customers.yml", @@ -91,7 +118,7 @@ def test_hand_authored_ossie_gets_a_generated_view(): def test_hand_authored_ossie_survives_the_round_trip(): - files, _ = convert_ossie_to_cube(_HAND_AUTHORED) + files, _ = convert_ossie_to_cube(load_fixture("hand_authored_ossie.yaml")) ossie2, _ = convert_cube_to_ossie(files) model = parse(ossie2)["semantic_model"][0] assert model["name"] == "ecommerce" @@ -106,7 +133,7 @@ def test_hand_authored_ossie_survives_the_round_trip(): def test_ossie_only_constructs_are_parked_not_dropped(): """`unique_keys` and a foreign vendor's extensions have no Cube field, so they ride under `meta.ossie` and come back intact.""" - files, _ = convert_ossie_to_cube(_HAND_AUTHORED) + files, _ = convert_ossie_to_cube(load_fixture("hand_authored_ossie.yaml")) orders = parse(files["model/cubes/orders.yml"])["cubes"][0] parked = orders["meta"]["ossie"] assert parked["unique_keys"] == [["order_number"]] @@ -117,78 +144,3 @@ def test_ossie_only_constructs_are_parked_not_dropped(): assert ds["orders"]["unique_keys"] == [["order_number"]] vendors = {e["vendor_name"] for e in ds["orders"]["custom_extensions"]} assert "SNOWFLAKE" in vendors - - -_HAND_AUTHORED = """ -version: 0.2.0.dev0 -semantic_model: -- name: ecommerce - description: Orders and customers - ai_context: - instructions: Use for sales analysis. - synonyms: - - sales - - purchases - datasets: - - name: orders - source: sales.public.orders - primary_key: - - id - unique_keys: - - - order_number - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: id - datatype: Integer - - name: customer_id - expression: - dialects: - - dialect: ANSI_SQL - expression: customer_id - datatype: Integer - - name: ordered_at - expression: - dialects: - - dialect: ANSI_SQL - expression: ordered_at - datatype: Date - custom_extensions: - - vendor_name: SNOWFLAKE - data: '{"warehouse": "ANALYTICS_WH"}' - - name: customers - source: sales.public.customers - primary_key: - - id - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: id - datatype: Integer - - name: email - expression: - dialects: - - dialect: ANSI_SQL - expression: LOWER(email) - datatype: String - relationships: - - name: orders_to_customers - from: orders - to: customers - from_columns: - - customer_id - to_columns: - - id - metrics: - - name: total_revenue - expression: - dialects: - - dialect: ANSI_SQL - expression: SUM(orders.amount) - description: Total revenue - datatype: Decimal -""" From 5d7dc5f25c82da12e6aec4b52971e4af3c78f6d9 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 12:12:08 +0500 Subject: [PATCH 09/25] Resolve dimension names once; make the Hypothesis driver honour p MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both from the Copilot review on #289. Dimension names were sanitized separately in _convert_model (to decide which members a cube has) and again in _build_dimensions (to name them). The first pass used a fresh `taken` set per field, so a collision was silently swallowed by a set comprehension there and only rejected later in the second pass -- meaning the member set that decides `{CUBE.member}` vs `{CUBE}.column`, and where a measure lands, could be short a name while measures were being placed. Demonstrated: "Order Status" and "order status" collapsed to one name with no error. Now resolved once in _resolve_dimension_names and reused. That also fixes a defect the review did not mention: the old set included the two halves of a split geo dimension (location_latitude, location_longitude), which never exist as Cube dimensions since they merge back into `location`, so a metric referencing one would emit an unresolvable `{CUBE.location_…}`. The halves now resolve to the dimension they merge into. _HypothesisRnd.chance() ignored its `p` argument and always drew an unweighted boolean, so the Hypothesis driver explored a different distribution than the seeded one despite the docstring claiming they share a generator. Now weighted, and drawn so the minimal value means False -- shrinking toward the smallest model rather than the largest. 231 tests. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/osi_to_cube.py | 60 +++++++++++++++---- converters/cube/tests/test_osi_to_cube.py | 25 ++++++++ .../cube/tests/test_roundtrip_properties.py | 8 ++- 3 files changed, 80 insertions(+), 13 deletions(-) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 5d163d54..8fc1731d 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -131,14 +131,19 @@ def _convert_model(model, dialect, base_cube, issues): model_stash = read_stash(model) # Per-cube facts the join and measure stages need. + # Field -> dimension names are resolved once here and reused by every stage. + # Sanitizing per stage would let a collision go undetected in one place and be + # rejected in another, and would disagree about which members a cube actually + # has -- which decides `{CUBE.member}` vs `{CUBE}.column` and where a measure + # lands. + dim_names_by_cube = {} members_by_cube = {} pk_by_cube = {} for ds_name, ds in datasets.items(): cname = cube_names[ds_name] - members_by_cube[cname] = { - sanitize_name(f["name"], f"dataset '{ds_name}': field", set()) - for f in (ds.get("fields") or []) - } + dim_names_by_cube[cname] = _resolve_dimension_names( + ds, f"Model '{name}': dataset '{ds_name}'") + members_by_cube[cname] = set(dim_names_by_cube[cname].values()) pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] joins_by_cube = _build_joins(relationships, cube_names, issues) @@ -152,7 +157,7 @@ def _convert_model(model, dialect, base_cube, issues): files_content = {} for ds_name, ds in datasets.items(): cname = cube_names[ds_name] - cube = _build_cube(ds, cname, members_by_cube[cname], + cube = _build_cube(ds, cname, dim_names_by_cube[cname], joins_by_cube.get(cname), measures_by_cube.get(cname), dialect, issues) path = stashed_paths.get(cname) or cube_file(cname) @@ -233,7 +238,7 @@ def _ordered(obj, order): # --- cubes ---------------------------------------------------------------------- -def _build_cube(ds, cname, members, joins, measures, dialect, issues): +def _build_cube(ds, cname, dim_names, joins, measures, dialect, issues): ds_name = ds["name"] scope = f"dataset '{ds_name}'" stash = read_stash(ds) @@ -262,7 +267,8 @@ def _build_cube(ds, cname, members, joins, measures, dialect, issues): "Cube's agent reads ai_context only on views and members, " "so this cube-level value has no effect in Cube") - dimensions, covered = _build_dimensions(ds, cname, members, dialect, issues) + dimensions, covered = _build_dimensions( + ds, cname, dim_names, dialect, issues) # A primary-key column no field covers still has to exist as a dimension for # Cube to join or roll up the cube. pk_names = [] @@ -301,17 +307,47 @@ def _build_cube(ds, cname, members, joins, measures, dialect, issues): return _ordered(cube, _CUBE_KEY_ORDER) -def _build_dimensions(ds, cname, members, dialect, issues): +def _resolve_dimension_names(ds, scope): + """Map each of a dataset's fields to the Cube dimension name it becomes. + + Sanitization and collision detection happen here and nowhere else, so every + stage agrees on the result. Two subtleties the mapping has to get right: + + - A collision is an error, not a silent merge. Sanitizing with a fresh `taken` + set per field would hide one. + - The two halves of a split `geo` dimension map back to the *single* dimension + they merge into, so `location_latitude` resolves to `location`. Treating the + halves as members of their own would let a metric emit a `{CUBE.…}` reference + to a dimension the exported cube does not have. + """ + names = {} + taken = set() + for field in (ds.get("fields") or []): + fname = require_str(field, "name", f"{scope}: field") + geo = read_stash(field).get("geo") + if geo: + base = geo["of"] + names[fname] = base + taken.add(base.lower()) + continue + dname = sanitize_name(fname, f"{scope}: field", taken) + taken.add(dname.lower()) + names[fname] = dname + return names + + +def _build_dimensions(ds, cname, dim_names, dialect, issues): """Build a cube's dimensions from an Ossie dataset's fields. Returns (dimensions, {column or field name: dimension name}) -- the second value is what primary-key resolution matches against. Fields carrying a `geo` stash are re-merged into the single Cube dimension they were split from. + Dimension names come from `dim_names` (see `_resolve_dimension_names`) rather + than being sanitized again here. """ ds_name = ds["name"] dimensions = [] covered = {} - taken = set() geo_parts = {} for field in (ds.get("fields") or []): fname = require_str(field, "name", f"dataset '{ds_name}': field") @@ -326,8 +362,7 @@ def _build_dimensions(ds, cname, members, dialect, issues): dimensions.append(None) # placeholder, filled in below continue - dname = sanitize_name(fname, f"dataset '{ds_name}': field", taken) - taken.add(dname.lower()) + dname = dim_names[fname] expr = pick_expression(field.get("expression"), dialect) if expr is None: issues.add(IssueType.NO_USABLE_DIALECT, f"{ds_name}.{fname}", @@ -339,7 +374,8 @@ def _build_dimensions(ds, cname, members, dialect, issues): # The exact Cube spelling a prior import saw. dim["sql"] = stash["sql"] else: - dim["sql"] = ossie_expr_to_cube_sql(expr, cname, members, ()) + dim["sql"] = ossie_expr_to_cube_sql( + expr, cname, set(dim_names.values()), ()) dim["type"] = _dimension_type(field, stash, f"{ds_name}.{fname}", issues) if field.get("label"): dim["title"] = field["label"] diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 87099837..331ce2ff 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -218,6 +218,31 @@ def test_field_name_is_sanitized_and_collisions_are_rejected(): convert_ossie_to_cube(_ossie(ds)) +def test_field_collision_is_rejected_before_any_metric_is_placed(): + """Dimension names are resolved once, up front. Resolving them per stage let a + collision go undetected while measures were being placed -- so the member set + that decides `{CUBE.member}` vs `{CUBE}.column` could be silently short a name, + and the error surfaced later and less clearly.""" + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: Order Status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status\n" + " - name: order status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status2\n" + ) + metrics = _metric("m", "SUM(orders.amount)") + with pytest.raises(ConversionError, match="collides"): + convert_ossie_to_cube(_ossie(ds, metrics=metrics)) + + def test_missing_dialect_drops_the_field_with_an_issue(): ds = ( " - name: orders\n" diff --git a/converters/cube/tests/test_roundtrip_properties.py b/converters/cube/tests/test_roundtrip_properties.py index 0a7e4106..3137245f 100644 --- a/converters/cube/tests/test_roundtrip_properties.py +++ b/converters/cube/tests/test_roundtrip_properties.py @@ -52,7 +52,13 @@ def __init__(self, data): self.data = data def chance(self, p=0.5): - return self.data.draw(st.booleans()) + # `st.booleans()` is unweighted, so it would ignore `p` and explore a + # different distribution than RandomRnd -- defeating the point of the + # two drivers sharing one generator. Drawn so the minimal value (0) + # means False, which shrinks toward the smallest model rather than the + # largest. + return self.data.draw( + st.integers(min_value=0, max_value=99)) >= 100 - round(p * 100) def count(self, lo, hi): return self.data.draw(st.integers(min_value=lo, max_value=hi)) From d676e76658acf022e9e71e33e1aff78f1e2c9319 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 12:26:05 +0500 Subject: [PATCH 10/25] Inline a split geo dimension's SQL where its halves are referenced A Cube `type: geo` dimension holds two SQL expressions where an Ossie field holds one, so import splits it into `_latitude` / `_longitude`. Export merges them back, so the round trip was already exact. But those half names exist only in Ossie. Cube has neither a column nor a member called `home_latitude` -- the halves merge into `home` -- so a metric or field expression referencing one had nothing valid to emit: AVG(users.home_latitude) -> sql: '{CUBE}.home_latitude' which names a column that does not exist (the column is `lat`). The member-reference form would have been just as wrong, failing at Cube compile time instead of in the database. The half's real SQL is already in the stash, so a reference to one is now replaced by that SQL: AVG(users.home_latitude) -> AVG({CUBE}.lat) AVG(users.home_latitude) - MIN(orders.amt) -> AVG({users}.lat) - MIN({CUBE.amt}) `{CUBE}` means "the cube this is declared on", so an inlined snippet is requalified to name its original cube when it crosses into another cube's SQL -- otherwise it would silently rebind to the wrong cube. One normalization follows and is documented: after a round trip such a metric names the column the half actually reads (`users.lat`) rather than the Ossie-only field name. Same reference, and the only form Cube can express. 234 tests. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 29 ++++++- converters/cube/src/ossie_cube/_common.py | 31 ++++++- converters/cube/src/ossie_cube/osi_to_cube.py | 54 +++++++----- converters/cube/tests/test_edge_cases.py | 86 +++++++++++++++++++ 4 files changed, 175 insertions(+), 25 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 118f2884..94572ddb 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -129,7 +129,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. | | `field.label` / `description` | dimension `title` / `description` | | | `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | -| — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. | +| — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. See [Geo dimensions](#geo-dimensions). | | relationship | `joins[]` on a cube | `many_to_one` on cube A -> `from: A`(many), `to: B`(one). `one_to_many` is flipped so Ossie's `from` is the many side; the declared side and type are stashed so export restores the original. | | `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities between two member references maps. Anything else (non-equi, range, literal, third cube) is preserved verbatim in the stash. | | metric | `measures[]` on the cube its expression references | Import hoists cube-scoped measures to the model level, qualifying a colliding name as `__` and stashing the original name and owning cube. | @@ -195,6 +195,31 @@ different number. > `non_additive_dimension` is the nearest precedent, and this repo's dbt converter > already loses the same information. Worth raising on `dev@`. +## Geo dimensions + +A Cube `type: geo` dimension carries two SQL expressions where an Ossie field carries one, so it splits on import: + +```yaml +# Cube # Ossie +- name: home - name: home_latitude (expression: lat) + type: geo - name: home_longitude (expression: lon) + latitude: { sql: "{CUBE}.lat" } + longitude: { sql: "{CUBE}.lon" } +``` + +Export merges the halves back into the single geo dimension, so the round trip is exact. + +The half names exist **only in Ossie** — Cube has neither a column nor a member called `home_latitude`. So when an Ossie metric or field expression references a half, export substitutes the half's own SQL rather than emitting a reference Cube cannot resolve: + +``` +AVG(users.home_latitude) -> sql: AVG({CUBE}.lat) +AVG(users.home_latitude) - MIN(orders.amt) -> sql: AVG({users}.lat) - MIN({CUBE.amt}) +``` + +`{CUBE}` means "the cube this is declared on", so an inlined snippet is requalified to name its original cube when it crosses into another cube's SQL. + +One documented normalization follows: after a round trip such a metric names the column the half actually reads (`users.lat`) rather than the Ossie-only field name (`users.home_latitude`). Same reference, and it is the form Cube can express. + ## Conversion issues `convert_cube_to_ossie` returns `(yaml, IssueLog)`. Each issue carries a type, the @@ -250,7 +275,7 @@ uv sync uv run pytest ``` -226 tests at 96% line coverage: example-based unit tests per direction, CLI +234 tests at 96% line coverage: example-based unit tests per direction, CLI behavior tests, fixture round-trip tests (including the [TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks for as a baseline), core-spec JSON Schema validation of every emitted Ossie diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 806d0a4d..5cb4ab28 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,7 +385,22 @@ def repl(m): return out, changed -def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=()): +def requalify_self_refs(sql, cube_name): + """Rewrite `{CUBE}` / `{TABLE}` in a Cube SQL snippet to name `cube_name`. + + Needed when a snippet written for one cube is inlined into another cube's SQL: + `{CUBE}` means "the cube this is declared on", so it changes meaning on the + move, while `{orders}.col` is explicit and does not. + """ + return re.sub( + r"\$?\{\s*(?:CUBE|TABLE)\s*(\.\s*[A-Za-z_][A-Za-z0-9_]*\s*)?\}", + lambda m: "{" + cube_name + (m.group(1).strip() if m.group(1) else "") + "}", + str(sql), + ) + + +def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), + inline_sql=None): """Rewrite an Ossie expression into Cube member-reference form. Only *dotted* `cube.name` references are rewritten -- a bare identifier stays @@ -403,13 +418,27 @@ def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=()): The own cube is always referenced as `{CUBE}` rather than by name, so the model keeps working when the cube is extended. Literal braces in the incoming expression are escaped. + + `inline_sql` maps `{cube: {field: cube_sql}}` for Ossie fields that have no + addressable Cube counterpart, and whose SQL therefore has to be substituted + inline. The case that needs it is a split `geo` dimension: `location_latitude` + exists only in Ossie -- Cube has neither a column nor a member by that name -- + so a reference to it becomes the half's own SQL (`{CUBE}.lat`), requalified when + it crosses cubes. """ escaped = str(expr).replace("{", "\\{").replace("}", "\\}") known = set(cube_names) members = set(own_members) + inline = inline_sql or {} def repl(m): head, name = m.group(1), m.group(2) + substitute = (inline.get(head) or {}).get(name) + if substitute is not None: + # Already-Cube SQL, so it bypasses the escaping above; `{CUBE}` inside + # it means `head`, which only stays true while head is the own cube. + return (str(substitute) if head == own_cube + else requalify_self_refs(substitute, head)) if head == own_cube: return "{CUBE." + name + "}" if name in members else "{CUBE}." + name if head in known: diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 8fc1731d..b87995a3 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -138,18 +138,19 @@ def _convert_model(model, dialect, base_cube, issues): # lands. dim_names_by_cube = {} members_by_cube = {} + inline_sql_by_cube = {} pk_by_cube = {} for ds_name, ds in datasets.items(): cname = cube_names[ds_name] - dim_names_by_cube[cname] = _resolve_dimension_names( - ds, f"Model '{name}': dataset '{ds_name}'") + dim_names_by_cube[cname], inline_sql_by_cube[cname] = ( + _resolve_dimension_names(ds, f"Model '{name}': dataset '{ds_name}'")) members_by_cube[cname] = set(dim_names_by_cube[cname].values()) pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] joins_by_cube = _build_joins(relationships, cube_names, issues) measures_by_cube = _build_measures( - model, cube_names, members_by_cube, pk_by_cube, datasets, relationships, - base_cube, dialect, issues) + model, cube_names, members_by_cube, inline_sql_by_cube, pk_by_cube, + datasets, relationships, base_cube, dialect, issues) # Cubes, grouped by the file they belong in: several datasets can share one # stashed original path, in which case they go back into the same file. @@ -158,8 +159,8 @@ def _convert_model(model, dialect, base_cube, issues): for ds_name, ds in datasets.items(): cname = cube_names[ds_name] cube = _build_cube(ds, cname, dim_names_by_cube[cname], - joins_by_cube.get(cname), measures_by_cube.get(cname), - dialect, issues) + inline_sql_by_cube[cname], joins_by_cube.get(cname), + measures_by_cube.get(cname), dialect, issues) path = stashed_paths.get(cname) or cube_file(cname) files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) @@ -238,7 +239,8 @@ def _ordered(obj, order): # --- cubes ---------------------------------------------------------------------- -def _build_cube(ds, cname, dim_names, joins, measures, dialect, issues): +def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, + issues): ds_name = ds["name"] scope = f"dataset '{ds_name}'" stash = read_stash(ds) @@ -268,7 +270,7 @@ def _build_cube(ds, cname, dim_names, joins, measures, dialect, issues): "so this cube-level value has no effect in Cube") dimensions, covered = _build_dimensions( - ds, cname, dim_names, dialect, issues) + ds, cname, dim_names, inline_sql, dialect, issues) # A primary-key column no field covers still has to exist as a dimension for # Cube to join or roll up the cube. pk_names = [] @@ -316,11 +318,14 @@ def _resolve_dimension_names(ds, scope): - A collision is an error, not a silent merge. Sanitizing with a fresh `taken` set per field would hide one. - The two halves of a split `geo` dimension map back to the *single* dimension - they merge into, so `location_latitude` resolves to `location`. Treating the - halves as members of their own would let a metric emit a `{CUBE.…}` reference - to a dimension the exported cube does not have. + they merge into, so `location_latitude` resolves to `location`. + + Returns (names, inline_sql). `inline_sql` holds the fields whose name exists + only in Ossie -- the two halves of a split geo dimension -- mapped to the Cube + SQL a reference to them must be replaced by, since Cube has neither a column nor + a member of that name. """ - names = {} + names, inline_sql = {}, {} taken = set() for field in (ds.get("fields") or []): fname = require_str(field, "name", f"{scope}: field") @@ -328,15 +333,16 @@ def _resolve_dimension_names(ds, scope): if geo: base = geo["of"] names[fname] = base + inline_sql[fname] = geo["sql"] taken.add(base.lower()) continue dname = sanitize_name(fname, f"{scope}: field", taken) taken.add(dname.lower()) names[fname] = dname - return names + return names, inline_sql -def _build_dimensions(ds, cname, dim_names, dialect, issues): +def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): """Build a cube's dimensions from an Ossie dataset's fields. Returns (dimensions, {column or field name: dimension name}) -- the second @@ -375,7 +381,8 @@ def _build_dimensions(ds, cname, dim_names, dialect, issues): dim["sql"] = stash["sql"] else: dim["sql"] = ossie_expr_to_cube_sql( - expr, cname, set(dim_names.values()), ()) + expr, cname, set(dim_names.values()), (), + inline_sql={cname: inline_sql}) dim["type"] = _dimension_type(field, stash, f"{ds_name}.{fname}", issues) if field.get("label"): dim["title"] = field["label"] @@ -501,8 +508,9 @@ def _build_joins(relationships, cube_names, issues): # --- measures ------------------------------------------------------------------- -def _build_measures(model, cube_names, members_by_cube, pk_by_cube, datasets, - relationships, base_cube, dialect, issues): +def _build_measures(model, cube_names, members_by_cube, inline_sql_by_cube, + pk_by_cube, datasets, relationships, base_cube, dialect, + issues): """Group Ossie metrics into per-cube `measures` lists.""" name = model.get("name", "") sanitized = set(cube_names.values()) @@ -548,7 +556,8 @@ def resolve_base(): next(iter(referenced)) if len(referenced) == 1 else resolve_base()) measure = _measure_from_expression( expr, target, mname, stash, members_by_cube.get(target, set()), - pk_by_cube.get(target, []), sanitized, scope, issues) + inline_sql_by_cube, pk_by_cube.get(target, []), sanitized, scope, + issues) _apply_measure_metadata(metric, measure, stash) _place(measures_by_cube, target, measure, name) return measures_by_cube @@ -563,8 +572,8 @@ def _place(measures_by_cube, target, measure, model_name): bucket.append(measure) -def _measure_from_expression(expr, target, mname, stash, members, primary_key, - sanitized, scope, issues): +def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_cube, + primary_key, sanitized, scope, issues): """Turn an Ossie metric expression back into a structured Cube measure. `COUNT(DISTINCT )` is Cube's bare `type: count` -- @@ -590,14 +599,15 @@ def _measure_from_expression(expr, target, mname, stash, members, primary_key, agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) if agg is not None: measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( - inner, target, members, sanitized) + inner, target, members, sanitized, + inline_sql=inline_sql_by_cube) measure["type"] = agg return measure # A ratio, a window expression, or a multi-dataset aggregate: Cube expresses # these as a calculated measure whose sql carries the aggregation. measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( - expr, target, members, sanitized) + expr, target, members, sanitized, inline_sql=inline_sql_by_cube) measure["type"] = "number" if len({ ref for ref in re.findall( diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 22b650dd..d4c9007d 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -603,6 +603,92 @@ def test_geo_dimension_extras_survive_the_split_and_merge(): assert issues.of_type(IssueType.GEO_DIMENSION_SPLIT) +_GEO_MODEL = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: users\n" + " source: public.users\n" + " fields:\n" + " - name: home_latitude\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: lat\n" + " datatype: Float\n" + " custom_extensions:\n" + " - vendor_name: CUBE\n" + " data: '{\"_v\": 1, \"geo\": {\"of\": \"home\", \"part\": \"latitude\"," + " \"sql\": \"{CUBE}.lat\"}}'\n" + " - name: home_longitude\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: lon\n" + " datatype: Float\n" + " custom_extensions:\n" + " - vendor_name: CUBE\n" + " data: '{\"_v\": 1, \"geo\": {\"of\": \"home\", \"part\": \"longitude\"," + " \"sql\": \"{CUBE}.lon\"}}'\n" + " metrics:\n" + " - name: avg_lat\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: AVG(users.home_latitude)\n" +) + + +def test_a_metric_referencing_a_geo_half_inlines_its_sql(): + """A split geo half's name exists only in Ossie: Cube has neither a column nor a + member called `home_latitude`, since the halves merge into the `home` dimension. + So a reference to one is replaced by the half's own SQL, which is valid Cube.""" + files, _ = convert_ossie_to_cube(_GEO_MODEL) + cube = parse(files["model/cubes/users.yml"])["cubes"][0] + assert cube["measures"] == [ + {"name": "avg_lat", "sql": "{CUBE}.lat", "type": "avg"}] + # And the dimension itself still merges back to a single geo member. + assert cube["dimensions"] == [{ + "name": "home", "type": "geo", + "latitude": {"sql": "{CUBE}.lat"}, + "longitude": {"sql": "{CUBE}.lon"}}] + + +def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): + """`{CUBE}` means "the cube this is declared on", so inlining a snippet into + another cube's SQL has to name the original cube explicitly.""" + model = _GEO_MODEL.replace( + " - name: users\n", " - name: orders\n source: public.orders\n" + " fields:\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" + " - name: users\n", 1 + ).replace(" expression: AVG(users.home_latitude)\n", + " expression: AVG(users.home_latitude) - MIN(orders.amount)\n") + files, _ = convert_ossie_to_cube(model, base_cube="orders") + measures = parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"] + # `{users}.lat` names the cube explicitly, since `{CUBE}` here would mean + # `orders`. `{CUBE.amount}` stays a member reference because `amount` is a + # declared field of the cube the measure lands on. + assert measures[0]["sql"] == "AVG({users}.lat) - MIN({CUBE.amount})" + assert measures[0]["type"] == "number" + + +def test_geo_half_references_normalize_to_the_underlying_column(): + """Documented normalization: after a round trip the metric names the column the + geo half actually reads rather than the Ossie-only field name. Semantically the + same reference, and it is what Cube can express.""" + files, _ = convert_ossie_to_cube(_GEO_MODEL) + ossie2, _ = convert_cube_to_ossie(files) + metric = model_of(ossie2)["metrics"][0] + assert expr_of(metric) == "AVG(users.lat)" + + def test_geo_dimension_missing_a_half_is_rejected(): files = _files(users=( "cubes:\n" From 210f6da6da7a7bf26e97b27359a720273954b9db Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 12:46:02 +0500 Subject: [PATCH 11/25] Use a deque for the generated view's BFS; fix a licence/license typo Both from the Copilot review on #289. The BFS popped from the front of a list, which is O(n) per pop; a deque makes it O(1). Semantic models are small enough that this was never going to matter in practice, but the deque is also the more idiomatic form. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/osi_to_cube.py | 5 +++-- converters/cube/tests/test_roundtrip.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index b87995a3..69d700c9 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -32,6 +32,7 @@ """ import re +from collections import deque from ._common import ( DATATYPE_TO_DIM_TYPE, @@ -710,9 +711,9 @@ def _view_cubes(cube_names, relationships, base): entries = [{"join_path": base, "includes": "*"}] paths = {base: base} - queue = [base] + queue = deque([base]) while queue: - current = queue.pop(0) + current = queue.popleft() for neighbor in adjacency.get(current, []): if neighbor in paths: continue diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py index d5914c8a..8416088c 100644 --- a/converters/cube/tests/test_roundtrip.py +++ b/converters/cube/tests/test_roundtrip.py @@ -39,7 +39,7 @@ def test_cube_roundtrip_is_lossless(fixture): """Cube -> Ossie -> Cube reproduces the original model, structurally. Compared parsed rather than byte-for-byte: YAML comments (including the - licence headers on the fixtures) are not part of the data model, and key order + license headers on the fixtures) are not part of the data model, and key order within a mapping is not semantic. """ files = load_fixture_dir(fixture) From d9c853d21988cdabc6180b3098fbd38c4a4d208a Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 14:32:50 +0500 Subject: [PATCH 12/25] Refuse rather than drop unparkable extensions; separate drops from parks Both from the Copilot review on #289. Model-level foreign-vendor custom_extensions ride on the view that represents the model. When the source Cube model had several views and none was chosen, there is no such view -- and export silently dropped them. Reachable in practice: import a multi-view Cube model, add a SNOWFLAKE extension to the Ossie model, export, and it is gone. Confirmed by reproducing it. Now refused, with the fix in the message (re-import with `--view`). Parking on an arbitrary view was considered and rejected: only the mapped view's parked extensions are read back on import, so it would look lossless while still losing them. The review also noted the issue type contradicted its own message -- PARKED_IN_META for something reported as "dropped". That was true in two places, not one, and it matters: the README defines PARKED_IN_META as preserved-but-invisible-to-Cube, so a pipeline gating on issue types would have concluded the data survived. Adds DROPPED_NO_CUBE_EQUIVALENT for values that genuinely cannot be preserved, and uses it for relationship ai_context -- a Cube join entry takes only name/sql/relationship, with no `meta` field, making it the one construct with nowhere to go. Also drops the hard-coded test count from the README. It had already drifted out of sync with the PR description, which is the reviewer's point: the number carries no information a reader needs, while the description of what the suite covers does. 236 tests, 97% line coverage. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 9 ++++-- .../cube/src/ossie_cube/converter_issues.py | 9 +++++- converters/cube/src/ossie_cube/osi_to_cube.py | 30 ++++++++++++++----- converters/cube/tests/test_edge_cases.py | 30 +++++++++++++++++++ converters/cube/tests/test_osi_to_cube.py | 12 ++++++-- 5 files changed, 76 insertions(+), 14 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 94572ddb..6641a1c5 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -233,7 +233,8 @@ element it concerns, and a detail string. | `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | | `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | | `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | -| `PARKED_IN_META` | An element preserved in the stash with no native mapping | +| `PARKED_IN_META` | An element with no native mapping, preserved in the stash or under `meta.ossie` — invisible to Cube but intact through a round trip | +| `DROPPED_NO_CUBE_EQUIVALENT` | A value Cube has nowhere to hold *and* that cannot be parked, so it is genuinely gone. Currently only relationship `ai_context`, since a Cube join entry has no `meta` field. Kept distinct from `PARKED_IN_META` so a caller can tell real loss from "preserved but unreadable by Cube" | ## Requirements @@ -249,6 +250,10 @@ invalid) when an input breaks one of these: - two cubes, two views, or two derived metric names collide; - a dimension has an unknown `type`, or a `geo` dimension is missing `latitude.sql` / `longitude.sql`; +- the model carries foreign-vendor `custom_extensions` but no view is mapped, so + there is nowhere to park them (re-import with `--view `); model-level + metadata rides on the view representing the model, and picking one arbitrarily + would not survive a re-import; - there are no convertible cubes at all; the input YAML is malformed. ## Notes and limitations @@ -275,7 +280,7 @@ uv sync uv run pytest ``` -234 tests at 96% line coverage: example-based unit tests per direction, CLI +Example-based unit tests per direction, CLI behavior tests, fixture round-trip tests (including the [TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks for as a baseline), core-spec JSON Schema validation of every emitted Ossie diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index de29c586..0c79610d 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -62,9 +62,16 @@ class IssueType(Enum): # An Ossie field or metric with no usable expression dialect (export). NO_USABLE_DIALECT = "NO_USABLE_DIALECT" - # An Ossie construct Cube has no slot for, parked under `meta.ossie`. + # An Ossie construct Cube has no slot for, parked under `meta.ossie` -- so the + # value survives the round trip even though Cube itself cannot read it. PARKED_IN_META = "PARKED_IN_META" + # A value Cube has nowhere to hold *and* that cannot be parked, so it is gone + # from the output. Distinct from PARKED_IN_META on purpose: a caller gating on + # issue types has to be able to tell "preserved but invisible to Cube" from + # "actually lost". + DROPPED_NO_CUBE_EQUIVALENT = "DROPPED_NO_CUBE_EQUIVALENT" + @dataclass(frozen=True) class ConverterIssue: diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 69d700c9..16bed836 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -166,7 +166,7 @@ def _convert_model(model, dialect, base_cube, issues): files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) for vpath, view in _build_views(model, model_stash, cube_names, relationships, - datasets, base_cube, issues).items(): + datasets, base_cube).items(): files_content.setdefault(vpath, {}).setdefault("views", []).append(view) files = {path: dump_yaml(content) for path, content in files_content.items()} @@ -499,9 +499,12 @@ def _build_joins(relationships, cube_names, issues): if key not in ("declared_on", "relationship", "sql"): join[key] = value if rel.get("ai_context"): - issues.add(IssueType.PARKED_IN_META, f"relationship '{rname}'", - "Cube joins carry no metadata, so relationship ai_context " - "has no home; dropped") + # A Cube join entry takes only name/sql/relationship -- no `meta` -- so + # unlike every other level there is nowhere to park this. + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, + f"relationship '{rname}'", + "a Cube join carries no metadata field, so relationship " + "ai_context has nowhere to go and is dropped") joins_by_cube.setdefault(own, []).append( _ordered(join, ["name", "sql", "relationship"])) return joins_by_cube @@ -651,7 +654,7 @@ def _balanced(s): # --- views ---------------------------------------------------------------------- def _build_views(model, model_stash, cube_names, relationships, datasets, - base_cube, issues): + base_cube): """Return {file path: view dict}. Stashed views restore verbatim, with the natively mapped description and AI @@ -671,9 +674,20 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, mapped = model_stash.get("mapped_view") paths = model_stash.get("view_files") or {} if foreign and mapped is None: - issues.add(IssueType.PARKED_IN_META, "model", - "no mapped view to park foreign-vendor custom_extensions on; " - "they have no Cube home and are dropped") + # The model's own metadata rides on the view that represents it, and + # there isn't one: the source Cube model had several views and none was + # chosen. Dropping the extensions would be silent data loss, and + # picking a view arbitrarily would not survive a re-import (only the + # mapped view's parked extensions are restored). So this is refused + # with the fix in the message. + vendors = ", ".join( + sorted({str(e.get("vendor_name")) for e in foreign})) + raise ConversionError( + f"Model carries custom_extensions for {vendors}, which have no Cube " + f"field and ride on the view representing the model -- but no view " + f"is mapped, so there is nowhere to put them without losing them. " + f"Re-import naming the view the model maps to (`--view `), or " + f"remove the foreign-vendor extensions.") for vname, view in (model_stash["views"] or {}).items(): view = dict(view) if vname == mapped: diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index d4c9007d..009454f4 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -34,6 +34,7 @@ convert_cube_to_ossie, convert_ossie_to_cube, ) +from ossie_cube._common import dump_yaml def _files(**named): @@ -772,6 +773,35 @@ def test_choosing_a_view_maps_its_metadata_onto_the_model(): assert set(stash_of(model)["views"]) == {"a", "b"} +def test_foreign_extensions_with_no_mapped_view_are_refused_not_dropped(): + """Model-level foreign-vendor extensions ride on the view that represents the + model. With several views and none mapped there is no such view, and picking one + arbitrarily would not survive a re-import -- only the mapped view's parked + extensions are read back. So this is refused rather than silently losing them.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS) + doc = parse(ossie) + doc["semantic_model"][0].setdefault("custom_extensions", []).append( + {"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "ANALYTICS_WH"}'}) + with pytest.raises(ConversionError, match="SNOWFLAKE"): + convert_ossie_to_cube(dump_yaml(doc)) + + +def test_foreign_extensions_survive_once_a_view_is_mapped(): + """The fix the error message points at: choose the view the model maps to, and + the extensions have a home again.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") + doc = parse(ossie) + doc["semantic_model"][0].setdefault("custom_extensions", []).append( + {"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "ANALYTICS_WH"}'}) + files, _ = convert_ossie_to_cube(dump_yaml(doc)) + parked = parse(files["model/views/b.yml"])["views"][0]["meta"]["ossie"] + assert parked["custom_extensions"][0]["vendor_name"] == "SNOWFLAKE" + # And they come back as Ossie extensions, not just stashed text. + ossie2, _ = convert_cube_to_ossie(files, view="b") + vendors = {e["vendor_name"] for e in model_of(ossie2)["custom_extensions"]} + assert "SNOWFLAKE" in vendors + + def test_both_views_are_restored_on_export(): ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") back, _ = convert_ossie_to_cube(ossie) diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 331ce2ff..6372b5bf 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -319,11 +319,17 @@ def test_composite_relationship_becomes_an_and_chain(): "{CUBE}.user_id = {users.id} AND {CUBE}.region = {users.region}") -def test_relationship_ai_context_has_no_cube_home(): +def test_relationship_ai_context_is_reported_as_dropped_not_parked(): + """A Cube join entry takes only name/sql/relationship -- no `meta` -- so this is + one of the few things that genuinely cannot be preserved. It is reported under + DROPPED_NO_CUBE_EQUIVALENT rather than PARKED_IN_META, so a caller gating on + issue types can tell real loss from "preserved but invisible to Cube".""" rel = _REL + " ai_context:\n instructions: Join carefully.\n" _, issues = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) - assert any("ai_context" in i.detail for i in issues.of_type( - IssueType.PARKED_IN_META)) + dropped = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT) + assert [i.element_name for i in dropped] == ["relationship 'orders_to_users'"] + assert "ai_context" in dropped[0].detail + assert not issues.of_type(IssueType.PARKED_IN_META) # --- metrics -------------------------------------------------------------------- From 0499913c99cb16ade3ca4b86602777ecdc25ebb0 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 15:31:17 +0500 Subject: [PATCH 13/25] Assemble geo dimensions by name, not by a mid-loop list index From the Copilot review on #289, which found that the placeholder holding a geo dimension's position was only reserved when the half encountered first happened to be `latitude`. With `longitude` first, the recorded index pointed at whatever real dimension had already been appended, and `dimensions[index] = dim` overwrote it. Reproduced: a `city` dimension between the two halves disappeared from the output entirely. Rather than reserve the placeholder earlier, the index arithmetic is gone. Dimensions are now built into a dict keyed by target name, with order taken from each name's first appearance -- which is well defined however the two halves are arranged, adjacent or not, in either order. Probing around the fix turned up two more silent-corruption paths in the same code, both order-dependent: - A geo base colliding with an ordinary field of the same name emitted two dimensions called `home` (invalid Cube) when the ordinary field came first, but was correctly rejected when it came second. Now checked during name resolution, so order does not decide. - Two fields both claiming the same half silently discarded one. Now rejected. Also validates the geo `part` and `of` values, and moves the missing-half check into name resolution so every geo problem is caught in one place before anything is built. 241 tests, 97% line coverage. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/osi_to_cube.py | 69 +++++++++++---- converters/cube/tests/test_edge_cases.py | 84 +++++++++++++++++++ 2 files changed, 135 insertions(+), 18 deletions(-) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 16bed836..70f0ed1d 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -328,18 +328,46 @@ def _resolve_dimension_names(ds, scope): """ names, inline_sql = {}, {} taken = set() + geo_halves = {} # base -> {part: field name}, for validating the pair for field in (ds.get("fields") or []): fname = require_str(field, "name", f"{scope}: field") geo = read_stash(field).get("geo") if geo: - base = geo["of"] + base, part = geo.get("of"), geo.get("part") + if part not in ("latitude", "longitude"): + raise ConversionError( + f"{scope}: field '{fname}' has a geo part '{part}'; expected " + f"'latitude' or 'longitude'") + if not base: + raise ConversionError( + f"{scope}: field '{fname}' has a geo stash with no 'of'") + seen = geo_halves.setdefault(base, {}) + if part in seen: + raise ConversionError( + f"{scope}: fields '{seen[part]}' and '{fname}' both claim the " + f"{part} of geo dimension '{base}'") + if not seen and base.lower() in taken: + # The base is the name of the merged Cube dimension, so it cannot + # also be an ordinary dimension -- that would emit two members of + # the same name. Order must not decide whether this is caught, so + # it is checked here rather than left to sanitize_name. + raise ConversionError( + f"{scope}: geo dimension '{base}' collides with another field " + f"of that name; rename one in the Ossie model.") + seen[part] = fname + taken.add(base.lower()) names[fname] = base inline_sql[fname] = geo["sql"] - taken.add(base.lower()) continue dname = sanitize_name(fname, f"{scope}: field", taken) taken.add(dname.lower()) names[fname] = dname + for base, seen in geo_halves.items(): + missing = {"latitude", "longitude"} - set(seen) + if missing: + raise ConversionError( + f"{scope}: geo dimension '{base}' is missing its " + f"{' and '.join(sorted(missing))} half") return names, inline_sql @@ -353,23 +381,27 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): than being sanitized again here. """ ds_name = ds["name"] - dimensions = [] covered = {} - geo_parts = {} + # Built by target dimension name rather than by list position: a geo dimension + # is assembled from two fields that may appear in either order and need not be + # adjacent, so an insertion index computed mid-loop is not a safe way to hold + # its place. `order` records first appearance of each target name, which is + # well defined however the halves are arranged. + order, built, geo_parts = [], {}, {} for field in (ds.get("fields") or []): fname = require_str(field, "name", f"dataset '{ds_name}': field") stash = read_stash(field) + dname = dim_names[fname] + if dname not in order: + order.append(dname) if "geo" in stash: geo = stash["geo"] - slot = geo_parts.setdefault(geo["of"], {"index": len(dimensions)}) + slot = geo_parts.setdefault(dname, {}) slot[geo["part"]] = geo["sql"] if "host" in geo: slot["host"] = geo["host"] - if geo["part"] == "latitude": - dimensions.append(None) # placeholder, filled in below continue - dname = dim_names[fname] expr = pick_expression(field.get("expression"), dialect) if expr is None: issues.add(IssueType.NO_USABLE_DIALECT, f"{ds_name}.{fname}", @@ -400,24 +432,25 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): for key, value in extras.items(): dim[key] = value - dimensions.append(dim) + built[dname] = dim covered[dname] = dname if is_simple_identifier(expr): covered[expr.strip()] = dname - for of, slot in geo_parts.items(): - if "latitude" not in slot or "longitude" not in slot: - raise ConversionError( - f"dataset '{ds_name}': geo dimension '{of}' is missing its " - f"{'longitude' if 'latitude' in slot else 'latitude'} half") - dim = {"name": of, "type": "geo", + # Both halves are guaranteed present by _resolve_dimension_names, which + # validates the pair before anything is built. + for base, slot in geo_parts.items(): + dim = {"name": base, "type": "geo", "latitude": {"sql": slot["latitude"]}, "longitude": {"sql": slot["longitude"]}} for key, value in (slot.get("host") or {}).items(): dim[key] = value - dimensions[slot["index"]] = dim - covered[of] = of - return [d for d in dimensions if d is not None], covered + built[base] = dim + covered[base] = base + + # A name in `order` with nothing built is a field dropped for want of a usable + # dialect; it simply does not appear. + return [built[n] for n in order if n in built], covered def _dimension_type(field, stash, scope, issues): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 009454f4..4b82a07b 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -690,6 +690,90 @@ def test_geo_half_references_normalize_to_the_underlying_column(): assert expr_of(metric) == "AVG(users.lat)" +def _geo_stash(part, of="home"): + return ('{"_v": 1, "geo": {"of": "' + of + '", "part": "' + part + + '", "sql": "{CUBE}.' + part[:3] + '"}}') + + +def _ossie_fields(*specs): + """Build an Ossie model from (field name, expression, geo part or None) specs.""" + out = ("version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: users\n" + " source: public.users\n" + " fields:\n") + for fname, expr, part in specs: + out += (f" - name: {fname}\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + f" expression: {expr}\n" + " datatype: String\n") + if part: + out += (" custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{_geo_stash(part)}'\n") + return out + + +def test_geo_halves_may_appear_in_any_order_without_clobbering_a_dimension(): + """The geo dimension is assembled from two fields that need not be adjacent and + may come in either order. Holding its place with a list index computed mid-loop + overwrote whatever real dimension already sat at that index -- here `city` + vanished entirely.""" + model = _ossie_fields( + ("home_longitude", "lon", "longitude"), + ("city", "city", None), + ("home_latitude", "lat", "latitude"), + ) + files, _ = convert_ossie_to_cube(model) + dims = parse(files["model/cubes/users.yml"])["cubes"][0]["dimensions"] + assert [d["name"] for d in dims] == ["home", "city"] + assert by_name(dims)["home"] == { + "name": "home", "type": "geo", + "latitude": {"sql": "{CUBE}.lat"}, + "longitude": {"sql": "{CUBE}.lon"}} + assert by_name(dims)["city"]["sql"] == "city" + + +def test_a_geo_base_colliding_with_a_field_is_rejected_in_either_order(): + """The base is the merged dimension's name, so it cannot also be an ordinary + dimension -- that would emit two members of the same name. Whether the ordinary + field comes first must not decide whether this is caught.""" + for specs in ( + (("home", "home", None), ("home_latitude", "lat", "latitude"), + ("home_longitude", "lon", "longitude")), + (("home_latitude", "lat", "latitude"), + ("home_longitude", "lon", "longitude"), ("home", "home", None)), + ): + with pytest.raises(ConversionError, match="collides"): + convert_ossie_to_cube(_ossie_fields(*specs)) + + +def test_two_fields_claiming_the_same_geo_half_are_rejected(): + model = _ossie_fields( + ("a_lat", "lat", "latitude"), + ("b_lat", "lat2", "latitude"), + ("home_longitude", "lon", "longitude"), + ) + with pytest.raises(ConversionError, match="both claim the latitude"): + convert_ossie_to_cube(model) + + +def test_a_geo_dimension_missing_a_half_is_rejected_on_export(): + model = _ossie_fields(("home_latitude", "lat", "latitude")) + with pytest.raises(ConversionError, match="missing its longitude half"): + convert_ossie_to_cube(model) + + +def test_an_unknown_geo_part_is_rejected(): + model = _ossie_fields(("home_altitude", "alt", "altitude")) + with pytest.raises(ConversionError, match="geo part 'altitude'"): + convert_ossie_to_cube(model) + + def test_geo_dimension_missing_a_half_is_rejected(): files = _files(users=( "cubes:\n" From 07e818647d01ab301508faaaa2c7705c754911df Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 15:54:50 +0500 Subject: [PATCH 14/25] Correct every mislabelled issue type, not just the flagged one From the Copilot review on #289: additional `semantic_model` entries were reported as PARKED_IN_META, but they are neither converted nor preserved anywhere -- a drop. That is the third instance of the same mislabelling, so rather than patch the flagged line I audited all seven export-side uses. Exactly one was a genuine park: unique_keys -> PARKED_IN_META (correct) extra semantic_model entries -> DROPPED (was parked) dimension.is_time role -> DROPPED (was parked) dimension.is_time opt-out -> DROPPED (was parked) synthesized primary-key dimension -> APPROXIMATED (was parked) no datatype -> Cube type 'string' -> APPROXIMATED (was parked) cross-dataset metric placement -> APPROXIMATED (was parked) The import-direction uses were all genuine parks and are unchanged. Adds APPROXIMATED for the middle case, which neither of the existing types described: nothing is lost and nothing is hidden, but Cube requires a value Ossie does not carry, so the converter chose one and the output asserts slightly more than the input did. Calling that "parked" was wrong in the same way as calling a drop "parked" -- nothing was parked. The point of keeping three types apart is that a caller gating on them can distinguish preserved-but-unreadable from actually-lost from emitted-with-a-guess. Two of the three could not be told apart before. 241 tests. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 9 +++++++-- .../cube/src/ossie_cube/converter_issues.py | 7 +++++++ converters/cube/src/ossie_cube/osi_to_cube.py | 18 ++++++++++-------- converters/cube/tests/test_edge_cases.py | 4 +++- converters/cube/tests/test_osi_to_cube.py | 11 +++++++---- 5 files changed, 34 insertions(+), 15 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 6641a1c5..72e3bb64 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -233,8 +233,13 @@ element it concerns, and a detail string. | `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | | `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | | `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | -| `PARKED_IN_META` | An element with no native mapping, preserved in the stash or under `meta.ossie` — invisible to Cube but intact through a round trip | -| `DROPPED_NO_CUBE_EQUIVALENT` | A value Cube has nowhere to hold *and* that cannot be parked, so it is genuinely gone. Currently only relationship `ai_context`, since a Cube join entry has no `meta` field. Kept distinct from `PARKED_IN_META` so a caller can tell real loss from "preserved but unreadable by Cube" | +| `PARKED_IN_META` | Preserved in the stash or under `meta.ossie` — invisible to Cube, but intact through a round trip | +| `DROPPED_NO_CUBE_EQUIVALENT` | **Gone from the output.** Cube has nowhere to hold it and it cannot be parked: relationship `ai_context` (a Cube join entry has no `meta`), a `dimension.is_time` role or opt-out that Cube expresses only through `type`, and the second and later `semantic_model` entries | +| `APPROXIMATED` | Emitted, but not an exact equivalent: a value Cube requires and Ossie does not carry (so the converter chose one), or a construct rendered in the nearest form Cube has | + +These three are kept distinct on purpose. A caller gating on issue types has to be +able to tell "preserved but unreadable by Cube" from "actually lost" from "emitted, +but asserting slightly more than the input did". ## Requirements diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index 0c79610d..0b3dec73 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -72,6 +72,13 @@ class IssueType(Enum): # "actually lost". DROPPED_NO_CUBE_EQUIVALENT = "DROPPED_NO_CUBE_EQUIVALENT" + # Something *was* emitted, but it is not an exact equivalent: a value Cube + # requires and Ossie does not carry (so the converter had to choose one), or a + # construct rendered in the nearest form Cube has. Nothing is lost and nothing + # is hidden -- but the output asserts a little more than the input did, so it + # is worth a look. + APPROXIMATED = "APPROXIMATED" + @dataclass(frozen=True) class ConverterIssue: diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 70f0ed1d..ab2dd712 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -100,8 +100,9 @@ def convert_ossie_to_cube(ossie_yaml_str, dialect=None, base_cube=None): issues = IssueLog() if len(models) > 1: - issues.add(IssueType.PARKED_IN_META, "model", - f"{len(models)} semantic models found; converting only the first") + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, "model", + f"{len(models)} semantic models found; only the first is " + f"converted and the rest are not preserved anywhere") return _convert_model(models[0], dialect, base_cube, issues) @@ -280,7 +281,7 @@ def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, if col in covered: pk_names.append(covered[col]) continue - issues.add(IssueType.PARKED_IN_META, scope, + issues.add(IssueType.APPROXIMATED, scope, f"primary key column '{col}' has no field; emitted as a " f"non-public dimension with type 'string' (Cube requires a type " f"and Ossie carries none here)") @@ -466,18 +467,19 @@ def _dimension_type(field, stash, scope, issues): if ctype is None: raise ConversionError(f"{scope}: unknown datatype '{datatype}'") if explicit_is_time is True and ctype != "time": - issues.add(IssueType.PARKED_IN_META, scope, + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, scope, f"is_time is true but datatype '{datatype}' maps to Cube " f"type '{ctype}'; Cube marks time dimensions by type, so " f"the temporal role is not carried") elif explicit_is_time is False and ctype == "time": - issues.add(IssueType.PARKED_IN_META, scope, + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, scope, f"is_time is false but datatype '{datatype}' maps to Cube " - f"type 'time', which Cube always treats as a time dimension") + f"type 'time', which Cube always treats as a time dimension; " + f"the opt-out is not carried") return ctype if explicit_is_time: return "time" - issues.add(IssueType.PARKED_IN_META, scope, + issues.add(IssueType.APPROXIMATED, scope, "no datatype; emitted as Cube type 'string', which Cube requires") return "string" @@ -652,7 +654,7 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ expr) if ref in sanitized }) > 1: - issues.add(IssueType.PARKED_IN_META, scope, + issues.add(IssueType.APPROXIMATED, scope, f"expression spans several datasets; emitted as a calculated " f"measure on cube '{target}' -- verify the join path") return measure diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 4b82a07b..1f29359b 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -989,4 +989,6 @@ def test_several_semantic_models_convert_the_first_with_an_issue(): ) files, issues = convert_ossie_to_cube(ossie) assert set(files) == {"model/cubes/orders.yml", "model/views/first.yml"} - assert any("converting only the first" in i.detail for i in issues) + # The other models are not preserved anywhere, so this is a drop. + dropped = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT) + assert any("only the first is converted" in i.detail for i in dropped) diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 6372b5bf..1df017ab 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -123,7 +123,8 @@ def test_every_dimension_declares_a_type(): ) files, issues = convert_ossie_to_cube(_ossie(no_type)) assert _cubes(files)["orders"]["dimensions"][0]["type"] == "string" - assert issues.of_type(IssueType.PARKED_IN_META) + # A guess, not a loss and not a park: Cube demands a type Ossie never gave. + assert issues.of_type(IssueType.APPROXIMATED) @pytest.mark.parametrize("datatype,expected", [ @@ -172,7 +173,8 @@ def test_is_time_on_a_non_temporal_datatype_is_reported(): files, issues = convert_ossie_to_cube(_ossie(ds)) dim = parse(files["model/cubes/date_dim.yml"])["cubes"][0]["dimensions"][0] assert dim["type"] == "number" - detail = issues.of_type(IssueType.PARKED_IN_META)[0].detail + # The temporal role is gone from the output, so this is a drop. + detail = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT)[0].detail assert "temporal role is not carried" in detail @@ -195,7 +197,8 @@ def test_primary_key_column_without_a_field_is_synthesized(): assert dims["ticket_no"] == { "name": "ticket_no", "sql": "ticket_no", "type": "string", "primary_key": True, "public": False} - assert issues.of_type(IssueType.PARKED_IN_META) + # `type: string` is chosen by the converter, not carried by Ossie. + assert issues.of_type(IssueType.APPROXIMATED) def test_field_name_is_sanitized_and_collisions_are_rejected(): @@ -387,7 +390,7 @@ def test_ratio_becomes_a_calculated_measure(): assert measure["type"] == "number" assert measure["sql"] == "SUM({CUBE.amount}) / COUNT(DISTINCT {users.id})" assert any("spans several datasets" in i.detail - for i in issues.of_type(IssueType.PARKED_IN_META)) + for i in issues.of_type(IssueType.APPROXIMATED)) def test_metric_lands_on_the_dataset_its_expression_references(): From c272df20123a63fc4ee3930de162b32532634f67 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 16:18:03 +0500 Subject: [PATCH 15/25] Fix five round-trip bugs found in review All five reproduce; each was confirmed before being fixed. 1. Multi-stage measures were lost outright. They get no `metrics` entry -- correctly, a window function over another grain has no static Ossie expression -- but nothing stashed them either, and `measures` is a natively-mapped key so `cube_extras` did not carry it. The issue message meanwhile claimed the measure had been preserved, which was simply untrue. They now ride on the owning dataset's stash with their index, the protocol unconvertible joins already use, and export interleaves them back among the measures rebuilt from metrics. Measure conversion moved ahead of dataset construction so the stash is known in time, which meant reading primary keys straight off the dimensions instead of taking them from _convert_cube's return. Renamed MULTI_STAGE_MEASURE_DROPPED -> MULTI_STAGE_MEASURE_PARKED: with the measure genuinely preserved, "dropped" was the same mislabelling corrected in c22e6d2. 2. One-to-one joins were treated as fan-out paths. Neither side of a one-to-one multiplies, so a valid `sum` on the `to` side was refused under strict mode. Now excluded, keyed off the normalized cardinality in the stash so `one_to_one` and legacy `hasOne` both count. A hand-authored relationship carries no Cube cardinality and keeps the conservative assumption. 3. Export could emit Cube its own importer rejects. `COUNT(*)` became a bare `type: count`, but that form is this converter's representation of `COUNT(DISTINCT )`, so importing it demanded a primary key the dataset need not have. `COUNT(*)` now becomes `type: number` with the expression intact -- a pair Cube's own BaseQuery special-cases -- so it round-trips exactly and needs no key. 4./5. Foreign-vendor extensions were parked under `meta.ossie` at field and metric level but only read back at dataset level, so they were dropped on re-import. Restored via one shared helper, after write_stash, so the CUBE entry stays first as it already did for datasets. 248 tests, 97% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 2 +- .../cube/src/ossie_cube/converter_issues.py | 8 +- converters/cube/src/ossie_cube/cube_to_osi.py | 120 ++++++++--- converters/cube/src/ossie_cube/osi_to_cube.py | 33 ++- converters/cube/tests/test_cube_to_osi.py | 2 +- converters/cube/tests/test_edge_cases.py | 201 ++++++++++++++++++ converters/cube/tests/test_osi_to_cube.py | 1 - 7 files changed, 323 insertions(+), 44 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 72e3bb64..ef5771e0 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -228,7 +228,7 @@ element it concerns, and a detail string. | Issue type | Meaning | |---|---| | `FANOUT_UNSAFE_METRIC` | A non-idempotent aggregate on a dataset the graph fans out; see [Fan-out](#fan-out) | -| `MULTI_STAGE_MEASURE_DROPPED` | A `multi_stage` measure (`group_by`/`reduce_by`/`time_shift`/`rank`) renders as a window function over another grain | +| `MULTI_STAGE_MEASURE_PARKED` | A `multi_stage` measure (`group_by`/`reduce_by`/`time_shift`/`rank`) renders as a window function over another grain, so it gets no `metrics` entry — the original is preserved verbatim in the dataset's stash and restored on export | | `CUBE_LEVEL_AI_CONTEXT_INERT` | Cube's agent ignores cube-level `meta.ai_context` | | `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | | `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index 0b3dec73..93df8842 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -39,10 +39,10 @@ class IssueType(Enum): FANOUT_UNSAFE_METRIC = "FANOUT_UNSAFE_METRIC" # A `multi_stage` measure (group_by / reduce_by / time_shift / rank). These - # render as window functions over a grain other than the query's, which an - # Ossie expression has no form for; the measure is preserved in the stash - # and omitted from `metrics`. - MULTI_STAGE_MEASURE_DROPPED = "MULTI_STAGE_MEASURE_DROPPED" + # render as window functions over a grain other than the query's, which an Ossie + # expression has no form for -- so the measure gets no `metrics` entry, and the + # original is preserved verbatim in the owning dataset's stash instead. + MULTI_STAGE_MEASURE_PARKED = "MULTI_STAGE_MEASURE_PARKED" # A cube-level `meta.ai_context`. Cube's own agent only consumes ai_context # on views and on individual members, so this value is inert in Cube; it is diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 8ae1bd48..2508942d 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -57,6 +57,7 @@ snake, snake_keys, view_file, + read_stash, write_stash, ) from .converter_issues import IssueLog, IssueType @@ -128,25 +129,25 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True) if ai: model["ai_context"] = ai - # Joins are decomposed first: a join with no Ossie form is parked on its - # declaring cube's stash, which has to be known before the dataset is built. + # Anything a cube's stash has to carry is worked out before the dataset is + # built: joins with no Ossie form, and measures with no static Ossie + # expression. Primary keys are read straight off the dimensions so this + # ordering does not depend on the datasets existing yet. relationships, extra_joins = _convert_joins(cubes, sorted(extra_files), issues) - - datasets = [] - pk_by_cube = {} - for cname, cube in cubes.items(): - ds, primary_key = _convert_cube(cname, cube, extra_joins.get(cname), issues) - datasets.append(ds) - pk_by_cube[cname] = primary_key - model["datasets"] = datasets if relationships: model["relationships"] = relationships - - # A dataset on the `to` (one) side of a relationship can be fanned out by rows - # from the `from` (many) side. Derived entirely from the Ossie graph. - fanned_out = {rel["to"]: rel["name"] for rel in relationships} - - metrics = _convert_measures(cubes, pk_by_cube, fanned_out, issues) + fanned_out = _fanned_out_datasets(relationships) + pk_by_cube = {cname: _primary_key_of(cube, cname) + for cname, cube in cubes.items()} + + metrics, extra_measures = _convert_measures( + cubes, pk_by_cube, fanned_out, issues) + + model["datasets"] = [ + _convert_cube(cname, cube, extra_joins.get(cname), + extra_measures.get(cname), issues) + for cname, cube in cubes.items() + ] if metrics: model["metrics"] = metrics @@ -393,10 +394,57 @@ def _meta_without_ai_context(meta): return {k: v for k, v in meta.items() if k not in ("ai_context", "ossie")} +def _fanned_out_datasets(relationships): + """{dataset: relationship name} for datasets a join can multiply rows of. + + A dataset on the `to` (one) side of a many-to-one join is fanned out by rows from + the `from` (many) side. A **one-to-one** join multiplies neither side, so it is + excluded -- otherwise a perfectly safe `sum` on either side would be refused + under strict fan-out mode. The cardinality comes from the stash Cube's join left + behind, in normalized form, so `one_to_one` and the legacy `has_one` both count. + + A hand-authored Ossie relationship carries no Cube cardinality, and Ossie's own + `from`/`to` says only many/one -- so it keeps the conservative assumption. + """ + out = {} + for rel in relationships: + declared = read_stash(rel).get("relationship") + if declared and _RELATIONSHIP_ALIASES.get(snake(declared)) == "one_to_one": + continue + out[rel["to"]] = rel["name"] + return out + + +def _restore_parked_extensions(obj, meta): + """Reattach foreign-vendor extensions a previous export parked under + `meta.ossie.custom_extensions`. + + Called after `write_stash`, so the CUBE entry stays first and the restored + foreign entries follow -- the ordering datasets already used. Without this the + parked entries are stripped by `_meta_without_ai_context` and never come back, + which would make `Ossie -> Cube -> Ossie` lose them. + """ + parked = ((meta or {}).get("ossie") or {}).get("custom_extensions") + if parked: + obj.setdefault("custom_extensions", []).extend(parked) + + # --- cubes ---------------------------------------------------------------------- -def _convert_cube(cname, cube, extra_joins, issues): - """Build one Ossie dataset from a Cube cube. Returns (dataset, primary_key).""" +def _primary_key_of(cube, cname): + """The names of a cube's `primary_key: true` dimensions. + + Read directly off the dimensions so the stages that need it -- measures, and the + fan-out check -- do not have to wait for the dataset to be built. + """ + return [require_str(dim, "name", f"cube '{cname}': dimension") + for dim in _as_named_list(cube.get("dimensions"), + f"cube '{cname}' dimensions") + if dim.get("primary_key")] + + +def _convert_cube(cname, cube, extra_joins, extra_measures, issues): + """Build one Ossie dataset from a Cube cube.""" scope = f"cube '{cname}'" ds = {"name": cname} stash = {} @@ -418,18 +466,22 @@ def _convert_cube(cname, cube, extra_joins, issues): ds["unique_keys"] = [list(k) for k in parked["unique_keys"]] fields = [] - primary_key = [] for dim in _as_named_list(cube.get("dimensions"), f"{scope} dimensions"): dname = require_str(dim, "name", f"{scope}: dimension") - if dim.get("primary_key"): - primary_key.append(dname) fields.extend(_convert_dimension(cname, dname, dim, issues)) if fields: ds["fields"] = fields + primary_key = _primary_key_of(cube, cname) if primary_key: ds["primary_key"] = primary_key if extra_joins: stash["extra_joins"] = extra_joins + if extra_measures: + # Measures with no static Ossie expression (multi-stage ones) ride here with + # their original positions, so export can put them back among the measures it + # rebuilds from metrics. Without this they would be lost outright: `measures` + # is a natively-mapped key, so `cube_extras` does not carry it. + stash["extra_measures"] = extra_measures extras = {snake(k): v for k, v in cube.items() if snake(k) not in _CUBE_NATIVE_KEYS} @@ -444,7 +496,7 @@ def _convert_cube(cname, cube, extra_joins, issues): # stash is written, so the CUBE entry stays first and both survive. if parked.get("custom_extensions"): ds.setdefault("custom_extensions", []).extend(parked["custom_extensions"]) - return ds, primary_key + return ds def _convert_dimension(cname, dname, dim, issues): @@ -504,6 +556,10 @@ def _convert_dimension(cname, dname, dim, issues): if leftover_meta: stash["meta"] = leftover_meta write_stash(field, stash) + # Foreign-vendor extensions a previous export parked under the dimension's + # `meta.ossie` are restored after the stash is written, so the CUBE entry stays + # first -- the same ordering datasets use. + _restore_parked_extensions(field, dim.get("meta")) return [field] @@ -734,7 +790,7 @@ def expression(self, cname, mname, stack=()): # group_by / reduce_by / time_shift / rank render as window functions # over a grain other than the query's; Ossie has no form for that. self._issues.add( - IssueType.MULTI_STAGE_MEASURE_DROPPED, scope, + IssueType.MULTI_STAGE_MEASURE_PARKED, scope, f"multi_stage measure (type '{mtype}'); preserved in " f"custom_extensions only") return None @@ -825,6 +881,12 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): A metric name is the measure name when globally unique, else `__`; the original name and owning cube are stashed so export puts the measure back where it came from. + + Returns (metrics, {cube: [{"index": i, "measure": ...}]}). The second value holds + measures with no static Ossie expression -- a multi-stage measure renders as a + window function over another grain -- which have no `metrics` entry and would + otherwise vanish. They ride on the owning dataset's stash with their positions, + the same protocol unconvertible joins use. """ resolver = _MeasureResolver(cubes, pk_by_cube, issues) @@ -833,10 +895,12 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): counts[mname] = counts.get(mname, 0) + 1 metrics = [] + extra_measures = {} seen = set() for cname, cube in cubes.items(): - for measure in _as_named_list(cube.get("measures"), - f"cube '{cname}' measures"): + for index, measure in enumerate( + _as_named_list(cube.get("measures"), + f"cube '{cname}' measures")): mname = measure["name"] metric_name = mname if counts[mname] == 1 else f"{cname}__{mname}" if metric_name in seen: @@ -848,7 +912,10 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): fanned_out, issues) if metric is not None: metrics.append(metric) - return metrics + else: + extra_measures.setdefault(cname, []).append( + {"index": index, "measure": measure}) + return metrics, extra_measures def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, @@ -916,4 +983,5 @@ def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, if leftover_meta: stash["meta"] = leftover_meta write_stash(metric, stash) + _restore_parked_extensions(metric, measure.get("meta")) return metric diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index ab2dd712..6cf2f6be 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -303,8 +303,14 @@ def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, joins.insert(min(item.get("index", 0), len(joins)), item["join"]) if joins: cube["joins"] = joins + measures = [_ordered(m, _MEASURE_KEY_ORDER) for m in (measures or [])] + # Measures a prior import could not express in Ossie (multi-stage ones) go back + # at their original indices, interleaved with the ones rebuilt from metrics. + for item in sorted(stash.get("extra_measures") or [], + key=lambda x: x.get("index", 0)): + measures.insert(min(item.get("index", 0), len(measures)), item["measure"]) if measures: - cube["measures"] = [_ordered(m, _MEASURE_KEY_ORDER) for m in measures] + cube["measures"] = measures for key, value in cube_extras.items(): cube[key] = value @@ -632,16 +638,21 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ measure["type"] = "count" return measure func = "COUNT_DISTINCT" - if func == "COUNT" and inner == "*": - measure["type"] = "count" - return measure - agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) - if agg is not None: - measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( - inner, target, members, sanitized, - inline_sql=inline_sql_by_cube) - measure["type"] = agg - return measure + # `COUNT(*)` deliberately falls through to the calculated measure below. + # A bare Cube `type: count` is this converter's representation of + # `COUNT(DISTINCT )` -- handled above -- so emitting one here + # would round-trip back as a different expression, and on a dataset with no + # primary key it would produce a measure the importer refuses. Cube renders + # `type: number` with `count(*)` natively (BaseQuery special-cases exactly + # that pair), so the expression survives intact either way. + if not (func == "COUNT" and inner == "*"): + agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) + if agg is not None: + measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( + inner, target, members, sanitized, + inline_sql=inline_sql_by_cube) + measure["type"] = agg + return measure # A ratio, a window expression, or a multi-dataset aggregate: Cube expresses # these as a calculated measure whose sql carries the aggregation. diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index faedcbf7..66da6106 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -360,7 +360,7 @@ def test_multi_stage_measure_is_dropped_with_an_issue(): } out, issues = convert_cube_to_ossie(files) assert "metrics" not in model_of(out) - assert issues.of_type(IssueType.MULTI_STAGE_MEASURE_DROPPED) + assert issues.of_type(IssueType.MULTI_STAGE_MEASURE_PARKED) # --- fan-out -------------------------------------------------------------------- diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 1f29359b..5b34565f 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -156,6 +156,140 @@ def test_count_over_an_expression_is_fanout_unsafe(): assert issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) +_MULTI_STAGE = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: revenue\n" + " sql: amount\n" + " type: sum\n" + " - name: rolling\n" + " sql: amount\n" + " type: sum\n" + " multi_stage: true\n" + " rolling_window:\n" + " trailing: 3 month\n" + " - name: cnt\n" + " type: count\n" +)) + + +def test_a_multi_stage_measure_is_not_an_ossie_metric(): + """It renders as a window function over another grain, which an Ossie expression + has no form for -- so it gets no `metrics` entry, and that is reported.""" + ossie, issues = convert_cube_to_ossie(_MULTI_STAGE) + assert [m["name"] for m in model_of(ossie)["metrics"]] == ["revenue", "cnt"] + parked = issues.of_type(IssueType.MULTI_STAGE_MEASURE_PARKED) + assert [i.element_name for i in parked] == ["orders.rolling"] + + +def test_a_multi_stage_measure_survives_the_round_trip_in_place(): + """It used to be lost outright: no metric, and `measures` is a natively-mapped key + so `cube_extras` did not carry it either -- while the issue claimed it had been + preserved. Now it rides on the dataset's stash with its position, like an + unconvertible join, and comes back interleaved with the rebuilt measures.""" + ossie, _ = convert_cube_to_ossie(_MULTI_STAGE) + stashed = stash_of(by_name(model_of(ossie)["datasets"])["orders"]) + assert stashed["extra_measures"] == [ + {"index": 1, "measure": { + "name": "rolling", "sql": "amount", "type": "sum", + "multi_stage": True, "rolling_window": {"trailing": "3 month"}}}] + + back, _ = convert_ossie_to_cube(ossie) + assert parse_files(back) == parse_files(_MULTI_STAGE) + # Order matters: it goes back between the two ordinary measures. + names = [m["name"] for m in parse( + back["model/cubes/orders.yml"])["cubes"][0]["measures"]] + assert names == ["revenue", "rolling", "cnt"] + + +def test_count_star_is_not_emitted_as_a_bare_cube_count(): + """A bare Cube `type: count` is this converter's form for + `COUNT(DISTINCT )`. Emitting one for `COUNT(*)` round-tripped back as a + different expression, and on a dataset with no primary key produced a measure + the importer refuses -- export generating what its own import rejects.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " metrics:\n" + " - name: n\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: COUNT(*)\n" + ) + files, _ = convert_ossie_to_cube(ossie) + measure = parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"][0] + assert measure == {"name": "n", "sql": "COUNT(*)", "type": "number"} + + # And it survives the trip back, without a primary key anywhere in sight. + ossie2, _ = convert_cube_to_ossie(files) + assert expr_of(model_of(ossie2)["metrics"][0]) == "COUNT(*)" + + +def test_field_and_metric_foreign_extensions_survive_the_round_trip(): + """Foreign-vendor extensions are parked under `meta.ossie` at every level, but + only datasets were reading them back -- so field- and metric-level ones were + parked and then silently dropped on re-import.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " fields:\n" + " - name: status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status\n" + " datatype: String\n" + " custom_extensions:\n" + " - vendor_name: SNOWFLAKE\n" + " data: '{\"collation\": \"en\"}'\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" + " metrics:\n" + " - name: total\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: SUM(orders.amount)\n" + " custom_extensions:\n" + " - vendor_name: DBT\n" + " data: '{\"model\": \"fct_orders\"}'\n" + ) + files, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files) + model = model_of(ossie2) + + field = by_name(by_name(model["datasets"])["orders"]["fields"])["status"] + exts = {e["vendor_name"]: e["data"] for e in field["custom_extensions"]} + assert exts["SNOWFLAKE"] == '{"collation": "en"}' + # The CUBE stash is written first, foreign entries appended -- as for datasets. + assert field["custom_extensions"][0]["vendor_name"] == "CUBE" + + metric = by_name(model["metrics"])["total"] + mexts = {e["vendor_name"]: e["data"] for e in metric["custom_extensions"]} + assert mexts["DBT"] == '{"model": "fct_orders"}' + assert metric["custom_extensions"][0]["vendor_name"] == "CUBE" + + # --- join orientation, both ways ------------------------------------------------ _ONE_TO_MANY = _files(m=( @@ -191,6 +325,73 @@ def test_one_to_many_is_flipped_back_onto_its_original_cube(): assert "joins" not in cubes["orders"] +@pytest.mark.parametrize("declared", ["one_to_one", "hasOne", "has_one"]) +def test_a_one_to_one_join_does_not_make_its_target_fanned_out(declared): + """A one-to-one join multiplies neither side, so a `sum` across it is safe. It was + being treated like any other relationship, whose `to` side *is* fanned out, and a + valid measure was refused under strict mode.""" + files = _files(m=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: profiles\n" + " sql: \"{CUBE}.id = {profiles}.user_id\"\n" + f" relationship: {declared}\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " - name: profiles\n" + " sql_table: public.profiles\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: score_total\n" + " sql: \"{CUBE}.score\"\n" + " type: sum\n" + )) + # Strict mode is the default; this must simply convert. + ossie, issues = convert_cube_to_ossie(files) + assert not issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + assert expr_of(by_name(model_of(ossie)["metrics"])["score_total"]) == ( + "SUM(profiles.score)") + + +def test_a_many_to_one_join_still_makes_its_target_fanned_out(): + """The counterpart: excluding one-to-one must not weaken the ordinary case.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: ltv\n" + " sql: \"{CUBE}.ltv\"\n" + " type: sum\n" + )) + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files) + + def test_one_to_one_keeps_its_declared_orientation(): files = _files(m=_ONE_TO_MANY["model/cubes/m.yml"].replace( "one_to_many", "one_to_one")) diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 1df017ab..44af252b 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -355,7 +355,6 @@ def _metric(name, expr): {"type": "count_distinct", "sql": "{CUBE.amount}"}), ("APPROX_COUNT_DISTINCT(orders.amount)", {"type": "count_distinct_approx", "sql": "{CUBE.amount}"}), - ("COUNT(*)", {"type": "count"}), ]) def test_aggregate_expressions_become_structured_measures(expr, expected): files, _ = convert_ossie_to_cube(_ossie(_ORDERS, metrics=_metric("m", expr))) From d42c6f89b1631b5136cf0e32bd7e5b4b83c830d4 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 17:03:10 +0500 Subject: [PATCH 16/25] Keep every view when several share one YAML file _build_views keyed one view per file path, so two views in the same file collapsed to whichever was written last. Reproduced: a file with `alpha` and `beta` came back holding only `beta`. Worse than a plain drop, because the survivor is arbitrary. In the reproduction the lost view was `alpha` -- the *mapped* one, which is where the model's description and AI context live, so the model's own metadata lost its home too. Now grouped path -> [views] and extended into files_content, preserving declaration order. The existing two-view test did not catch this: it put each view in its own file, so the paths never collided. The new tests use one shared file and assert both the round trip and that the mapped view is still the one carrying model metadata. 250 tests, 97% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/osi_to_cube.py | 16 ++++--- converters/cube/tests/test_edge_cases.py | 47 +++++++++++++++++++ 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 6cf2f6be..3033c940 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -166,9 +166,9 @@ def _convert_model(model, dialect, base_cube, issues): path = stashed_paths.get(cname) or cube_file(cname) files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) - for vpath, view in _build_views(model, model_stash, cube_names, relationships, - datasets, base_cube).items(): - files_content.setdefault(vpath, {}).setdefault("views", []).append(view) + for vpath, views in _build_views(model, model_stash, cube_names, relationships, + datasets, base_cube).items(): + files_content.setdefault(vpath, {}).setdefault("views", []).extend(views) files = {path: dump_yaml(content) for path, content in files_content.items()} @@ -701,7 +701,10 @@ def _balanced(s): def _build_views(model, model_stash, cube_names, relationships, datasets, base_cube): - """Return {file path: view dict}. + """Return {file path: [view dict, ...]}. + + A list per path, not a single view: several views can share one YAML file, and + keying one view per path silently kept only the last. Stashed views restore verbatim, with the natively mapped description and AI context re-injected on the mapped one. The `views` stash key being *present* -- @@ -742,7 +745,8 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, meta = _build_meta(model.get("ai_context"), view.get("meta"), parked) if meta: view["meta"] = meta - out[paths.get(vname) or view_file(vname)] = view + path = paths.get(vname) or view_file(vname) + out.setdefault(path, []).append(view) return out vname = sanitize_name(model.get("name", "model"), "Model", set()) @@ -756,7 +760,7 @@ def _build_views(model, model_stash, cube_names, relationships, datasets, cube_names, relationships, cube_names[_pick_base_cube(model.get("name", ""), datasets, relationships, base_cube)]) - out[view_file(vname)] = view + out[view_file(vname)] = [view] return out diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 5b34565f..7232f2e4 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -1087,6 +1087,53 @@ def test_foreign_extensions_survive_once_a_view_is_mapped(): assert "SNOWFLAKE" in vendors +_TWO_VIEWS_ONE_FILE = { + "model/all.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + "views:\n" + " - name: alpha\n" + " description: A\n" + " cubes:\n" + " - join_path: orders\n" + " includes: '*'\n" + " - name: beta\n" + " description: B\n" + ), +} + + +def test_several_views_in_one_file_all_survive(): + """Views were keyed one-per-path on export, so two sharing a file meant the second + overwrote the first. The lost one here is `alpha` -- the *mapped* view, which is + where the model's own description and AI context live.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS_ONE_FILE, view="alpha") + back, _ = convert_ossie_to_cube(ossie) + assert set(back) == {"model/all.yml"} + rebuilt = parse(back["model/all.yml"]) + # Declaration order preserved, both present. + assert [v["name"] for v in rebuilt["views"]] == ["alpha", "beta"] + assert [c["name"] for c in rebuilt["cubes"]] == ["orders"] + assert parse_files(back) == parse_files(_TWO_VIEWS_ONE_FILE) + + +def test_the_mapped_view_in_a_shared_file_still_carries_model_metadata(): + """The mapped view is the model's home for description and AI context, so it has + to be the one updated -- not whichever view happens to be written last.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS_ONE_FILE, view="alpha") + model = model_of(ossie) + assert model["name"] == "alpha" + assert model["description"] == "A" + + model["description"] = "edited" + files, _ = convert_ossie_to_cube(dump_yaml({ + "version": "0.2.0.dev0", "semantic_model": [model]})) + views = by_name(parse(files["model/all.yml"])["views"]) + assert views["alpha"]["description"] == "edited" + assert views["beta"]["description"] == "B" + + def test_both_views_are_restored_on_export(): ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") back, _ = convert_ossie_to_cube(ossie) From a377cab9b85c715f178087456b37aa4410d56e9f Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Thu, 30 Jul 2026 17:33:21 +0500 Subject: [PATCH 17/25] Mark a Cube primary key only on a dimension that actually is that column `primary_key: true` in Cube declares that dimension's own `sql` to be the key, but coverage was satisfied by name equality alone. Two ways that declared the wrong thing, both reproduced: primary_key: [id] + dimension `id` sql `LOWER(email)` -> {name: id, sql: LOWER(email), primary_key: true} the lowercased email became the key primary_key: [location] + merged geo dimension `location` -> {name: location, type: geo, primary_key: true} a dimension with two sql expressions and no single one Coverage now requires the dimension to be *scalar* -- its expression a single source column -- reachable either by that column's name or by its own name. A computed dimension and a merged geo dimension qualify under neither. Removing the name match outright would have regressed the round trip: import records the *dimension name* in `primary_key`, not the column, so a Cube dimension `order_id` with `sql: id` comes back as `primary_key: [order_id]`. Gating the name match on the dimension being scalar keeps that working while excluding the two bad cases -- verified against a Cube -> Ossie -> Cube trip that stays exact. Synthesizing the replacement now avoids collisions. The obvious name is the key entry itself, but a computed or geo dimension may already own it, and emitting a second dimension of that name would produce an invalid cube while overwriting would lose a member. A `_pk` suffix is added until the name is free, and the issue says so when it happens. 255 tests, 97% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 2 +- converters/cube/src/ossie_cube/osi_to_cube.py | 84 +++++++++---- converters/cube/tests/test_edge_cases.py | 112 ++++++++++++++++++ 3 files changed, 173 insertions(+), 25 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index ef5771e0..ecb19241 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -121,7 +121,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `dataset.source` (`SELECT ...`) | `sql` | Cube requires exactly one of `sql` / `sql_table`. | | `dataset.description` | cube `description` | | | `dataset.ai_context` | cube `meta.ai_context` | Preserved for the round trip, but **inert in Cube** -- its agent ignores cube-level `ai_context`. Recorded as an issue. | -| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. Export: a key column no field covers becomes a `public: false` dimension. | +| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. Export marks a dimension only when it is **scalar** — a single source column — since `primary_key: true` declares that dimension's own `sql` to be the key; a computed dimension or a merged `geo` one would declare the wrong thing even if its name matches. Anything left uncovered becomes a `public: false` scalar dimension, suffixed (`id_pk`) if the obvious name is taken. | | `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot; parked rather than dropped. | | field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | | `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 3033c940..8ac3d7b1 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -271,25 +271,35 @@ def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, "Cube's agent reads ai_context only on views and members, " "so this cube-level value has no effect in Cube") - dimensions, covered = _build_dimensions( + dimensions, by_name_scalar, by_column = _build_dimensions( ds, cname, dim_names, inline_sql, dialect, issues) - # A primary-key column no field covers still has to exist as a dimension for - # Cube to join or roll up the cube. + # Resolve each `primary_key` entry to the dimension Cube should mark. A + # dimension only qualifies when it is *scalar* -- backed by a single source + # column -- because `primary_key: true` in Cube declares that dimension's own + # sql to be the key. A computed dimension would declare the wrong expression, + # and a merged geo dimension has no single sql at all, so neither counts even + # when its name matches. Anything left uncovered gets a private dimension. pk_names = [] - for col in (ds.get("primary_key") or []): - col = str(col) - if col in covered: - pk_names.append(covered[col]) + taken = {d["name"].lower() for d in dimensions} + for entry in (ds.get("primary_key") or []): + entry = str(entry) + # Import records the *dimension name*, so the name match is checked first; + # a hand-authored model naming the source column resolves by column. + match = by_name_scalar.get(entry) or by_column.get(entry) + if match: + pk_names.append(match) continue - issues.add(IssueType.APPROXIMATED, scope, - f"primary key column '{col}' has no field; emitted as a " - f"non-public dimension with type 'string' (Cube requires a type " - f"and Ossie carries none here)") - synth = {"name": col, "sql": col, "type": "string", - "primary_key": True, "public": False} - dimensions.append(synth) - covered[col] = col - pk_names.append(col) + name = _unique_pk_dimension_name(entry, taken) + taken.add(name.lower()) + detail = (f"primary key '{entry}' is not backed by a scalar dimension; " + f"emitted as a non-public dimension with type 'string' (Cube " + f"requires a type and Ossie carries none here)") + if name != entry: + detail += f", named '{name}' to avoid colliding with the existing member" + issues.add(IssueType.APPROXIMATED, scope, detail) + dimensions.append({"name": name, "sql": entry, "type": "string", + "primary_key": True, "public": False}) + pk_names.append(name) for dim in dimensions: if dim["name"] in pk_names: dim["primary_key"] = True @@ -381,14 +391,19 @@ def _resolve_dimension_names(ds, scope): def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): """Build a cube's dimensions from an Ossie dataset's fields. - Returns (dimensions, {column or field name: dimension name}) -- the second - value is what primary-key resolution matches against. Fields carrying a `geo` - stash are re-merged into the single Cube dimension they were split from. + Returns (dimensions, by_name_scalar, by_column) -- the two maps are what + primary-key resolution matches against, and both hold only *scalar* dimensions + (those whose expression is a single source column). A computed dimension and a + merged geo dimension are deliberately absent from both: Cube's + `primary_key: true` declares that dimension's own sql to be the key, so marking + either would declare something other than the column Ossie named. Fields + carrying a `geo` stash are re-merged into the single Cube dimension they were + split from. Dimension names come from `dim_names` (see `_resolve_dimension_names`) rather than being sanitized again here. """ ds_name = ds["name"] - covered = {} + by_name_scalar, by_column = {}, {} # Built by target dimension name rather than by list position: a geo dimension # is assembled from two fields that may appear in either order and need not be # adjacent, so an insertion index computed mid-loop is not a safe way to hold @@ -440,9 +455,11 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): dim[key] = value built[dname] = dim - covered[dname] = dname if is_simple_identifier(expr): - covered[expr.strip()] = dname + # Scalar: this dimension is exactly one source column, so Cube can mark + # it as the key. Reachable by its own name and by that column's name. + by_name_scalar[dname] = dname + by_column.setdefault(expr.strip(), dname) # Both halves are guaranteed present by _resolve_dimension_names, which # validates the pair before anything is built. @@ -453,11 +470,30 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): for key, value in (slot.get("host") or {}).items(): dim[key] = value built[base] = dim - covered[base] = base # A name in `order` with nothing built is a field dropped for want of a usable # dialect; it simply does not appear. - return [built[n] for n in order if n in built], covered + return [built[n] for n in order if n in built], by_name_scalar, by_column + + +def _unique_pk_dimension_name(entry, taken): + """A valid, unused Cube identifier for a synthesized primary-key dimension. + + The obvious name is the primary-key entry itself, but a computed or geo + dimension may already own it -- in which case emitting a second dimension of + that name would produce an invalid cube, and overwriting the existing one would + lose a member. So a suffix is added until the name is free. + """ + base = sanitize_name(entry, "primary key", set()) + if base.lower() not in taken: + return base + for n in range(1, 100): + candidate = f"{base}_pk" if n == 1 else f"{base}_pk_{n}" + if candidate.lower() not in taken: + return candidate + raise ConversionError( + f"cannot find a free dimension name for primary key '{entry}'; rename the " + f"colliding members in the Ossie model.") def _dimension_type(field, stash, scope, issues): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 7232f2e4..a86656d1 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -919,6 +919,118 @@ def _ossie_fields(*specs): return out +def _ossie_pk(primary_key, *specs): + """An Ossie model with a primary_key and (name, expression, geo part) fields.""" + out = ("version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " primary_key:\n") + for col in primary_key: + out += f" - {col}\n" + out += " fields:\n" + for fname, expr, part in specs: + out += (f" - name: {fname}\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + f" expression: {expr}\n" + " datatype: String\n") + if part: + out += (" custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{_geo_stash(part, of=fname.rsplit('_', 1)[0])}'\n") + return out + + +def _dims(files): + return parse(files["model/cubes/orders.yml"])["cubes"][0]["dimensions"] + + +def test_a_computed_dimension_does_not_cover_a_primary_key(): + """`primary_key: true` in Cube declares that dimension's own sql to be the key. + Marking a computed dimension would declare `LOWER(email)` as the key when Ossie + named the `id` column -- so a name match alone must not count as coverage.""" + files, issues = convert_ossie_to_cube( + _ossie_pk(["id"], ("id", "LOWER(email)", None))) + dims = by_name(_dims(files)) + assert "primary_key" not in dims["id"] + assert dims["id"]["sql"] == "LOWER(email)" + # A private scalar dimension carries the key instead, under a free name. + assert dims["id_pk"] == {"name": "id_pk", "sql": "id", "type": "string", + "primary_key": True, "public": False} + assert issues.of_type(IssueType.APPROXIMATED) + + +def test_a_merged_geo_dimension_does_not_cover_a_primary_key(): + """A geo dimension has two sql expressions and no single one, so it cannot be + the key even though its name matches.""" + files, _ = convert_ossie_to_cube(_ossie_pk( + ["location"], + ("location_latitude", "lat", "latitude"), + ("location_longitude", "lon", "longitude"))) + dims = by_name(_dims(files)) + assert dims["location"]["type"] == "geo" + assert "primary_key" not in dims["location"] + assert dims["location_pk"] == { + "name": "location_pk", "sql": "location", "type": "string", + "primary_key": True, "public": False} + + +def test_a_scalar_dimension_backed_by_the_key_column_covers_it(): + """The legitimate case: a differently-named dimension whose sql *is* the key + column. It stays the key, and nothing is synthesized alongside it.""" + files, issues = convert_ossie_to_cube( + _ossie_pk(["id"], ("order_id", "id", None))) + dims = _dims(files) + assert len(dims) == 1 + assert dims[0]["name"] == "order_id" + assert dims[0]["primary_key"] is True + assert not issues.of_type(IssueType.APPROXIMATED) + + +def test_a_scalar_dimension_named_as_the_key_covers_it(): + """Import records the *dimension name* in `primary_key`, not the column, so a + scalar dimension matching by name has to keep covering it -- otherwise + `Cube -> Ossie -> Cube` would synthesize a bogus duplicate key.""" + src = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: order_id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + )) + ossie, _ = convert_cube_to_ossie(src) + assert by_name(model_of(ossie)["datasets"])["orders"]["primary_key"] == [ + "order_id"] + back, _ = convert_ossie_to_cube(ossie) + assert parse_files(back) == parse_files(src) + + +def test_a_synthesized_key_name_avoids_every_existing_member(): + """Suffixing has to keep going while names are taken, and the result must still + be a single non-public scalar dimension.""" + files, _ = convert_ossie_to_cube(_ossie_pk( + ["id"], + ("id", "LOWER(email)", None), + ("id_pk", "UPPER(email)", None), + ("id_pk_2", "TRIM(email)", None))) + dims = by_name(_dims(files)) + keys = [n for n, d in dims.items() if d.get("primary_key")] + assert keys == ["id_pk_3"] + assert dims["id_pk_3"] == {"name": "id_pk_3", "sql": "id", "type": "string", + "primary_key": True, "public": False} + # Nothing was overwritten. + assert dims["id"]["sql"] == "LOWER(email)" + assert dims["id_pk"]["sql"] == "UPPER(email)" + assert dims["id_pk_2"]["sql"] == "TRIM(email)" + + def test_geo_halves_may_appear_in_any_order_without_clobbering_a_dimension(): """The geo dimension is assembled from two fields that need not be adjacent and may come in either order. Holding its place with a list index computed mid-loop From 27938071bad2165e74310d87d216e33d39278c14 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 16:02:38 +0500 Subject: [PATCH 18/25] Stop stashing what is not Cube-specific (first pass) Groundwork for the interop critique: the stash was dominated by round-trip bookkeeping rather than Cube features, and every entry becomes a warning-and-discard in converters that do not read foreign extensions. Measured on Cube -> Ossie -> Databricks, 32 of the warnings were purely this converter's own stash. Three sources removed: - A member's exact `sql` spelling. Only a *member* reference (`{CUBE.member}`) genuinely needs it, because Cube inlines the referenced member's own SQL; a plain `{CUBE}.column` regenerates faithfully. Added `sql_is_reversible` to tell them apart. - A `many_to_one` join declared on the many side, which is exactly what Ossie's `from`(many) -> `to`(one) already says. A legacy spelling (`belongsTo`) is still kept exact, so preserving it costs the modern spelling nothing. - The join `sql` in the common case, by emitting the alias-dot raw-column form on both sides (`{users}.id`) rather than a member reference (`{users.id}`). Ossie's from_columns/to_columns name columns, so this is also the more faithful form. Consequence, accepted deliberately: `{CUBE}.column` and a bare `column` mean the same thing and are no longer distinguished, so Cube -> Ossie -> Cube is now lossless *semantically* rather than byte-for-byte. Expressed in the tests as one narrow normalization in `_util.canon_sql`, which leaves `{CUBE.member}` alone precisely because that one is not equivalent. Fixture stash entries: fixtureA 18 -> 13, and Databricks foreign-extension warnings 32 -> 17 on the TPC-DS model. 255 tests, 96% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/_common.py | 27 +++ converters/cube/src/ossie_cube/cube_to_osi.py | 31 +++- converters/cube/src/ossie_cube/osi_to_cube.py | 2 +- converters/cube/tests/_roundtrip_helpers.py | 5 +- converters/cube/tests/_util.py | 42 ++++- .../cube/tests/fixtures/fixtureA_ossie.yaml | 38 ++--- .../cube/tests/fixtures/tpcds_ossie.yaml | 159 +++++++----------- converters/cube/tests/test_cube_to_osi.py | 7 +- converters/cube/tests/test_edge_cases.py | 5 +- converters/cube/tests/test_osi_to_cube.py | 6 +- 10 files changed, 170 insertions(+), 152 deletions(-) diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 5cb4ab28..f6db16e1 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,6 +385,33 @@ def repl(m): return out, changed +def sql_is_reversible(sql): + """True if translating this Cube SQL to Ossie and back reproduces it. + + Only `{CUBE}.column` / `{TABLE}.column` -- a raw physical column of the owning + cube -- survives the trip, because Ossie expressions address columns and the + exporter re-emits them bare. A *member* reference (`{CUBE.member}`, `{member}`, + `{other.member}`) does not: Cube inlines the referenced member's own SQL, which + can differ from a column of that name, so the original spelling has to be kept. + + Used to decide whether the exact Cube `sql` needs stashing at all -- most + dimensions reference plain columns, so most need nothing. + """ + if not isinstance(sql, str): + sql = str(sql) + protected = sql.replace("\\{", "").replace("\\}", "") + for m in _CUBE_REF_RE.finditer(protected): + body = m.group(1).strip() + if body not in _SELF_REFS: + return False + # `{CUBE}` on its own (no trailing `.column`) is the cube's alias, which an + # Ossie expression cannot express either. + rest = protected[m.end():] + if not rest.startswith("."): + return False + return True + + def requalify_self_refs(sql, cube_name): """Rewrite `{CUBE}` / `{TABLE}` in a Cube SQL snippet to name `cube_name`. diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 2508942d..c313e62d 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -56,6 +56,7 @@ require_str, snake, snake_keys, + sql_is_reversible, view_file, read_stash, write_stash, @@ -516,11 +517,14 @@ def _convert_dimension(cname, dname, dim, issues): # No `sql` means the same-named physical column. expr = dname else: - expr, changed = cube_sql_to_ossie(sql, cname) - if changed or str(sql).strip() == dname: - # Stashed when the Ossie expression differs from the Cube sql, and also - # when the sql is an explicit same-named bare column -- which export - # would otherwise normalize away to the implicit form. + expr, _ = cube_sql_to_ossie(sql, cname) + if not sql_is_reversible(sql): + # Only a *member* reference needs the original spelling kept: Cube + # inlines the referenced member's own SQL, which a bare column name in + # the Ossie expression would not reproduce. A plain `{CUBE}.column` (or + # a bare column) regenerates faithfully, so nothing is stashed -- which + # is the common case, and stashing it only added noise for every other + # converter reading the model. stash["sql"] = sql field = { @@ -645,7 +649,17 @@ def _convert_joins(cubes, skipped_files, issues): from_cube, to_cube = cname, target from_cols = [p[0] for p in pairs] to_cols = [p[1] for p in pairs] - stash = {"declared_on": cname, "relationship": raw_rel} + # A `many_to_one` join declared on the many side is exactly what Ossie's + # `from`(many) -> `to`(one) already says, so nothing is stashed for the + # common case. Only an orientation Ossie cannot express on its own -- + # one_to_many (flipped) or one_to_one (no many side) -- needs recording. + stash = {} + if (rel_type != "many_to_one" or cname != from_cube + or raw_rel != "many_to_one"): + # The last clause keeps a legacy spelling (`belongsTo`) exact + # without costing the modern spelling a stash entry. + stash["declared_on"] = cname + stash["relationship"] = raw_rel if rel_type == "one_to_many": from_cube, to_cube = to_cube, from_cube from_cols, to_cols = to_cols, from_cols @@ -730,9 +744,10 @@ def _ref_target(side, own_cube, target): def _rebuild_join_sql(target, pairs): """The canonical form export emits, used to decide whether the original has to be stashed. The own side is always `{CUBE}` so the join keeps working when the - cube is extended.""" + cube is extended, and both sides use the alias-dot raw-column form because + Ossie's from_columns/to_columns name columns, not members.""" return " AND ".join( - "{CUBE}." + own + " = {" + target + "." + other + "}" + "{CUBE}." + own + " = {" + target + "}." + other for own, other in pairs ) diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 8ac3d7b1..80845f80 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -570,7 +570,7 @@ def _build_joins(relationships, cube_names, issues): join["sql"] = stash["sql"] else: join["sql"] = " AND ".join( - "{CUBE}." + str(a) + " = {" + other + "." + str(b) + "}" + "{CUBE}." + str(a) + " = {" + other + "}." + str(b) for a, b in zip(own_cols, other_cols)) for key, value in stash.items(): if key not in ("declared_on", "relationship", "sql"): diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py index f7d20f16..2343f683 100644 --- a/converters/cube/tests/_roundtrip_helpers.py +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -185,7 +185,10 @@ def _build_dimension(rnd, name): def _parse_files(files): - return {name: load_yaml(text, name) for name, text in files.items()} + # Same documented normalization the fixture tests use; see _util.canon_sql. + from _util import canon_sql + return {name: canon_sql(load_yaml(text, name)) + for name, text in files.items()} def assert_cube_roundtrip_is_lossless(files): diff --git a/converters/cube/tests/_util.py b/converters/cube/tests/_util.py index bf11b481..5e18bd28 100644 --- a/converters/cube/tests/_util.py +++ b/converters/cube/tests/_util.py @@ -20,6 +20,7 @@ import copy import json import pathlib +import re from ossie_cube._common import load_yaml # src is on sys.path via conftest.py @@ -46,17 +47,46 @@ def parse(yaml_str): return load_yaml(yaml_str) +_ALIAS_DOT_RE = re.compile(r"\$?\{\s*(?:CUBE|TABLE)\s*\}\s*\.") + + +def canon_sql(node): + """Canonicalize the one documented Cube SQL normalization, in place-ish. + + `{CUBE}.column` and a bare `column` are the same thing -- a raw physical column + of the owning cube -- so the converter no longer stashes the original spelling + just to reproduce it. Round-trip assertions therefore compare with the alias + prefix removed. + + Deliberately narrow: `{CUBE.member}` is a *member* reference and means something + else, so it is left alone (and is still stashed, so it round-trips exactly). + """ + if isinstance(node, dict): + return {k: (_ALIAS_DOT_RE.sub("", v) + if k == "sql" and isinstance(v, str) else canon_sql(v)) + for k, v in node.items()} + if isinstance(node, list): + return [canon_sql(v) for v in node] + return node + + def parse_files(files): - """Parse every file of a Cube model dict for structural comparison. + """Parse a Cube model dict into the form round-trip fidelity is asserted on. + + Comments and key order are not part of the data model, so comparison happens on + parsed structures. A non-YAML file (a `.js` model preserved verbatim) is + compared as text. - Comments and key order are not part of the data model, so round-trip fidelity - is asserted on the parsed structures. A non-YAML file (a `.js` model preserved - verbatim) is compared as text. + Also applies `canon_sql`: the converter no longer stashes a member's exact SQL + spelling just to reproduce `{CUBE}.column` over a bare `column`, since the two + mean the same thing and stashing it put noise into every other converter's view + of the model. That spelling is therefore a documented normalization, not a + difference worth failing on. """ out = {} for name, text in files.items(): - out[name] = (load_yaml(text, name) if name.lower().endswith((".yml", ".yaml")) - else text) + out[name] = (canon_sql(load_yaml(text, name)) + if name.lower().endswith((".yml", ".yaml")) else text) return out diff --git a/converters/cube/tests/fixtures/fixtureA_ossie.yaml b/converters/cube/tests/fixtures/fixtureA_ossie.yaml index d78abf3d..8247d6c2 100644 --- a/converters/cube/tests/fixtures/fixtureA_ossie.yaml +++ b/converters/cube/tests/fixtures/fixtureA_ossie.yaml @@ -25,6 +25,14 @@ semantic_model: ai_context: instructions: | Primary view for revenue analysis. Use it for any question about sales, orders, or customer spend. + relationships: + - name: orders_to_users + from: orders + to: users + from_columns: + - user_id + to_columns: + - id datasets: - name: orders source: public.orders @@ -37,7 +45,7 @@ semantic_model: expression: id custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "id", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: user_id expression: dialects: @@ -45,7 +53,7 @@ semantic_model: expression: user_id custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "user_id", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: status expression: dialects: @@ -56,9 +64,6 @@ semantic_model: description: Current order status ai_context: instructions: Values are pending, shipped, and completed. - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "status"}' - name: created_at expression: dialects: @@ -67,18 +72,12 @@ semantic_model: datatype: DateTime dimension: is_time: true - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "created_at"}' - name: is_large expression: dialects: - dialect: ANSI_SQL expression: amount > 500 datatype: Boolean - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "{CUBE}.amount > 500"}' primary_key: - id - name: users @@ -91,16 +90,13 @@ semantic_model: expression: id custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "id", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: city expression: dialects: - dialect: ANSI_SQL expression: city datatype: String - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "city"}' - name: location_latitude expression: dialects: @@ -125,18 +121,6 @@ semantic_model: - vendor_name: CUBE data: '{"_v": 1, "cube_extras": {"segments": [{"name": "active", "sql": "{CUBE}.status = ''active''"}]}}' - relationships: - - name: orders_to_users - from: orders - to: users - from_columns: - - user_id - to_columns: - - id - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "declared_on": "orders", "relationship": "many_to_one", "sql": - "{CUBE}.user_id = {users}.id"}' metrics: - name: orders__count expression: diff --git a/converters/cube/tests/fixtures/tpcds_ossie.yaml b/converters/cube/tests/fixtures/tpcds_ossie.yaml index 5ee7e1ae..0e6ad634 100644 --- a/converters/cube/tests/fixtures/tpcds_ossie.yaml +++ b/converters/cube/tests/fixtures/tpcds_ossie.yaml @@ -26,6 +26,47 @@ semantic_model: sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics. + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "{CUBE}.ss_sold_date_sk = {date_dim.d_date_sk}"}' + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "{CUBE}.ss_customer_sk = {customer.c_customer_sk}"}' + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "{CUBE}.ss_item_sk = {item.i_item_sk}"}' + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "sql": "{CUBE}.ss_store_sk = {store.s_store_sk}"}' datasets: - name: store_sales source: tpcds.public.store_sales @@ -52,7 +93,7 @@ semantic_model: - transaction date custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_sold_date_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_item_sk expression: dialects: @@ -65,7 +106,7 @@ semantic_model: - item custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_item_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_customer_sk expression: dialects: @@ -78,7 +119,7 @@ semantic_model: - buyer custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_customer_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_store_sk expression: dialects: @@ -91,7 +132,7 @@ semantic_model: - location custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_store_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_quantity expression: dialects: @@ -104,7 +145,7 @@ semantic_model: - quantity custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_quantity", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_sales_price expression: dialects: @@ -117,7 +158,7 @@ semantic_model: - price custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_sales_price", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_ext_sales_price expression: dialects: @@ -130,7 +171,7 @@ semantic_model: - line total custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_ext_sales_price", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_net_profit expression: dialects: @@ -143,7 +184,7 @@ semantic_model: - margin custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_net_profit", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: ss_ticket_number expression: dialects: @@ -152,7 +193,7 @@ semantic_model: datatype: String custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "ss_ticket_number", "public": false}' + data: '{"_v": 1, "public": false}' primary_key: - ss_item_sk - ss_ticket_number @@ -175,7 +216,7 @@ semantic_model: description: Surrogate key for date custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "d_date_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: d_date expression: dialects: @@ -189,9 +230,6 @@ semantic_model: synonyms: - date - calendar date - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "d_date"}' - name: d_year expression: dialects: @@ -203,7 +241,7 @@ semantic_model: - year custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "d_year", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: d_quarter_name expression: dialects: @@ -217,9 +255,6 @@ semantic_model: synonyms: - quarter - fiscal quarter - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "d_quarter_name"}' - name: d_month_name expression: dialects: @@ -232,9 +267,6 @@ semantic_model: ai_context: synonyms: - month - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "d_month_name"}' primary_key: - d_date_sk - name: customer @@ -256,7 +288,7 @@ semantic_model: description: Surrogate key for customer custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "c_customer_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: c_customer_id expression: dialects: @@ -268,9 +300,6 @@ semantic_model: synonyms: - customer ID - customer number - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "c_customer_id"}' - name: c_first_name expression: dialects: @@ -278,9 +307,6 @@ semantic_model: expression: c_first_name datatype: String description: Customer first name - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "c_first_name"}' - name: c_last_name expression: dialects: @@ -288,9 +314,6 @@ semantic_model: expression: c_last_name datatype: String description: Customer last name - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "c_last_name"}' - name: customer_full_name expression: dialects: @@ -313,9 +336,6 @@ semantic_model: synonyms: - email - contact - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "c_email_address"}' primary_key: - c_customer_sk - name: item @@ -337,7 +357,7 @@ semantic_model: description: Surrogate key for item custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_item_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: i_item_id expression: dialects: @@ -350,9 +370,6 @@ semantic_model: - item ID - product ID - SKU - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_item_id"}' - name: i_item_desc expression: dialects: @@ -364,9 +381,6 @@ semantic_model: synonyms: - product description - item name - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_item_desc"}' - name: i_brand expression: dialects: @@ -378,9 +392,6 @@ semantic_model: synonyms: - brand - manufacturer - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_brand"}' - name: i_category expression: dialects: @@ -392,9 +403,6 @@ semantic_model: synonyms: - product category - department - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_category"}' - name: i_current_price expression: dialects: @@ -407,7 +415,7 @@ semantic_model: - list price custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "i_current_price", "type": "number"}' + data: '{"_v": 1, "type": "number"}' primary_key: - i_item_sk - name: store @@ -429,7 +437,7 @@ semantic_model: description: Surrogate key for store custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_store_sk", "type": "number"}' + data: '{"_v": 1, "type": "number"}' - name: s_store_id expression: dialects: @@ -441,9 +449,6 @@ semantic_model: synonyms: - store ID - store number - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_store_id"}' - name: s_store_name expression: dialects: @@ -455,9 +460,6 @@ semantic_model: synonyms: - store name - location name - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_store_name"}' - name: s_city expression: dialects: @@ -469,9 +471,6 @@ semantic_model: synonyms: - city - location - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_city"}' - name: s_state expression: dialects: @@ -483,9 +482,6 @@ semantic_model: synonyms: - state - region - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_state"}' - name: s_number_employees expression: dialects: @@ -498,50 +494,9 @@ semantic_model: - staff size custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "sql": "s_number_employees", "type": "number"}' + data: '{"_v": 1, "type": "number"}' primary_key: - s_store_sk - relationships: - - name: store_sales_to_date_dim - from: store_sales - to: date_dim - from_columns: - - ss_sold_date_sk - to_columns: - - d_date_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' - - name: store_sales_to_customer - from: store_sales - to: customer - from_columns: - - ss_customer_sk - to_columns: - - c_customer_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' - - name: store_sales_to_item - from: store_sales - to: item - from_columns: - - ss_item_sk - to_columns: - - i_item_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' - - name: store_sales_to_store - from: store_sales - to: store - from_columns: - - ss_store_sk - to_columns: - - s_store_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "declared_on": "store_sales", "relationship": "many_to_one"}' metrics: - name: total_sales expression: diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index 66da6106..fcf2335e 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -155,9 +155,10 @@ def test_many_to_one_join_becomes_a_relationship(model_a): assert rel["to"] == "users" assert rel["from_columns"] == ["user_id"] assert rel["to_columns"] == ["id"] - # The declaring side and the exact Cube spelling round-trip via the stash. - assert stash_of(rel)["declared_on"] == "orders" - assert stash_of(rel)["relationship"] == "many_to_one" + # Nothing is stashed: a many_to_one join declared on the many side is exactly + # what `from`(many) -> `to`(one) already says, so recording it again would only + # add a custom_extension for every other converter to warn about and discard. + assert stash_of(rel) == {} def test_one_to_many_join_is_flipped_to_many_side_first(): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index a86656d1..e7e92beb 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -281,8 +281,9 @@ def test_field_and_metric_foreign_extensions_survive_the_round_trip(): field = by_name(by_name(model["datasets"])["orders"]["fields"])["status"] exts = {e["vendor_name"]: e["data"] for e in field["custom_extensions"]} assert exts["SNOWFLAKE"] == '{"collation": "en"}' - # The CUBE stash is written first, foreign entries appended -- as for datasets. - assert field["custom_extensions"][0]["vendor_name"] == "CUBE" + # A plain scalar field needs no CUBE stash at all any more, so the foreign + # extension is the only entry -- which is the point of the reduction. + assert list(exts) == ["SNOWFLAKE"] metric = by_name(model["metrics"])["total"] mexts = {e["vendor_name"]: e["data"] for e in metric["custom_extensions"]} diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 44af252b..f2ee960b 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -306,7 +306,9 @@ def test_preferred_dialect_wins_over_ansi(): def test_relationship_lands_on_the_many_side_as_many_to_one(): files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL)) join = _cubes(files)["orders"]["joins"][0] - assert join == {"name": "users", "sql": "{CUBE}.user_id = {users.id}", + # Alias-dot on both sides: Ossie's from_columns/to_columns name columns, so the + # far side is a raw column reference too, not a member reference. + assert join == {"name": "users", "sql": "{CUBE}.user_id = {users}.id", "relationship": "many_to_one"} # The one side declares nothing; Cube needs the join on one side only. assert "joins" not in _cubes(files, "model/cubes/users.yml")["users"] @@ -319,7 +321,7 @@ def test_composite_relationship_becomes_an_and_chain(): " to_columns: [id, region]\n") files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) assert _cubes(files)["orders"]["joins"][0]["sql"] == ( - "{CUBE}.user_id = {users.id} AND {CUBE}.region = {users.region}") + "{CUBE}.user_id = {users}.id AND {CUBE}.region = {users}.region") def test_relationship_ai_context_is_reported_as_dropped_not_parked(): From a397e85df4c5b3a913d0e41ce1084272a7b49a15 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 16:19:32 +0500 Subject: [PATCH 19/25] Map datatypes natively and narrow member references Two more stash sources removed, and both come out better than the stash they replace rather than merely smaller. Cube's dimension `type` was stashed because it is coarser than Ossie's `datatype` -- Integer, Decimal and Float all become `number` -- so no mapping back is exact. Omitting the datatype and stashing `number` gave every other spoke a warning and no type information at all. Now `number` maps to `Decimal`, the safe reading for the money and quantity columns it overwhelmingly holds, and export parks the *precise* datatype under `meta.ossie.datatype` whenever the default would not recover it. `meta.ossie` is Cube-side, so it costs the Ossie document nothing. The result is strictly better in both directions: all nine Ossie datatypes now survive Ossie -> Cube -> Ossie exactly, where `Integer` and `Float` were previously lost outright, and the Ossie document carries a real datatype for other converters to act on. Measure operands stopped stashing their spelling too. `{CUBE.member}` is only required when the member is *not* just its own same-named column, because that form makes Cube inline the member's SQL; for a plain member the raw `{CUBE}.column` form is identical and regenerates. Cross-cube references are deliberately left in member form -- that is what makes Cube add the implicit join, so the two are not interchangeable there. Regenerated tpcds_cube from examples/tpcds_semantic_model.yaml so the fixture reflects what the converter now emits rather than an older shape. CUBE stash entries: TPC-DS 41 -> 7, fixtureA 18 -> 10. Foreign-extension warnings: databricks 32 -> 2, snowflake 41 -> 7, omni 41 -> 7, wisdom 41 -> 7. 255 tests, 96% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/src/ossie_cube/_common.py | 59 +++++++++---- converters/cube/src/ossie_cube/cube_to_osi.py | 50 +++++++---- converters/cube/src/ossie_cube/osi_to_cube.py | 53 +++++++++--- .../cube/tests/fixtures/fixtureA_ossie.yaml | 14 +--- .../tpcds_cube/model/cubes/store_sales.yml | 18 ++-- .../cube/tests/fixtures/tpcds_ossie.yaml | 82 +++++-------------- converters/cube/tests/test_cube_to_osi.py | 12 +-- converters/cube/tests/test_edge_cases.py | 2 +- converters/cube/tests/test_osi_to_cube.py | 14 ++-- 9 files changed, 167 insertions(+), 137 deletions(-) diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index f6db16e1..b25aa2ca 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,30 +385,45 @@ def repl(m): return out, changed -def sql_is_reversible(sql): +def sql_is_reversible(sql, plain_members=(), own_cube=None): """True if translating this Cube SQL to Ossie and back reproduces it. - Only `{CUBE}.column` / `{TABLE}.column` -- a raw physical column of the owning - cube -- survives the trip, because Ossie expressions address columns and the - exporter re-emits them bare. A *member* reference (`{CUBE.member}`, `{member}`, - `{other.member}`) does not: Cube inlines the referenced member's own SQL, which - can differ from a column of that name, so the original spelling has to be kept. + `{CUBE}.column` / `{TABLE}.column` -- a raw physical column of the owning cube -- + always survives, because Ossie expressions address columns and the exporter + re-emits them in that form. - Used to decide whether the exact Cube `sql` needs stashing at all -- most - dimensions reference plain columns, so most need nothing. + A *member* reference (`{CUBE.member}`, `{member}`) survives only when the member + is **plain**: its own `sql` is just the same-named column, so the reference and + the raw column are the same thing. Otherwise Cube inlines the member's own SQL, + which a bare column name would not reproduce, and the original spelling has to be + kept. + + A **cross-cube** reference never survives: `{other.member}` is what makes Cube + add the implicit join, and the raw `{other}.column` form does not, so the two are + not interchangeable. """ if not isinstance(sql, str): sql = str(sql) + plain = set(plain_members) protected = sql.replace("\\{", "").replace("\\}", "") for m in _CUBE_REF_RE.finditer(protected): body = m.group(1).strip() - if body not in _SELF_REFS: - return False - # `{CUBE}` on its own (no trailing `.column`) is the cube's alias, which an - # Ossie expression cannot express either. - rest = protected[m.end():] - if not rest.startswith("."): - return False + head, _, rest = body.partition(".") + if not rest: + if body in _SELF_REFS or (own_cube and body == own_cube): + # A bare alias only makes sense followed by `.column`. + if not protected[m.end():].startswith("."): + return False + continue + # `{member}` -- an unqualified own-cube member reference. + if body not in plain: + return False + continue + if head in _SELF_REFS or (own_cube and head == own_cube): + if rest not in plain: + return False + continue + return False # cross-cube reference; carries join semantics return True @@ -530,8 +545,22 @@ def join_source(cube, cube_name): "boolean": "Boolean", "time": "DateTime", "switch": "String", + # Cube collapses Integer/Decimal/Float into one type, so no mapping back is + # exact. `Decimal` is chosen over omitting a datatype because a downstream + # converter can use it: exact base-10 is the safe reading for the money and + # quantity columns `number` overwhelmingly holds, and asserting it beats + # emitting nothing plus a Cube-only extension no other spoke reads. When the + # model came from Ossie in the first place, the precise datatype is recovered + # from `meta.ossie.datatype` instead of guessed. + "number": "Decimal", } +# The datatype each Cube type maps back to by default. Export parks the original in +# `meta.ossie.datatype` only when it is *not* the default -- Cube cannot hold the +# distinction, and `meta.ossie` is Cube-side, so this keeps Ossie -> Cube -> Ossie +# exact without putting anything in `custom_extensions`. +DEFAULT_DATATYPE_FOR_CUBE_TYPE = dict(DIM_TYPE_TO_DATATYPE) + # Ossie `datatype` -> Cube dimension `type`, which is required on every # dimension. Lossy in the numeric and temporal directions by construction. DATATYPE_TO_DIM_TYPE = { diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index c313e62d..c2e5f0f1 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -432,6 +432,23 @@ def _restore_parked_extensions(obj, meta): # --- cubes ---------------------------------------------------------------------- +def _plain_members(cube, cname): + """Dimension names whose `sql` is just the same-named column. + + For those, `{CUBE.member}`, `{CUBE}.member` and a bare `member` all mean the + same thing, so the spelling carries no information worth stashing. Any other + member inlines its own SQL when referenced, which a column name would not + reproduce. + """ + plain = set() + for dim in _as_named_list(cube.get("dimensions"), f"cube '{cname}' dimensions"): + name = dim.get("name") + sql = dim.get("sql") + if name and (sql is None or str(sql).strip() == name): + plain.add(name) + return plain + + def _primary_key_of(cube, cname): """The names of a cube's `primary_key: true` dimensions. @@ -445,6 +462,7 @@ def _primary_key_of(cube, cname): def _convert_cube(cname, cube, extra_joins, extra_measures, issues): + plain = _plain_members(cube, cname) """Build one Ossie dataset from a Cube cube.""" scope = f"cube '{cname}'" ds = {"name": cname} @@ -469,7 +487,7 @@ def _convert_cube(cname, cube, extra_joins, extra_measures, issues): fields = [] for dim in _as_named_list(cube.get("dimensions"), f"{scope} dimensions"): dname = require_str(dim, "name", f"{scope}: dimension") - fields.extend(_convert_dimension(cname, dname, dim, issues)) + fields.extend(_convert_dimension(cname, dname, dim, plain, issues)) if fields: ds["fields"] = fields primary_key = _primary_key_of(cube, cname) @@ -500,7 +518,7 @@ def _convert_cube(cname, cube, extra_joins, extra_measures, issues): return ds -def _convert_dimension(cname, dname, dim, issues): +def _convert_dimension(cname, dname, dim, plain, issues): """Build the Ossie field(s) for one Cube dimension. Returns a list because a `type: geo` dimension carries two SQL expressions @@ -518,7 +536,7 @@ def _convert_dimension(cname, dname, dim, issues): expr = dname else: expr, _ = cube_sql_to_ossie(sql, cname) - if not sql_is_reversible(sql): + if not sql_is_reversible(sql, plain, cname): # Only a *member* reference needs the original spelling kept: Cube # inlines the referenced member's own SQL, which a bare column name in # the Ossie expression would not reproduce. A plain `{CUBE}.column` (or @@ -532,16 +550,13 @@ def _convert_dimension(cname, dname, dim, issues): "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, } datatype = DIM_TYPE_TO_DATATYPE.get(dtype) - if datatype: - field["datatype"] = datatype - elif dtype == "number": - # Cube collapses Integer/Decimal/Float into `number`, so no Ossie datatype - # is asserted -- the spec says to omit it when unknown. The original type - # rides in the stash so export reproduces it. - stash["type"] = dtype - else: + if not datatype: raise ConversionError( f"cube '{cname}': dimension '{dname}' has unknown type '{dtype}'") + # A precise datatype parked by a previous export wins over the default the Cube + # type maps to, since Cube itself cannot hold the distinction. + parked_dt = ((dim.get("meta") or {}).get("ossie") or {}).get("datatype") + field["datatype"] = parked_dt or datatype if dtype == "time": field["dimension"] = {"is_time": True} if dim.get("title"): @@ -924,7 +939,8 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): f"colliding measures in Cube") seen.add(metric_name) metric = _convert_measure(cname, mname, metric_name, measure, resolver, - fanned_out, issues) + fanned_out, _plain_members(cube, cname), + issues) if metric is not None: metrics.append(metric) else: @@ -934,7 +950,7 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, - issues): + plain, issues): scope = f"{cname}.{mname}" expr = resolver.expression(cname, mname) if expr is None: @@ -985,10 +1001,10 @@ def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, snake(k): v for k, v in measure.items() if snake(k) not in ("description", "meta") } - elif sql is not None: - # The operand's exact Cube spelling: `{CUBE}.city` and `{CUBE.city}` are - # equivalent but not interchangeable byte-for-byte, and export cannot tell - # which one the author wrote from the Ossie expression alone. + elif sql is not None and not sql_is_reversible(sql, plain, cname): + # Only a reference export cannot regenerate needs the original spelling: a + # non-plain member (whose own SQL is inlined) or a cross-cube reference + # (which is what adds the implicit join). stash["sql"] = sql if metric_name != mname: stash["name"] = mname diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 80845f80..beaee615 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -36,6 +36,7 @@ from ._common import ( DATATYPE_TO_DIM_TYPE, + DEFAULT_DATATYPE_FOR_CUBE_TYPE, OSSIE_FUNC_TO_AGG, OSSIE_VERSION, ConversionError, @@ -146,7 +147,9 @@ def _convert_model(model, dialect, base_cube, issues): cname = cube_names[ds_name] dim_names_by_cube[cname], inline_sql_by_cube[cname] = ( _resolve_dimension_names(ds, f"Model '{name}': dataset '{ds_name}'")) - members_by_cube[cname] = set(dim_names_by_cube[cname].values()) + # Not every member: only those the `{CUBE.member}` form is required for. + members_by_cube[cname] = _reference_members( + ds, dim_names_by_cube[cname], dialect) pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] joins_by_cube = _build_joins(relationships, cube_names, issues) @@ -161,8 +164,9 @@ def _convert_model(model, dialect, base_cube, issues): for ds_name, ds in datasets.items(): cname = cube_names[ds_name] cube = _build_cube(ds, cname, dim_names_by_cube[cname], - inline_sql_by_cube[cname], joins_by_cube.get(cname), - measures_by_cube.get(cname), dialect, issues) + inline_sql_by_cube[cname], members_by_cube[cname], + joins_by_cube.get(cname), measures_by_cube.get(cname), + dialect, issues) path = stashed_paths.get(cname) or cube_file(cname) files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) @@ -241,8 +245,8 @@ def _ordered(obj, order): # --- cubes ---------------------------------------------------------------------- -def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, - issues): +def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, + dialect, issues): ds_name = ds["name"] scope = f"dataset '{ds_name}'" stash = read_stash(ds) @@ -272,7 +276,7 @@ def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, "so this cube-level value has no effect in Cube") dimensions, by_name_scalar, by_column = _build_dimensions( - ds, cname, dim_names, inline_sql, dialect, issues) + ds, cname, dim_names, inline_sql, ref_members, dialect, issues) # Resolve each `primary_key` entry to the dimension Cube should mark. A # dimension only qualifies when it is *scalar* -- backed by a single source # column -- because `primary_key: true` in Cube declares that dimension's own @@ -327,6 +331,26 @@ def _build_cube(ds, cname, dim_names, inline_sql, joins, measures, dialect, return _ordered(cube, _CUBE_KEY_ORDER) +def _reference_members(ds, dim_names, dialect): + """Members that must be addressed as `{CUBE.member}` rather than `{CUBE}.column`. + + Only a member whose expression is something other than its own same-named column + needs the reference form, because that form makes Cube inline the member's SQL. A + plain member is identical either way, and the raw-column form is what survives a + round trip without stashing the spelling. + """ + needed = set() + for field in (ds.get("fields") or []): + fname = field.get("name") + dname = dim_names.get(fname) + if not dname: + continue + expr = pick_expression(field.get("expression"), dialect) + if expr is None or not is_simple_identifier(expr) or expr.strip() != dname: + needed.add(dname) + return needed + + def _resolve_dimension_names(ds, scope): """Map each of a dataset's fields to the Cube dimension name it becomes. @@ -388,7 +412,8 @@ def _resolve_dimension_names(ds, scope): return names, inline_sql -def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): +def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, + issues): """Build a cube's dimensions from an Ossie dataset's fields. Returns (dimensions, by_name_scalar, by_column) -- the two maps are what @@ -436,8 +461,7 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): dim["sql"] = stash["sql"] else: dim["sql"] = ossie_expr_to_cube_sql( - expr, cname, set(dim_names.values()), (), - inline_sql={cname: inline_sql}) + expr, cname, ref_members, (), inline_sql={cname: inline_sql}) dim["type"] = _dimension_type(field, stash, f"{ds_name}.{fname}", issues) if field.get("label"): dim["title"] = field["label"] @@ -447,6 +471,14 @@ def _build_dimensions(ds, cname, dim_names, inline_sql, dialect, issues): foreign = foreign_vendor_extensions(field) if foreign: parked["custom_extensions"] = foreign + # Cube's `type` is coarser than Ossie's `datatype` (Integer/Decimal/Float all + # become `number`), so the precise one is parked whenever importing would not + # recover it. `meta.ossie` is Cube-side, so this costs the Ossie document + # nothing -- unlike a custom_extension, which every other spoke would warn + # about and discard. + dt = field.get("datatype") + if dt and DEFAULT_DATATYPE_FOR_CUBE_TYPE.get(dim["type"]) != dt: + parked["datatype"] = dt extras = {k: v for k, v in stash.items() if k not in ("sql", "type", "meta")} meta = _build_meta(field.get("ai_context"), stash.get("meta"), parked) if meta: @@ -499,8 +531,7 @@ def _unique_pk_dimension_name(entry, taken): def _dimension_type(field, stash, scope, issues): """Choose the Cube `type`, which every dimension must declare.""" if "type" in stash: - # Cube collapses Integer/Decimal/Float into `number`, so import parks the - # original rather than asserting an Ossie datatype; restore it here. + # An older stash from before datatypes were mapped natively. return stash["type"] datatype = field.get("datatype") explicit_is_time = (field.get("dimension") or {}).get("is_time") diff --git a/converters/cube/tests/fixtures/fixtureA_ossie.yaml b/converters/cube/tests/fixtures/fixtureA_ossie.yaml index 8247d6c2..96efcf5d 100644 --- a/converters/cube/tests/fixtures/fixtureA_ossie.yaml +++ b/converters/cube/tests/fixtures/fixtureA_ossie.yaml @@ -43,17 +43,13 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: id - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' + datatype: Decimal - name: user_id expression: dialects: - dialect: ANSI_SQL expression: user_id - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' + datatype: Decimal - name: status expression: dialects: @@ -88,9 +84,7 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: id - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' + datatype: Decimal - name: city expression: dialects: @@ -179,7 +173,7 @@ semantic_model: datatype: Integer custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "cube": "users", "sql": "{CUBE}.city"}' + data: '{"_v": 1, "cube": "users"}' custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "views": {"sales": {"name": "sales", "cubes": [{"join_path": diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml index 48f2b70f..45eb7c44 100644 --- a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml @@ -39,16 +39,16 @@ cubes: - POS data joins: - name: date_dim - sql: '{CUBE}.ss_sold_date_sk = {date_dim.d_date_sk}' + sql: '{CUBE}.ss_sold_date_sk = {date_dim}.d_date_sk' relationship: many_to_one - name: customer - sql: '{CUBE}.ss_customer_sk = {customer.c_customer_sk}' + sql: '{CUBE}.ss_customer_sk = {customer}.c_customer_sk' relationship: many_to_one - name: item - sql: '{CUBE}.ss_item_sk = {item.i_item_sk}' + sql: '{CUBE}.ss_item_sk = {item}.i_item_sk' relationship: many_to_one - name: store - sql: '{CUBE}.ss_store_sk = {store.s_store_sk}' + sql: '{CUBE}.ss_store_sk = {store}.s_store_sk' relationship: many_to_one dimensions: - name: ss_sold_date_sk @@ -147,7 +147,7 @@ cubes: public: false measures: - name: total_sales - sql: '{CUBE.ss_ext_sales_price}' + sql: '{CUBE}.ss_ext_sales_price' type: sum description: Total sales revenue across all transactions meta: @@ -159,7 +159,7 @@ cubes: - gross sales - sales amount - name: total_profit - sql: '{CUBE.ss_net_profit}' + sql: '{CUBE}.ss_net_profit' type: sum description: Total net profit from store sales meta: @@ -171,7 +171,7 @@ cubes: - total earnings - profit - name: customer_lifetime_value - sql: SUM({CUBE.ss_ext_sales_price}) / COUNT(DISTINCT {customer.c_customer_sk}) + sql: SUM({CUBE}.ss_ext_sales_price) / COUNT(DISTINCT {customer.c_customer_sk}) type: number description: Average lifetime sales value per customer meta: @@ -184,7 +184,7 @@ cubes: - customer value - lifetime revenue - name: sales_by_brand - sql: '{CUBE.ss_ext_sales_price}' + sql: '{CUBE}.ss_ext_sales_price' type: sum description: Total sales by brand (requires grouping by item.i_brand) meta: @@ -196,7 +196,7 @@ cubes: - brand performance - brand revenue - name: store_productivity - sql: SUM({CUBE.ss_ext_sales_price}) / NULLIF(SUM({store.s_number_employees}), + sql: SUM({CUBE}.ss_ext_sales_price) / NULLIF(SUM({store.s_number_employees}), 0) type: number description: Sales per employee across stores diff --git a/converters/cube/tests/fixtures/tpcds_ossie.yaml b/converters/cube/tests/fixtures/tpcds_ossie.yaml index 0e6ad634..0b323fb9 100644 --- a/converters/cube/tests/fixtures/tpcds_ossie.yaml +++ b/converters/cube/tests/fixtures/tpcds_ossie.yaml @@ -34,9 +34,6 @@ semantic_model: - ss_sold_date_sk to_columns: - d_date_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "{CUBE}.ss_sold_date_sk = {date_dim.d_date_sk}"}' - name: store_sales_to_customer from: store_sales to: customer @@ -44,9 +41,6 @@ semantic_model: - ss_customer_sk to_columns: - c_customer_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "{CUBE}.ss_customer_sk = {customer.c_customer_sk}"}' - name: store_sales_to_item from: store_sales to: item @@ -54,9 +48,6 @@ semantic_model: - ss_item_sk to_columns: - i_item_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "{CUBE}.ss_item_sk = {item.i_item_sk}"}' - name: store_sales_to_store from: store_sales to: store @@ -64,9 +55,6 @@ semantic_model: - ss_store_sk to_columns: - s_store_sk - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "sql": "{CUBE}.ss_store_sk = {store.s_store_sk}"}' datasets: - name: store_sales source: tpcds.public.store_sales @@ -86,105 +74,89 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: ss_sold_date_sk + datatype: Decimal description: Foreign key to date dimension ai_context: synonyms: - sale date - transaction date - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_item_sk expression: dialects: - dialect: ANSI_SQL expression: ss_item_sk + datatype: Decimal description: Foreign key to item dimension ai_context: synonyms: - product - item - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_customer_sk expression: dialects: - dialect: ANSI_SQL expression: ss_customer_sk + datatype: Decimal description: Foreign key to customer dimension ai_context: synonyms: - customer - buyer - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_store_sk expression: dialects: - dialect: ANSI_SQL expression: ss_store_sk + datatype: Decimal description: Foreign key to store dimension ai_context: synonyms: - store - location - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_quantity expression: dialects: - dialect: ANSI_SQL expression: ss_quantity + datatype: Decimal description: Quantity of items sold ai_context: synonyms: - units sold - quantity - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_sales_price expression: dialects: - dialect: ANSI_SQL expression: ss_sales_price + datatype: Decimal description: Sales price per unit ai_context: synonyms: - unit price - price - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_ext_sales_price expression: dialects: - dialect: ANSI_SQL expression: ss_ext_sales_price + datatype: Decimal description: Extended sales price (quantity * price) ai_context: synonyms: - total price - line total - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_net_profit expression: dialects: - dialect: ANSI_SQL expression: ss_net_profit + datatype: Decimal description: Net profit from the sale ai_context: synonyms: - profit - margin - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: ss_ticket_number expression: dialects: @@ -213,10 +185,8 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: d_date_sk + datatype: Decimal description: Surrogate key for date - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: d_date expression: dialects: @@ -235,13 +205,11 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: d_year + datatype: Decimal description: Year ai_context: synonyms: - year - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: d_quarter_name expression: dialects: @@ -285,10 +253,8 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: c_customer_sk + datatype: Decimal description: Surrogate key for customer - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: c_customer_id expression: dialects: @@ -354,10 +320,8 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: i_item_sk + datatype: Decimal description: Surrogate key for item - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: i_item_id expression: dialects: @@ -408,14 +372,12 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: i_current_price + datatype: Decimal description: Current price of the item ai_context: synonyms: - price - list price - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' primary_key: - i_item_sk - name: store @@ -434,10 +396,8 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: s_store_sk + datatype: Decimal description: Surrogate key for store - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' - name: s_store_id expression: dialects: @@ -487,14 +447,12 @@ semantic_model: dialects: - dialect: ANSI_SQL expression: s_number_employees + datatype: Decimal description: Number of employees at the store ai_context: synonyms: - employee count - staff size - custom_extensions: - - vendor_name: CUBE - data: '{"_v": 1, "type": "number"}' primary_key: - s_store_sk metrics: @@ -511,7 +469,7 @@ semantic_model: - sales amount custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_ext_sales_price}"}' + data: '{"_v": 1, "cube": "store_sales"}' - name: total_profit expression: dialects: @@ -525,7 +483,7 @@ semantic_model: - profit custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_net_profit}"}' + data: '{"_v": 1, "cube": "store_sales"}' - name: customer_lifetime_value expression: dialects: @@ -541,7 +499,7 @@ semantic_model: custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "cube": "store_sales", "measure": {"name": "customer_lifetime_value", - "sql": "SUM({CUBE.ss_ext_sales_price}) / COUNT(DISTINCT {customer.c_customer_sk})", + "sql": "SUM({CUBE}.ss_ext_sales_price) / COUNT(DISTINCT {customer.c_customer_sk})", "type": "number"}}' - name: sales_by_brand expression: @@ -556,7 +514,7 @@ semantic_model: - brand revenue custom_extensions: - vendor_name: CUBE - data: '{"_v": 1, "cube": "store_sales", "sql": "{CUBE.ss_ext_sales_price}"}' + data: '{"_v": 1, "cube": "store_sales"}' - name: store_productivity expression: dialects: @@ -572,7 +530,7 @@ semantic_model: custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "cube": "store_sales", "measure": {"name": "store_productivity", - "sql": "SUM({CUBE.ss_ext_sales_price}) / NULLIF(SUM({store.s_number_employees}), + "sql": "SUM({CUBE}.ss_ext_sales_price) / NULLIF(SUM({store.s_number_employees}), 0)", "type": "number"}}' custom_extensions: - vendor_name: CUBE diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index fcf2335e..aa9c7874 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -111,13 +111,15 @@ def test_dimension_types_map_to_datatypes(model_a): assert fields["created_at"]["dimension"]["is_time"] is True -def test_number_dimension_asserts_no_datatype(model_a): - """Cube collapses Integer/Decimal/Float into `number`, so the converter omits - `datatype` rather than assert a precision the model does not carry.""" +def test_number_dimension_maps_to_a_native_datatype(model_a): + """Cube collapses Integer/Decimal/Float into `number`, so no mapping back is + exact. `Decimal` is asserted anyway, because a downstream converter can act on it + -- where omitting it and stashing Cube's `type` in a custom_extension gave every + other spoke a warning and nothing else.""" model, _ = model_a fields = by_name(by_name(model["datasets"])["orders"]["fields"]) - assert "datatype" not in fields["id"] - assert stash_of(fields["id"])["type"] == "number" + assert fields["id"]["datatype"] == "Decimal" + assert stash_of(fields["id"]) == {} def test_dimension_title_becomes_label_and_ai_context_maps(model_a): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index e7e92beb..eebf5d6b 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -878,7 +878,7 @@ def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): # `{users}.lat` names the cube explicitly, since `{CUBE}` here would mean # `orders`. `{CUBE.amount}` stays a member reference because `amount` is a # declared field of the cube the measure lands on. - assert measures[0]["sql"] == "AVG({users}.lat) - MIN({CUBE.amount})" + assert measures[0]["sql"] == "AVG({users}.lat) - MIN({CUBE}.amount)" assert measures[0]["type"] == "number" diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index f2ee960b..b1e128e7 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -349,14 +349,14 @@ def _metric(name, expr): @pytest.mark.parametrize("expr,expected", [ - ("SUM(orders.amount)", {"type": "sum", "sql": "{CUBE.amount}"}), - ("AVG(orders.amount)", {"type": "avg", "sql": "{CUBE.amount}"}), - ("MIN(orders.amount)", {"type": "min", "sql": "{CUBE.amount}"}), - ("MAX(orders.amount)", {"type": "max", "sql": "{CUBE.amount}"}), + ("SUM(orders.amount)", {"type": "sum", "sql": "{CUBE}.amount"}), + ("AVG(orders.amount)", {"type": "avg", "sql": "{CUBE}.amount"}), + ("MIN(orders.amount)", {"type": "min", "sql": "{CUBE}.amount"}), + ("MAX(orders.amount)", {"type": "max", "sql": "{CUBE}.amount"}), ("COUNT(DISTINCT orders.amount)", - {"type": "count_distinct", "sql": "{CUBE.amount}"}), + {"type": "count_distinct", "sql": "{CUBE}.amount"}), ("APPROX_COUNT_DISTINCT(orders.amount)", - {"type": "count_distinct_approx", "sql": "{CUBE.amount}"}), + {"type": "count_distinct_approx", "sql": "{CUBE}.amount"}), ]) def test_aggregate_expressions_become_structured_measures(expr, expected): files, _ = convert_ossie_to_cube(_ossie(_ORDERS, metrics=_metric("m", expr))) @@ -389,7 +389,7 @@ def test_ratio_becomes_a_calculated_measure(): _metric("aov", "SUM(orders.amount) / COUNT(DISTINCT users.id)"))) measure = _cubes(files)["orders"]["measures"][0] assert measure["type"] == "number" - assert measure["sql"] == "SUM({CUBE.amount}) / COUNT(DISTINCT {users.id})" + assert measure["sql"] == "SUM({CUBE}.amount) / COUNT(DISTINCT {users.id})" assert any("spans several datasets" in i.detail for i in issues.of_type(IssueType.APPROXIMATED)) From 1de9e118eb97f9dd7b034b768f183fc6c6c081b7 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 16:27:26 +0500 Subject: [PATCH 20/25] Report a source that other Ossie converters will reject Found by running Cube -> Ossie -> every other spoke. A Cube `sql_table` of `public.orders` is perfectly ordinary and converts cleanly here, but three converters reject the resulting Ossie source outright: databricks source 'public.orders' must be a 3-part catalog.schema.table gsf source must resolve to database.schema.table snowflake must be a fully qualified db.schema.table or a subquery So a model can pass every test in this converter and still be unable to reach half the ecosystem. Import now reports SOURCE_NOT_FULLY_QUALIFIED, naming the converters and what to change, at the point the Ossie document is produced rather than three hops later. Deliberately a report and not a fix: inventing a catalog name would be guessing at the user's warehouse. Cube's own `sql_table` is legitimately one- or two-part, so this is a portability limit of the hub rather than a defect on either side -- which is why it gets its own issue type instead of being folded into the loss categories. Dots inside quoted identifiers are not path separators, so `"My.Catalog".public.orders` counts as three parts, not four. 261 tests, 96% line coverage. `git diff --check` clean. Co-Authored-By: Claude Opus 5 --- converters/cube/README.md | 27 ++++++++++++++ converters/cube/src/ossie_cube/_common.py | 21 +++++++++++ .../cube/src/ossie_cube/converter_issues.py | 6 ++++ converters/cube/src/ossie_cube/cube_to_osi.py | 14 ++++++++ converters/cube/tests/test_edge_cases.py | 35 +++++++++++++++++++ 5 files changed, 103 insertions(+) diff --git a/converters/cube/README.md b/converters/cube/README.md index ecb19241..4ca68bac 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -220,6 +220,32 @@ AVG(users.home_latitude) - MIN(orders.amt) -> sql: AVG({users}.lat) - MIN({CUB One documented normalization follows: after a round trip such a metric names the column the half actually reads (`users.lat`) rather than the Ossie-only field name (`users.home_latitude`). Same reference, and it is the form Cube can express. +## Onward conversion + +Ossie is a hub, so the useful question is not only whether `Cube → Ossie → Cube` +round-trips but whether the Ossie model then reaches the other spokes. Two things +matter in practice. + +**Keep Cube-only detail out of `custom_extensions`.** Converters that do not read +foreign extensions warn about and discard every one, so anything placed there is +noise to them. This converter therefore stashes only what is genuinely Cube-specific +— segments, pre-aggregations, hierarchies, view curation, geo reconstruction — and +maps everything else natively. On the TPC-DS model that is 7 stash entries rather +than 41, and 2 Databricks warnings rather than 32. + +**Qualify your `sql_table`.** Cube accepts `orders` or `public.orders`, but the +Databricks, Snowflake and NVIDIA GSF converters all require a three-part +`catalog.schema.table` and reject anything shorter: + +``` +Error: Dataset 'orders': source 'public.orders' must be a 3-part catalog.schema.table +Error: Dataset 'orders' source must resolve to database.schema.table +Error: Source 'public.orders' must be a fully qualified db.schema.table or a subquery +``` + +Import reports this as `SOURCE_NOT_FULLY_QUALIFIED` rather than guessing a catalog +name, so it surfaces where the Ossie document is produced instead of three hops later. + ## Conversion issues `convert_cube_to_ossie` returns `(yaml, IssueLog)`. Each issue carries a type, the @@ -233,6 +259,7 @@ element it concerns, and a detail string. | `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | | `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | | `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | +| `SOURCE_NOT_FULLY_QUALIFIED` | A `sql_table` shorter than `catalog.schema.table`. Valid Cube and nothing is lost, but the Databricks, Snowflake and NVIDIA GSF converters reject such a source, so the model cannot convert onward — see [Onward conversion](#onward-conversion) | | `PARKED_IN_META` | Preserved in the stash or under `meta.ossie` — invisible to Cube, but intact through a round trip | | `DROPPED_NO_CUBE_EQUIVALENT` | **Gone from the output.** Cube has nowhere to hold it and it cannot be parked: relationship `ai_context` (a Cube join entry has no `meta`), a `dimension.is_time` role or opt-out that Cube expresses only through `type`, and the second and later `semantic_model` entries | | `APPROXIMATED` | Emitted, but not an exact equivalent: a value Cube requires and Ossie does not carry (so the converter chose one), or a construct rendered in the nearest form Cube has | diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index b25aa2ca..6ca0666a 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,6 +385,27 @@ def repl(m): return out, changed +def source_part_count(source): + """How many identifier parts a dotted dataset `source` has, or None for a query. + + Dots inside double quotes or backticks belong to a quoted identifier, not to the + path -- `"My.Catalog".public.t` is three parts, not four. + """ + s = str(source).strip() + if re.match(r"(?i)(select|with)\b", s): + return None + parts, quote = 1, None + for ch in s: + if quote: + if ch == quote: + quote = None + elif ch in '"`': + quote = ch + elif ch == ".": + parts += 1 + return parts + + def sql_is_reversible(sql, plain_members=(), own_cube=None): """True if translating this Cube SQL to Ossie and back reproduces it. diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index 93df8842..f0306d2f 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -62,6 +62,12 @@ class IssueType(Enum): # An Ossie field or metric with no usable expression dialect (export). NO_USABLE_DIALECT = "NO_USABLE_DIALECT" + # A dataset `source` that is a valid Cube `sql_table` but not a three-part + # `catalog.schema.table`. Nothing is lost and Cube is happy, but several other + # Ossie converters reject such a source outright, so the model will not travel + # past this hub. Reported so that is discovered here rather than downstream. + SOURCE_NOT_FULLY_QUALIFIED = "SOURCE_NOT_FULLY_QUALIFIED" + # An Ossie construct Cube has no slot for, parked under `meta.ossie` -- so the # value survives the round trip even though Cube itself cannot read it. PARKED_IN_META = "PARKED_IN_META" diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index c2e5f0f1..c5bb38f3 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -56,6 +56,7 @@ require_str, snake, snake_keys, + source_part_count, sql_is_reversible, view_file, read_stash, @@ -469,6 +470,19 @@ def _convert_cube(cname, cube, extra_joins, extra_measures, issues): stash = {} ds["source"] = join_source(cube, cname) + parts = source_part_count(ds["source"]) + if parts is not None and parts < 3: + # Cube accepts a one- or two-part `sql_table`, but the Ossie spec describes + # `source` as `database.schema.table` and the Databricks, Snowflake and NVIDIA + # GSF converters all reject anything shorter -- so a model that converts + # cleanly here still cannot reach them. Better to say so at the point the + # Ossie document is produced than to have it fail three hops later. + ds_scope = f"cube '{cname}'" + issues.add(IssueType.SOURCE_NOT_FULLY_QUALIFIED, ds_scope, + f"source '{ds['source']}' has {parts} part(s); several Ossie " + f"converters (Databricks, Snowflake, NVIDIA GSF) require a " + f"3-part catalog.schema.table, so qualify the cube's `sql_table` " + f"if the model needs to convert onward") if cube.get("description"): ds["description"] = cube["description"] diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index eebf5d6b..0577528b 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -291,6 +291,41 @@ def test_field_and_metric_foreign_extensions_survive_the_round_trip(): assert metric["custom_extensions"][0]["vendor_name"] == "CUBE" +@pytest.mark.parametrize("sql_table,parts,warns", [ + ("orders", 1, True), + ("public.orders", 2, True), + ("tpcds.public.orders", 3, False), + ('"My.Catalog".public.orders', 3, False), # dots inside quotes are not parts + ("a.b.c.d", 4, False), +]) +def test_a_source_that_other_converters_reject_is_reported(sql_table, parts, warns): + """Cube is happy with a one- or two-part `sql_table`, but the Databricks, + Snowflake and NVIDIA GSF converters all reject a source shorter than + `catalog.schema.table` -- so a model that converts cleanly here still cannot + travel. Reported at the point the Ossie document is produced, rather than being + discovered three hops later.""" + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + # Single-quoted so a value containing double quotes stays one YAML scalar. + f" sql_table: '{sql_table}'\n")) + _, issues = convert_cube_to_ossie(files) + reported = issues.of_type(IssueType.SOURCE_NOT_FULLY_QUALIFIED) + assert bool(reported) is warns + if warns: + assert f"{parts} part(s)" in reported[0].detail + + +def test_a_sql_defined_cube_is_not_reported_as_unqualified(): + """A `sql:` cube is a query, not a table path, and every converter accepts one.""" + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql: SELECT * FROM public.orders\n")) + _, issues = convert_cube_to_ossie(files) + assert not issues.of_type(IssueType.SOURCE_NOT_FULLY_QUALIFIED) + + # --- join orientation, both ways ------------------------------------------------ _ONE_TO_MANY = _files(m=( From de748f15444486fd764e6a7cf8186d5000eccba2 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 16:46:17 +0500 Subject: [PATCH 21/25] Split a composite metric into one measure per aggregate An Ossie metric such as SUM(store_sales.amount) / COUNT(DISTINCT customer.id) used to become a single Cube measure of `type: number` holding the whole expression. Cube corrects row multiplication per measure, keyed on the cube the measure sits on, so one calculated measure gets one correction for an expression whose aggregates read different cubes. Export now emits one `public: false` measure per aggregate, each declared on the cube its own operand reads, plus the public measure referencing them. Each aggregate is then corrected on its own terms. Parts carry `meta.ossie.part_of`, and import skips them and inlines their SQL back through the references, so the original expression is recovered exactly. Locating the aggregates uses sqlglot rather than a regex, since an expression can nest them -- SUM(x) / NULLIF(SUM(y), 0). It is already a runtime dependency of the dbt and NVIDIA GSF converters for the same purpose. Also drops the parentheses import used to add around every inlined measure reference: a lone aggregate is one term already, and keeping them off is what makes the round trip exact. --- converters/cube/README.md | 22 ++- converters/cube/pyproject.toml | 4 + converters/cube/src/ossie_cube/cube_to_osi.py | 23 ++- converters/cube/src/ossie_cube/expressions.py | 176 ++++++++++++++++++ converters/cube/src/ossie_cube/osi_to_cube.py | 78 +++++++- .../cube/tests/fixtures/fixtureA_ossie.yaml | 2 +- converters/cube/tests/test_cube_to_osi.py | 4 +- converters/cube/tests/test_edge_cases.py | 47 +++-- converters/cube/tests/test_osi_to_cube.py | 51 ++++- converters/cube/uv.lock | 15 +- 10 files changed, 390 insertions(+), 32 deletions(-) create mode 100644 converters/cube/src/ossie_cube/expressions.py diff --git a/converters/cube/README.md b/converters/cube/README.md index 4ca68bac..71897937 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -55,7 +55,9 @@ pip install apache-ossie-cube # once published to PyPI pip install -e . ``` -The only runtime dependency is `PyYAML`. Python 3.11+. +Runtime dependencies are `PyYAML` and `sqlglot` (already a runtime dependency of +the dbt and NVIDIA GSF converters, used here to locate the aggregate calls inside a +composite metric). Python 3.11+. ## Usage @@ -125,7 +127,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot; parked rather than dropped. | | field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | | `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | -| `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back except for `number`, where it **omits `datatype`** -- Cube collapses three Ossie types into one, and the spec says to omit rather than assert. The original `type` is stashed. | +| `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back, choosing `Decimal` for `number` -- Cube collapses three Ossie types into one, so any single answer is a guess, and a stated datatype is what another converter can act on. Export parks the exact one in `meta.ossie.datatype`, which import prefers when present, so `Integer` and `Float` still survive a round trip. | | `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. | | `field.label` / `description` | dimension `title` / `description` | | | `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | @@ -137,6 +139,7 @@ to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). | `COUNT(DISTINCT x)` | `type: count_distinct` | | | `APPROX_COUNT_DISTINCT(x)` | `type: count_distinct_approx` | Cube resolves the warehouse-specific function itself. | | `COUNT(DISTINCT )` | bare `type: count` | See [Fan-out](#fan-out) -- the primary key is load-bearing here. | +| several aggregates in one expression | one `public: false` measure per aggregate + a `type: number` measure referencing them | Each part is declared on the cube its own operand reads, so Cube corrects row multiplication per aggregate rather than once for the whole expression. The parts carry `meta.ossie.part_of`, and import skips them and inlines their SQL back through the references -- recovering the original expression exactly. | | anything else | `type: number` (calculated) | A `{other_measure}` reference is **inlined**, because that is what Cube itself does; Ossie has no metric-to-metric reference. | | — | measure `filters` | Folded into `CASE WHEN … THEN … END` inside the aggregate, exactly as Cube's own `applyMeasureFilters` renders it. | | `metric.datatype` | — | Import emits `Integer` for the count family, whose result type Cube does know, and omits it otherwise. | @@ -191,6 +194,21 @@ Because a bare `count` maps through the primary key, a cube carrying one **must* declare `primary_key: true` on a dimension; its absence is an error, not a different number. +Going the other way, an Ossie metric combining several aggregates is **decomposed** +rather than emitted as one calculated measure, so Cube's correction applies to each +aggregate on its own cube: + +```yaml +# Ossie # Cube +SUM(store_sales.amount) store_sales: clv_part_1 (sum, public: false) + / COUNT(DISTINCT customer.id) customer: clv_part_2 (count, public: false) + store_sales: clv = {CUBE.clv_part_1} + / {customer.clv_part_2} +``` + +A single aggregate reading two datasets cannot be split this way and still lands on +one cube. + > Ossie has no additivity or grain declaration to record this properly -- dbt's > `non_additive_dimension` is the nearest precedent, and this repo's dbt converter > already loses the same information. Worth raising on `dev@`. diff --git a/converters/cube/pyproject.toml b/converters/cube/pyproject.toml index a78e12b8..72a087d4 100644 --- a/converters/cube/pyproject.toml +++ b/converters/cube/pyproject.toml @@ -38,6 +38,10 @@ classifiers = [ ] dependencies = [ "PyYAML>=6.0", + # Expression handling: locating the aggregate calls inside a composite metric so + # each can become its own Cube measure. Already a runtime dependency of the dbt + # and NVIDIA GSF converters, which use it for the same purpose. + "sqlglot>=20.0", ] [dependency-groups] diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index c5bb38f3..136cf114 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -63,6 +63,7 @@ write_stash, ) from .converter_issues import IssueLog, IssueType +from .expressions import has_top_level_operator # Cube keys the converter maps natively at the cube level; everything else is # stashed verbatim in the dataset's `cube_extras` and restored on export. @@ -903,7 +904,10 @@ def _inline(self, body, cname, stack): raise ConversionError( f"measure '{cname}': references '{target_cube}.{target_name}', " f"which has no static Ossie form") - return f"({inner})" + # A lone `SUM(x)` needs no parentheses; only a term with its own top-level + # operators does. Keeping them off means a decomposed metric inlines back to + # exactly the expression it was split from. + return f"({inner})" if has_top_level_operator(inner) else inner def _operand(self, cname, sql, stack): """Translate an aggregate's operand into an Ossie reference. @@ -919,6 +923,12 @@ def _operand(self, cname, sql, stack): return translated +def _is_generated_part(measure): + """True for a `public: false` measure a previous export created to hold one + aggregate of a composite metric (marked `meta.ossie.part_of`).""" + return bool(((measure.get("meta") or {}).get("ossie") or {}).get("part_of")) + + def _convert_measures(cubes, pk_by_cube, fanned_out, issues): """Hoist every cube's measures into Ossie model-level metrics. @@ -935,8 +945,9 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): resolver = _MeasureResolver(cubes, pk_by_cube, issues) counts = {} - for (_, mname) in resolver.measures(): - counts[mname] = counts.get(mname, 0) + 1 + for (cname, mname), measure in resolver.measures().items(): + if not _is_generated_part(measure): + counts[mname] = counts.get(mname, 0) + 1 metrics = [] extra_measures = {} @@ -946,6 +957,12 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): _as_named_list(cube.get("measures"), f"cube '{cname}' measures")): mname = measure["name"] + if _is_generated_part(measure): + # Emitted by a previous export to split a composite metric across + # cubes. It has no Ossie metric of its own -- the public measure's + # references inline back to the whole expression -- and export + # regenerates it, so it is not stashed either. + continue metric_name = mname if counts[mname] == 1 else f"{cname}__{mname}" if metric_name in seen: raise ConversionError( diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py new file mode 100644 index 00000000..68243c0f --- /dev/null +++ b/converters/cube/src/ossie_cube/expressions.py @@ -0,0 +1,176 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Reading the structure of an Ossie metric expression. + +Cube expects a measure to *be* an aggregation -- `type: sum` over a column -- and +falls back to a calculated `type: number` measure whose sql carries the whole +aggregate. A composite Ossie metric such as + + SUM(store_sales.amount) / COUNT(DISTINCT customer.id) + +can be emitted either way, and the difference matters: as one calculated measure +Cube sees a single opaque expression, whereas as two `public: false` measures on +their own cubes plus a ratio referencing them, **Cube applies its row-multiplication +correction to each aggregate independently**. So decomposition is a correctness +improvement for cross-dataset metrics, not a formatting choice. + +Locating the aggregate calls is done with sqlglot rather than a regex, since an +expression can nest them (`SUM(x) / NULLIF(SUM(y), 0)`) and string matching cannot +tell a top-level call from one inside another argument. sqlglot is already a runtime +dependency of the dbt and NVIDIA GSF converters for the same purpose. +""" + +import sqlglot +import sqlglot.expressions as exp + +# sqlglot node types for the aggregates this converter maps to a Cube measure type. +# `Count` covers COUNT / COUNT(DISTINCT x); ApproxDistinct covers +# APPROX_COUNT_DISTINCT. +_AGGREGATE_NODES = ( + exp.Sum, exp.Avg, exp.Min, exp.Max, exp.Count, exp.ApproxDistinct, +) + + +def parse(expr): + """Parse an Ossie expression, or None when sqlglot cannot. + + An unparseable expression is not an error: the converter falls back to treating + it as one opaque calculated measure, which is what it did for everything before. + """ + try: + return sqlglot.parse_one(str(expr).strip()) + except Exception: + return None + + +def is_single_aggregate(expr): + """True if the whole expression is exactly one aggregate call. + + Those already map to a structured Cube measure (`type: sum` + `sql`), so they + are never decomposed. + """ + tree = parse(expr) + return tree is not None and isinstance(tree, _AGGREGATE_NODES) + + +# The aggregate call names this converter maps to a Cube measure type. Scanned for +# in the source text: sqlglot renames some when it renders (`APPROX_COUNT_DISTINCT` +# comes back as `APPROX_DISTINCT`), and two calls of the same name render +# identically, so node text cannot be used to find them in the original string. +_AGGREGATE_NAMES = ( + "APPROX_COUNT_DISTINCT", "APPROX_DISTINCT", + "COUNT", "SUM", "AVG", "MIN", "MAX", +) + + +def aggregate_spans(expr): + """The outermost aggregate calls in `expr`, as (start, end) offsets. + + Offsets index the original string so a caller can substitute each span in place. + That matters because the surrounding text may carry Cube `{...}` references, + which sqlglot would not reproduce verbatim if the expression were re-rendered. + + Spans are found by scanning for an aggregate name followed by a balanced + parenthesis group, then confirmed with sqlglot -- which is also what rules out a + malformed expression. Nesting is resolved on the offsets themselves: a span + inside another span is not returned, so `SUM(x) / NULLIF(SUM(y), 0)` gives two + and `SUM(SUM(x))` gives one. Returns [] when the expression does not parse, or is + itself a single aggregate needing no decomposition. + """ + text = str(expr) + if parse(text) is None or is_single_aggregate(text): + return [] + + candidates = [] + upper = text.upper() + for name in _AGGREGATE_NAMES: + at = 0 + while True: + at = upper.find(name, at) + if at < 0: + break + start, after = at, at + len(name) + at = after + # A call, not part of a longer identifier: boundary before, `(` after. + if start and (text[start - 1].isalnum() or text[start - 1] == "_"): + continue + probe = after + while probe < len(text) and text[probe].isspace(): + probe += 1 + if probe >= len(text) or text[probe] != "(": + continue + close = _match_paren(text, probe) + if close is None: + continue + end = close + 1 + # Confirm the slice really is an aggregate and not, say, a UDF that + # happens to share a prefix. + node = parse(text[start:end]) + if isinstance(node, _AGGREGATE_NODES): + candidates.append((start, end)) + + # Drop any span contained within another: only the outermost becomes a measure. + candidates.sort() + out = [] + for start, end in candidates: + if any(s <= start and end <= e for s, e in out): + continue + out.append((start, end)) + return out + + +def _match_paren(text, open_at): + """Index of the `)` closing the `(` at `open_at`, honouring quotes.""" + depth, quote = 0, None + for i in range(open_at, len(text)): + ch = text[i] + if quote: + if ch == quote: + quote = None + elif ch in "'\"": + quote = ch + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return i + return None + + +def has_top_level_operator(expr): + """True if `expr` is not a single self-contained term. + + Used to decide whether inlining it back into a larger expression needs + parentheses: a lone `SUM(x)` does not, `SUM(x) / 2` does. + """ + depth, quote = 0, None + for ch in str(expr): + if quote: + if ch == quote: + quote = None + elif ch in "'\"": + quote = ch + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + elif depth == 0 and (ch in "+-*/%<>=|&" or ch.isspace()): + # Whitespace at depth 0 also implies structure (`CASE WHEN ...`). + return True + return False diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index beaee615..889b3fce 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -36,6 +36,7 @@ from ._common import ( DATATYPE_TO_DIM_TYPE, + DOTTED_REF_RE, DEFAULT_DATATYPE_FOR_CUBE_TYPE, OSSIE_FUNC_TO_AGG, OSSIE_VERSION, @@ -58,6 +59,7 @@ view_file, ) from .converter_issues import IssueLog, IssueType +from .expressions import aggregate_spans # An aggregate call the exporter can turn back into a structured Cube measure. _AGG_CALL_RE = re.compile( @@ -666,15 +668,83 @@ def resolve_base(): } target = stash.get("cube") or ( next(iter(referenced)) if len(referenced) == 1 else resolve_base()) - measure = _measure_from_expression( - expr, target, mname, stash, members_by_cube.get(target, set()), - inline_sql_by_cube, pk_by_cube.get(target, []), sanitized, scope, - issues) + + spans = [] if stash.get("sql") else aggregate_spans(expr) + if len(spans) > 1: + # A composite metric: give each aggregate its own measure on the cube its + # operand belongs to, and let the public measure reference them. Cube then + # applies its row-multiplication correction per aggregate instead of + # seeing one opaque expression -- see _decompose_measure. + public_sql = _decompose_measure( + expr, spans, mname, target, measures_by_cube, members_by_cube, + inline_sql_by_cube, pk_by_cube, sanitized, name, scope, issues) + measure = {"name": mname, "sql": public_sql, "type": "number"} + else: + measure = _measure_from_expression( + expr, target, mname, stash, members_by_cube.get(target, set()), + inline_sql_by_cube, pk_by_cube.get(target, []), sanitized, scope, + issues) _apply_measure_metadata(metric, measure, stash) _place(measures_by_cube, target, measure, name) return measures_by_cube +def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, + members_by_cube, inline_sql_by_cube, pk_by_cube, sanitized, + model_name, scope, issues): + """Emit one `public: false` measure per aggregate; return the sql referencing them. + + Cube corrects for row multiplication per measure, keyed on the cube that measure + sits on. A cross-dataset ratio emitted as a single calculated measure gets one + correction for the whole expression; split into a measure per aggregate, each on + the cube its operand comes from, each aggregate is corrected on its own terms. + That is why this is a correctness change and not a formatting one. + + Each part carries `meta.ossie.part_of` so import knows it is generated and skips + it, recovering the original expression by inlining the references instead. + """ + # A part name has to be free on whichever cube it lands on, and a Cube member name + # is unique across dimensions and measures alike -- so the check is over both, and + # over every cube rather than the one part it happens to land on. + taken = {m["name"].lower() for ms in measures_by_cube.values() for m in ms} + taken |= {n.lower() for ns in members_by_cube.values() for n in ns} + out, cursor, index = [], 0, 0 + for start, end in spans: + piece = expr[start:end] + # Each aggregate lands on the cube its own operand references. + refs = {m.group(1) for m in DOTTED_REF_RE.finditer(piece) + if m.group(1) in sanitized} + part_target = next(iter(refs)) if len(refs) == 1 else fallback + + index += 1 + part_name = f"{mname}_part_{index}" + while part_name.lower() in taken: + index += 1 + part_name = f"{mname}_part_{index}" + taken.add(part_name.lower()) + + part = _measure_from_expression( + piece, part_target, part_name, {}, + members_by_cube.get(part_target, set()), inline_sql_by_cube, + pk_by_cube.get(part_target, []), sanitized, scope, issues) + part["public"] = False + part["meta"] = {"ossie": {"part_of": mname}} + _place(measures_by_cube, part_target, part, model_name) + + out.append(ossie_expr_to_cube_sql( + expr[cursor:start], fallback, members_by_cube.get(fallback, set()), + sanitized, inline_sql=inline_sql_by_cube)) + # `{CUBE.x}` for a part on the same cube as the public measure: an explicit + # name pins the reference to this cube and breaks if it is extended. + qualifier = "CUBE" if part_target == fallback else part_target + out.append("{" + f"{qualifier}.{part_name}" + "}") + cursor = end + out.append(ossie_expr_to_cube_sql( + expr[cursor:], fallback, members_by_cube.get(fallback, set()), sanitized, + inline_sql=inline_sql_by_cube)) + return "".join(out) + + def _place(measures_by_cube, target, measure, model_name): bucket = measures_by_cube.setdefault(target, []) if any(m["name"].lower() == measure["name"].lower() for m in bucket): diff --git a/converters/cube/tests/fixtures/fixtureA_ossie.yaml b/converters/cube/tests/fixtures/fixtureA_ossie.yaml index 96efcf5d..577fd8ed 100644 --- a/converters/cube/tests/fixtures/fixtureA_ossie.yaml +++ b/converters/cube/tests/fixtures/fixtureA_ossie.yaml @@ -151,7 +151,7 @@ semantic_model: expression: dialects: - dialect: ANSI_SQL - expression: (SUM(orders.amount)) / (COUNT(DISTINCT orders.id)) + expression: SUM(orders.amount) / COUNT(DISTINCT orders.id) custom_extensions: - vendor_name: CUBE data: '{"_v": 1, "cube": "orders", "measure": {"name": "avg_order_value", "sql": diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index aa9c7874..a3609e98 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -326,7 +326,9 @@ def test_calculated_measure_inlines_its_measure_references(model_a): aggregate SQL; Ossie has no metric-to-metric reference, so it is inlined.""" model, _ = model_a metric = by_name(model["metrics"])["avg_order_value"] - assert expr_of(metric) == "(SUM(orders.amount)) / (COUNT(DISTINCT orders.id))" + # No redundant parentheses: a lone aggregate is already a single term, so an + # inlined reference reads exactly as the expression it stands for. + assert expr_of(metric) == "SUM(orders.amount) / COUNT(DISTINCT orders.id)" def test_measure_reference_cycle_is_rejected(): diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 0577528b..6cdd1b50 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -893,10 +893,9 @@ def test_a_metric_referencing_a_geo_half_inlines_its_sql(): "longitude": {"sql": "{CUBE}.lon"}}] -def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): - """`{CUBE}` means "the cube this is declared on", so inlining a snippet into - another cube's SQL has to name the original cube explicitly.""" - model = _GEO_MODEL.replace( +def _two_cube_geo_model(expression): + """`_GEO_MODEL` plus an `orders.amount` field, and `expression` as the metric.""" + return _GEO_MODEL.replace( " - name: users\n", " - name: orders\n source: public.orders\n" " fields:\n" " - name: amount\n" @@ -907,14 +906,40 @@ def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): " datatype: Decimal\n" " - name: users\n", 1 ).replace(" expression: AVG(users.home_latitude)\n", - " expression: AVG(users.home_latitude) - MIN(orders.amount)\n") + f" expression: {expression}\n") + + +def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): + """`{CUBE}` means "the cube this is declared on", so inlining a snippet into + another cube's SQL has to name the original cube explicitly. + + One aggregate reading two datasets cannot be decomposed, so it lands on the base + cube and the `users` half travels there with it. + """ + model = _two_cube_geo_model("AVG(users.home_latitude - orders.amount)") + files, _ = convert_ossie_to_cube(model, base_cube="orders") + cube = parse(files["model/cubes/orders.yml"])["cubes"][0] + assert cube["measures"] == [ + {"name": "avg_lat", "sql": "{users}.lat - {CUBE}.amount", "type": "avg"}] + + +def test_a_decomposed_part_lands_on_the_cube_its_operand_reads(): + """Two aggregates over two datasets: each part is declared on the cube it reads, + which is what lets Cube correct row multiplication for each independently. So the + geo half needs no requalification -- its part lives on `users` already.""" + model = _two_cube_geo_model( + "AVG(users.home_latitude) - MIN(orders.amount)") files, _ = convert_ossie_to_cube(model, base_cube="orders") - measures = parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"] - # `{users}.lat` names the cube explicitly, since `{CUBE}` here would mean - # `orders`. `{CUBE.amount}` stays a member reference because `amount` is a - # declared field of the cube the measure lands on. - assert measures[0]["sql"] == "AVG({users}.lat) - MIN({CUBE}.amount)" - assert measures[0]["type"] == "number" + on_users = by_name(parse(files["model/cubes/users.yml"])["cubes"][0]["measures"]) + on_orders = by_name(parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"]) + assert on_users["avg_lat_part_1"]["sql"] == "{CUBE}.lat" + assert on_users["avg_lat_part_1"]["public"] is False + assert on_orders["avg_lat_part_2"]["sql"] == "{CUBE}.amount" + # The public measure stays on the base cube, naming the foreign part by its cube + # and its own with `{CUBE.x}`. + assert on_orders["avg_lat"]["sql"] == ( + "{users.avg_lat_part_1} - {CUBE.avg_lat_part_2}") + assert "public" not in on_orders["avg_lat"] def test_geo_half_references_normalize_to_the_underlying_column(): diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index b1e128e7..985ebca5 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -18,9 +18,14 @@ """Apache Ossie semantic model -> Cube data model.""" import pytest -from _util import by_name, parse +from _util import by_name, expr_of, model_of, parse -from ossie_cube import ConversionError, IssueType, convert_ossie_to_cube +from ossie_cube import ( + ConversionError, + IssueType, + convert_cube_to_ossie, + convert_ossie_to_cube, +) from ossie_cube._common import OSSIE_VERSION @@ -383,15 +388,43 @@ def test_declared_member_gets_a_member_reference_and_a_raw_column_does_not(): assert _cubes(files)["orders"]["measures"][0]["sql"] == "{CUBE}.shipping_fee" -def test_ratio_becomes_a_calculated_measure(): - files, issues = convert_ossie_to_cube(_ossie( +def test_a_ratio_is_split_into_one_measure_per_aggregate(): + """Each aggregate becomes its own `public: false` measure on the cube its operand + comes from, and the public measure references them. Cube corrects for row + multiplication per measure, so splitting is what lets each aggregate be corrected + on its own cube instead of the whole ratio being one opaque expression.""" + files, _ = convert_ossie_to_cube(_ossie( _TWO_DATASETS, _REL, _metric("aov", "SUM(orders.amount) / COUNT(DISTINCT users.id)"))) - measure = _cubes(files)["orders"]["measures"][0] - assert measure["type"] == "number" - assert measure["sql"] == "SUM({CUBE}.amount) / COUNT(DISTINCT {users.id})" - assert any("spans several datasets" in i.detail - for i in issues.of_type(IssueType.APPROXIMATED)) + orders = by_name(_cubes(files)["orders"]["measures"]) + users = by_name(parse(files["model/cubes/users.yml"])["cubes"][0]["measures"]) + + assert orders["aov_part_1"] == { + "name": "aov_part_1", "sql": "{CUBE}.amount", "type": "sum", + "meta": {"ossie": {"part_of": "aov"}}, "public": False} + # `users.id` is that cube's primary key, so its aggregate is a bare Cube count -- + # the form Cube corrects for fan-out. + assert users["aov_part_2"] == { + "name": "aov_part_2", "type": "count", + "meta": {"ossie": {"part_of": "aov"}}, "public": False} + # `{CUBE.aov_part_1}` rather than `{orders.aov_part_1}`: an own-cube reference + # stays correct when the cube is extended. + assert orders["aov"] == { + "name": "aov", "type": "number", + "sql": "{CUBE.aov_part_1} / {users.aov_part_2}"} + + +def test_a_split_ratio_comes_back_as_the_metric_it_was_split_from(): + """The split is an implementation detail of the Cube side: the parts are marked + generated, so import skips them and inlines their SQL back through the public + measure's references, recovering the original expression verbatim.""" + expression = "SUM(orders.amount) / COUNT(DISTINCT users.id)" + files, _ = convert_ossie_to_cube( + _ossie(_TWO_DATASETS, _REL, _metric("aov", expression))) + ossie, _ = convert_cube_to_ossie(files, strict_fanout=False) + metrics = model_of(ossie)["metrics"] + assert [m["name"] for m in metrics] == ["aov"] + assert expr_of(metrics[0]) == expression def test_metric_lands_on_the_dataset_its_expression_references(): diff --git a/converters/cube/uv.lock b/converters/cube/uv.lock index 6b158087..05578dff 100644 --- a/converters/cube/uv.lock +++ b/converters/cube/uv.lock @@ -8,6 +8,7 @@ version = "0.2.0.dev0" source = { editable = "." } dependencies = [ { name = "pyyaml" }, + { name = "sqlglot" }, ] [package.dev-dependencies] @@ -18,7 +19,10 @@ dev = [ ] [package.metadata] -requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] +requires-dist = [ + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=20.0" }, +] [package.metadata.requires-dev] dev = [ @@ -390,6 +394,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, ] +[[package]] +name = "sqlglot" +version = "30.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/cd/39a94f0f98076ee8e7c7c38fd4bba8d7845b0c629ff967057c64ef2c0989/sqlglot-30.14.0.tar.gz", hash = "sha256:df2ef5d2b8ca814313781f4ff35bf63e58f821ef517eeddbd523c19a61fa9bb9", size = 5944410, upload-time = "2026-07-27T11:23:30.698Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/ec/a729883ceda22dcd9117ce182f64d884bf494e72c4dfce00c2ad0a5978e1/sqlglot-30.14.0-py3-none-any.whl", hash = "sha256:fc768e24889d63a5e1237dea7ad305e5ffb4356a98b0bed828f89591ebcd3636", size = 719007, upload-time = "2026-07-27T11:23:28.637Z" }, +] + [[package]] name = "typing-extensions" version = "4.16.0" From 8e7131815d9bf936caee58a09a989b930c06b45f Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 16:55:36 +0500 Subject: [PATCH 22/25] Convert a fan-out-unsafe metric and report it, rather than refusing Refusing mirrored Cube's own refusal, but it is the wrong default for a hub-and-spoke converter: a model with one fanning join and one SUM failed outright, so the spoke on the other side got nothing at all -- and the metrics most worth converting are exactly the ones that trip it. Import now emits the metric and records FANOUT_UNSAFE_METRIC, naming the metric, the dataset and the relationship responsible. `--strict-fanout` (was `--no-strict-fanout`, inverted) restores the refusal for a caller who would rather have nothing than a number that disagrees with Cube. --- converters/cube/README.md | 15 +++++++---- converters/cube/src/ossie_cube/cli.py | 18 +++++++------ .../cube/src/ossie_cube/converter_issues.py | 9 ++++--- converters/cube/src/ossie_cube/cube_to_osi.py | 10 ++++--- converters/cube/tests/test_cli.py | 10 +++---- converters/cube/tests/test_cube_to_osi.py | 27 ++++++++++--------- converters/cube/tests/test_edge_cases.py | 10 +++---- converters/cube/tests/test_osi_to_cube.py | 2 +- 8 files changed, 57 insertions(+), 44 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index 71897937..d95c6dd2 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -65,7 +65,7 @@ composite metric). Python 3.11+. ```bash ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] - [--no-strict-fanout] + [--strict-fanout] ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] ``` @@ -185,10 +185,15 @@ emit a silently-wrong one: | `sum`, `avg`, `count` + `sql` | `SUM(x)`, `AVG(x)`, `COUNT(x)` | **No** | Only the last row is at risk, and only when its own cube is the `to` (one) side of -a relationship in the model. The converter computes that from the Ossie graph and, -**by default, refuses** -- mirroring Cube's own refusal. Pass -`--no-strict-fanout` to emit the metric with a `FANOUT_UNSAFE_METRIC` issue -instead, naming the metric, the dataset, and the relationship responsible. +a relationship in the model. The converter computes that from the Ossie graph and +**records a `FANOUT_UNSAFE_METRIC` issue** naming the metric, the dataset and the +relationship responsible -- refusing a whole model over one such metric would leave +the spoke on the other side with nothing. Pass `--strict-fanout` to refuse instead, +mirroring Cube's own refusal. + +The issue is reported to the caller, not written into the Ossie model: the spec has +no additivity declaration to write it into (see below), and a `custom_extensions` +entry would only give every other converter something to warn about and discard. Because a bare `count` maps through the primary key, a cube carrying one **must** declare `primary_key: true` on a dimension; its absence is an error, not a diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py index ce4804b1..d665be0c 100644 --- a/converters/cube/src/ossie_cube/cli.py +++ b/converters/cube/src/ossie_cube/cli.py @@ -28,10 +28,12 @@ always needs `-o` (a directory). Conversions that could not carry something across print an issue list to stderr. -By default a metric whose value a static Ossie expression cannot keep correct -under row multiplication is refused on import, mirroring Cube's own refusal to -answer such a query; pass `--no-strict-fanout` to emit it with a recorded issue -instead. +A metric whose value a static Ossie expression cannot keep correct under row +multiplication is converted with a `FANOUT_UNSAFE_METRIC` issue naming the metric, +the dataset and the relationship responsible -- a hub-and-spoke converter that +refuses a whole model over one such metric is not much use to the spoke on the +other side. Pass `--strict-fanout` to refuse instead, mirroring Cube's own refusal +to answer such a query. """ import argparse @@ -64,10 +66,10 @@ def _build_parser(): imp.add_argument("--view", help="view whose name/description/AI context map onto the " "Ossie model (default: the sole view, if there is one)") - imp.add_argument("--no-strict-fanout", dest="strict_fanout", - action="store_false", default=True, - help="record fan-out-unsafe metrics as issues instead of " - "refusing the conversion") + imp.add_argument("--strict-fanout", dest="strict_fanout", + action="store_true", default=False, + help="refuse the conversion when a metric is fan-out-unsafe, " + "instead of converting it and recording an issue") exp = sub.add_parser( "export", help="Apache Ossie semantic model -> Cube data model directory") diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py index f0306d2f..d0285cd7 100644 --- a/converters/cube/src/ossie_cube/converter_issues.py +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -103,10 +103,11 @@ def __str__(self): class IssueLog: """Collects issues during a conversion. - `strict_types` names the issue types that should abort the conversion - instead of being recorded. The CLI puts `FANOUT_UNSAFE_METRIC` in there by - default, mirroring Cube's own refusal to answer a query whose measures - reference cubes that lead to row multiplication. + `strict_types` names the issue types that should abort the conversion instead of + being recorded. Nothing is in there by default: a converter that refuses a whole + model over one metric leaves the spoke on the other side with nothing. Passing + `--strict-fanout` adds `FANOUT_UNSAFE_METRIC`, mirroring Cube's own refusal to + answer a query whose measures reference cubes that lead to row multiplication. """ issues: list = field(default_factory=list) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index 136cf114..a0fbcc6c 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -98,15 +98,17 @@ _AND_SPLIT_RE = re.compile(r"\s+AND\s+", re.IGNORECASE) -def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=True): +def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=False): """Convert Cube model files ({relative filename: YAML str}) to Ossie YAML. Returns (ossie_yaml_str, IssueLog). `model_name` overrides the Ossie model name (default: the mapped view's name, else 'cube_model'). `view` names the view whose name/description/AI context map onto the Ossie model when the - directory holds more than one. `strict_fanout` refuses metrics whose value a - static Ossie expression cannot keep correct under row multiplication -- see - README "Fan-out". + directory holds more than one. + + A metric whose value a static Ossie expression cannot keep correct under row + multiplication is converted with a FANOUT_UNSAFE_METRIC issue; `strict_fanout` + refuses it instead -- see README "Fan-out". """ if not isinstance(files, dict) or not files: raise ConversionError("expected a non-empty mapping of {filename: YAML}") diff --git a/converters/cube/tests/test_cli.py b/converters/cube/tests/test_cli.py index e608bd4f..2314280d 100644 --- a/converters/cube/tests/test_cli.py +++ b/converters/cube/tests/test_cli.py @@ -227,7 +227,7 @@ def test_issues_go_to_stderr_so_stdout_stays_pipeable(tmp_path, capsys): parse(captured.out) # stdout is still clean YAML -def test_fanout_refusal_exits_nonzero_and_the_flag_downgrades_it(tmp_path, capsys): +def test_fanout_warns_by_default_and_the_flag_exits_nonzero(tmp_path, capsys): model = _write(tmp_path / "model", **{"cubes|m.yml": ( "cubes:\n" " - name: orders\n" @@ -248,14 +248,14 @@ def test_fanout_refusal_exits_nonzero_and_the_flag_downgrades_it(tmp_path, capsy " sql: \"{CUBE}.ltv\"\n" " type: sum\n" )}) - assert main(["import", "-i", str(model)]) == 1 - assert "FANOUT_UNSAFE_METRIC" in capsys.readouterr().err - - assert main(["import", "-i", str(model), "--no-strict-fanout"]) == 0 + assert main(["import", "-i", str(model)]) == 0 captured = capsys.readouterr() assert "FANOUT_UNSAFE_METRIC" in captured.err assert parse(captured.out)["semantic_model"][0]["metrics"] + assert main(["import", "-i", str(model), "--strict-fanout"]) == 1 + assert "FANOUT_UNSAFE_METRIC" in capsys.readouterr().err + def test_view_and_name_flags_take_effect(tmp_path, capsys): model = _write(tmp_path / "model", **{ diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py index a3609e98..15f6f26d 100644 --- a/converters/cube/tests/test_cube_to_osi.py +++ b/converters/cube/tests/test_cube_to_osi.py @@ -398,17 +398,13 @@ def test_multi_stage_measure_is_dropped_with_an_issue(): } -def test_fanout_unsafe_metric_is_refused_by_default(): +def test_fanout_unsafe_metric_is_recorded_by_default(): """`users` is the one side of a many-to-one join, so summing over it after the - join over-counts. Cube deduplicates on the primary key at query time; a static - Ossie expression cannot, so the default is to refuse rather than emit a number - that silently disagrees with Cube.""" - with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): - convert_cube_to_ossie(_FANOUT_MODEL) - - -def test_fanout_unsafe_metric_is_recorded_when_not_strict(): - out, issues = convert_cube_to_ossie(_FANOUT_MODEL, strict_fanout=False) + join over-counts. Cube deduplicates on the primary key at query time and a static + Ossie expression cannot -- so the metric converts and the risk is reported, named + down to the relationship responsible. Refusing the whole model over one metric + would leave the spoke on the other side with nothing to convert.""" + out, issues = convert_cube_to_ossie(_FANOUT_MODEL) metric = by_name(model_of(out)["metrics"])["lifetime_value"] assert expr_of(metric) == "SUM(users.ltv)" recorded = issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) @@ -417,10 +413,17 @@ def test_fanout_unsafe_metric_is_recorded_when_not_strict(): assert "over-count" in recorded[0].detail +def test_fanout_unsafe_metric_is_refused_under_strict_fanout(): + """Mirrors Cube's own refusal, for a caller who would rather have nothing than a + number that disagrees with Cube.""" + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(_FANOUT_MODEL, strict_fanout=True) + + def test_idempotent_aggregates_are_never_flagged(fixture_a): """count / count_distinct / min / max are unaffected by duplicate rows, so a - fanned-out dataset carrying only those converts cleanly under strict mode.""" - _, issues = convert_cube_to_ossie(fixture_a) + fanned-out dataset carrying only those raises nothing even under strict mode.""" + _, issues = convert_cube_to_ossie(fixture_a, strict_fanout=True) assert not issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 6cdd1b50..0f83c4ac 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -42,7 +42,7 @@ def _files(**named): def _roundtrip(files): - ossie, issues = convert_cube_to_ossie(files, strict_fanout=False) + ossie, issues = convert_cube_to_ossie(files) back, _ = convert_ossie_to_cube(ossie) return ossie, back, issues @@ -150,10 +150,10 @@ def test_count_over_an_expression_is_fanout_unsafe(): " sql: \"{CUBE}.email\"\n" " type: count\n" )) - with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): - convert_cube_to_ossie(files) - _, issues = convert_cube_to_ossie(files, strict_fanout=False) + _, issues = convert_cube_to_ossie(files) assert issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files, strict_fanout=True) _MULTI_STAGE = _files(orders=( @@ -425,7 +425,7 @@ def test_a_many_to_one_join_still_makes_its_target_fanned_out(): " type: sum\n" )) with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): - convert_cube_to_ossie(files) + convert_cube_to_ossie(files, strict_fanout=True) def test_one_to_one_keeps_its_declared_orientation(): diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 985ebca5..3fd94eca 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -421,7 +421,7 @@ def test_a_split_ratio_comes_back_as_the_metric_it_was_split_from(): expression = "SUM(orders.amount) / COUNT(DISTINCT users.id)" files, _ = convert_ossie_to_cube( _ossie(_TWO_DATASETS, _REL, _metric("aov", expression))) - ossie, _ = convert_cube_to_ossie(files, strict_fanout=False) + ossie, _ = convert_cube_to_ossie(files) metrics = model_of(ossie)["metrics"] assert [m["name"] for m in metrics] == ["aov"] assert expr_of(metrics[0]) == expression From a103b0c11d3bce4f67b838dd207a12c186fec400 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 18:52:56 +0500 Subject: [PATCH 23/25] Add the interop matrix the README's claims are measured with The README asserts that the stash reduction cut Databricks warnings from 32 to 2 and that three spokes reject a two-part source. Both were measured with a throwaway script, which is no use to a reviewer -- so it is now a committed tool. `tools/interop_matrix.py` converts a Cube model to Ossie, checks that intermediate against the repo's own validation/validate.py, then hands it to all nine Python spokes and reports result, warning count, and how many of those warnings exist only because of a foreign `custom_extensions` entry. Stdlib only, and outside pytest -- it drives the other converters' environments, not this one's. Nothing about it is Cube-specific past the first hop. If it is useful repo-wide it belongs somewhere like compliance/, which is a dev@ question. --- converters/cube/README.md | 51 +++++ converters/cube/tools/interop_matrix.py | 251 ++++++++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 converters/cube/tools/interop_matrix.py diff --git a/converters/cube/README.md b/converters/cube/README.md index d95c6dd2..d5833dc0 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -269,6 +269,52 @@ Error: Source 'public.orders' must be a fully qualified db.schema.table or a sub Import reports this as `SOURCE_NOT_FULLY_QUALIFIED` rather than guessing a catalog name, so it surfaces where the Ossie document is produced instead of three hops later. +### Measuring it + +Both claims above are measurements, so they are reproducible: + +```bash +uv run tools/interop_matrix.py # the committed TPC-DS fixture +uv run tools/interop_matrix.py path/to/cube/model # any Cube model directory +uv run tools/interop_matrix.py --spokes omni --keep # one spoke, keep its output +``` + +It converts a Cube model to Ossie, checks that intermediate against the repo's own +`validation/validate.py`, then hands it to every other converter and reports what +each made of it: + +``` +model: converters/cube/tests/fixtures/tpcds_cube +Ossie: 539 lines, 7 CUBE stash entries +issues: 5x CUBE_LEVEL_AI_CONTEXT_INERT +spec: valid (validation/validate.py) + +spoke result warns foreign note +---------------------------------------------------------------------------- +databricks OK 22 2 +dbt FAIL 0 0 AttributeError: 'PydanticSemanticManifes +gooddata OK 0 0 +gsf OK 0 0 +honeydew OK 0 0 +omni OK 15 7 +orionbelt OK 2 0 +snowflake OK 7 7 +wisdom OK 47 7 +polaris -- Java converter, needs Maven +salesforce -- Java converter, needs Maven +``` + +`foreign` counts warnings that name a `custom_extensions` vendor — the cost this +converter imposes on the others by stashing, and the number to watch when deciding +whether something belongs in a stash at all. The dbt `FAIL` is unrelated to this +converter: its CLI crashes on every input, including this repo's own examples +([#296](https://github.com/apache/ossie/issues/296)). + +Each spoke runs in its own `uv` environment, so the first run resolves that +converter's dependencies; the script is stdlib-only and needs none of its own. Nothing +about it is Cube-specific except the first hop — if it is useful repo-wide it belongs +somewhere like `compliance/`, which is a question for `dev@`. + ## Conversion issues `convert_cube_to_ossie` returns `(yaml, IssueLog)`. Each issue carries a type, the @@ -343,6 +389,11 @@ document, and Hypothesis property-based round-trip tests over generated Cube models -- which fall back to a seeded sweep when `hypothesis` is unavailable, so the properties still run. +`tools/interop_matrix.py` checks the other half of the job — whether the Ossie this +converter emits is any use to the other spokes. It is not part of `pytest`, because +it drives the other converters' environments rather than this one's. See +[Measuring it](#measuring-it). + ## Future effort Both the Apache Ossie specification and Cube's data model are still evolving. As diff --git a/converters/cube/tools/interop_matrix.py b/converters/cube/tools/interop_matrix.py new file mode 100644 index 00000000..2bc2ec3b --- /dev/null +++ b/converters/cube/tools/interop_matrix.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 +# +# /// script +# requires-python = ">=3.11" +# /// + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Does a converted Cube model actually reach the other spokes? + +Ossie is a hub: `Cube -> Ossie` is only half the point, and a converter can pass its +own round-trip tests while emitting something the next converter chokes on. This +script runs `Cube -> Ossie -> every other spoke` and prints what each one made of it, +so a change can be judged on interop instead of on self-consistency. + + uv run tools/interop_matrix.py # the committed tpcds fixture + uv run tools/interop_matrix.py path/to/cube/model # any Cube model directory + uv run tools/interop_matrix.py --keep # leave the outputs to read + +Columns: + + result OK / EMPTY (exit 0, nothing written) / FAIL / SKIP (deps not installed) + warns lines the spoke wrote to stderr that read as warnings + foreign those warnings that name a `custom_extensions` vendor -- the cost this + converter imposes on every other spoke by stashing, and the number to + watch when deciding whether something belongs in a stash at all + +Each spoke runs in its own `uv` environment, so the first run for a given spoke +resolves its dependencies (`uv sync` there first to keep this fast) -- which leaves a +`uv.lock` and a `.venv` in that converter's directory. Those belong to the converter, +not to this run: check `git status` before committing. The Java converters (polaris, +salesforce) are listed as unsupported rather than skipped silently; they need Maven, +not uv. + +Stdlib only, so it needs no environment of its own. +""" + +import argparse +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +# (directory under converters/, argv to convert Ossie -> spoke, output is a directory) +# +# The invocations differ per spoke because the CLIs do: some take an `export` +# subcommand, some a named direction, snowflake takes none, gooddata ships no CLI at +# all and is driven through its Python API. +SPOKES = [ + ("databricks", ["ossie-databricks", "export"], False), + ("dbt", ["ossie-dbt", "osi-to-msi"], False), + ("gooddata", None, False), # API-only; see _run_gooddata + ("gsf", ["ossie-gsf", "export"], False), + ("honeydew", ["honeydew-osi", "osi-to-honeydew"], True), + ("omni", ["osi-omni", "export"], True), + ("orionbelt", ["ossie-orionbelt", "osi-to-obml"], False), + ("snowflake", ["ossie-snowflake"], False), + ("wisdom", ["ossie-wisdom", "osi-to-wisdom"], False), +] + +# Converters written in Java: a different toolchain, not a missing dependency. +UNSUPPORTED = ["polaris", "salesforce"] + +_WARN_RE = re.compile(r"warn", re.I) +# Python's `warnings.warn` prints the message and then echoes the calling source +# line, which would otherwise count the same warning twice. +_ECHO_RE = re.compile(r"^\s*warnings\.warn\b") +_FOREIGN_RE = re.compile(r"custom_extension|vendor|foreign", re.I) + + +def repo_root(): + for parent in [Path(__file__).resolve(), *Path(__file__).resolve().parents]: + if (parent / "converters").is_dir() and (parent / "core-spec").is_dir(): + return parent + sys.exit("cannot locate the repository root from this script's path") + + +def run(cwd, argv): + return subprocess.run(argv, cwd=cwd, capture_output=True, text=True) + + +def count_warnings(stderr): + """(warnings, of which are about a foreign vendor extension). + + A line count, so a warning whose message wraps counts more than once. It is a + relative measure -- run it before and after a change -- not an exact tally. + """ + warns = [ln for ln in stderr.splitlines() + if _WARN_RE.search(ln) and not _ECHO_RE.match(ln)] + return len(warns), len([ln for ln in warns if _FOREIGN_RE.search(ln)]) + + +def import_issues(stderr): + """The issue types `ossie-cube import` reported, as {type: count}. + + Its own issues do not read as warnings -- they are `[TYPE] element: detail` lines + -- so they are counted from their structure rather than by keyword. + """ + found = {} + for ln in stderr.splitlines(): + m = re.match(r"\s+\[([A-Z_]+)\]", ln) + if m: + found[m.group(1)] = found.get(m.group(1), 0) + 1 + return found + + +def produced_output(dest, is_dir): + if not dest.exists(): + return False + return any(dest.rglob("*")) if is_dir else dest.stat().st_size > 0 + + +def _run_gooddata(root, ossie, dest): + """gooddata ships no console script, so drive its API the way its README does.""" + script = ( + "import json, sys, yaml\n" + "from ossie_gooddata import osi_to_gooddata\n" + "from ossie_gooddata.models import gd_model_to_dict\n" + "model = yaml.safe_load(open(sys.argv[1]).read())\n" + "out = gd_model_to_dict(osi_to_gooddata(model))\n" + "open(sys.argv[2], 'w').write(json.dumps(out, indent=2, default=str))\n" + ) + return run(root / "converters/gooddata", + ["uv", "run", "--quiet", "python", "-c", script, + str(ossie), str(dest)]) + + +def cube_to_ossie(root, model_dir, dest): + r = run(root / "converters/cube", + ["uv", "run", "--quiet", "ossie-cube", "import", + "-i", str(model_dir), "-o", str(dest)]) + return r + + +def validate_ossie(root, ossie): + """Run the repo's own validator on the intermediate model. + + A spoke rejecting the model is only interesting once the model is known good, so + this is checked before the matrix rather than left to be inferred from it. + """ + return run(root, ["uv", "run", "--quiet", "validation/validate.py", str(ossie)]) + + +def main(): + root = repo_root() + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument( + "model", nargs="?", + default=str(root / "converters/cube/tests/fixtures/tpcds_cube"), + help="Cube model directory (default: the committed tpcds fixture)") + ap.add_argument("--spokes", help="comma-separated subset to run") + ap.add_argument("--keep", action="store_true", + help="keep the converted outputs and print where they are") + args = ap.parse_args() + + model_dir = Path(args.model).expanduser().resolve() + if not model_dir.exists(): + sys.exit(f"no such Cube model: {model_dir}") + + wanted = None + if args.spokes: + wanted = {s.strip() for s in args.spokes.split(",") if s.strip()} + unknown = wanted - {name for name, _, _ in SPOKES} + if unknown: + sys.exit(f"unknown spoke(s): {', '.join(sorted(unknown))}") + + out = Path(tempfile.mkdtemp(prefix="ossie-interop-")) + try: + ossie = out / "from_cube.yaml" + r = cube_to_ossie(root, model_dir, ossie) + if r.returncode != 0: + print(f"Cube -> Ossie FAILED\n{r.stderr}", file=sys.stderr) + return 1 + + text = ossie.read_text() + reported = import_issues(r.stderr) + print(f"model: {model_dir}") + print(f"Ossie: {len(text.splitlines())} lines, " + f"{text.count('vendor_name: CUBE')} CUBE stash entries") + if reported: + print("issues: " + ", ".join( + f"{n}x {kind}" for kind, n in sorted(reported.items()))) + + v = validate_ossie(root, ossie) + print(f"spec: {'valid' if v.returncode == 0 else 'INVALID'} " + f"(validation/validate.py)") + if v.returncode != 0: + print(v.stdout.strip() or v.stderr.strip()) + + print() + print(f"{'spoke':<12} {'result':<7} {'warns':>5} {'foreign':>8} note") + print("-" * 76) + + failures = 0 + for name, argv, is_dir in SPOKES: + if wanted and name not in wanted: + continue + dest = out / (name if is_dir else f"{name}.out") + if argv is None: + r = _run_gooddata(root, ossie, dest) + else: + r = run(root / "converters" / name, + ["uv", "run", "--quiet", *argv, + "-i", str(ossie), "-o", str(dest)]) + + warns, foreign = count_warnings(r.stderr) + note = "" + if r.returncode != 0: + tail = (r.stderr.strip().splitlines() or [""])[-1] + # A missing environment is not the converter rejecting the model. + skipped = "No solution found" in r.stderr or "no such command" in tail + result = "SKIP" if skipped else "FAIL" + note = tail[:40] + failures += result == "FAIL" + else: + result = "OK" if produced_output(dest, is_dir) else "EMPTY" + print(f"{name:<12} {result:<7} {warns:>5} {foreign:>8} {note}") + + if not wanted: + for name in UNSUPPORTED: + print(f"{name:<12} {'--':<7} {'':>5} {'':>8} " + "Java converter, needs Maven") + + if args.keep: + print(f"\noutputs: {out}") + return 1 if failures else 0 + finally: + if not args.keep: + shutil.rmtree(out, ignore_errors=True) + + +if __name__ == "__main__": + sys.exit(main()) From 410abb9e293a3035b17b6135a06a1ca0655a7e30 Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 19:55:14 +0500 Subject: [PATCH 24/25] Fix two expression-handling defects found in review **String literals were rewritten as if they were code.** Cube compiles a YAML `sql` as a Python f-string (`f""` in YamlCompiler), so `{...}` interpolates anywhere in the value -- SQL's own quotes mean nothing to it. Export was rewriting inside literals, which means Cube then replaced the literal's own text with a column reference: SUM(orders.amount) || ' per COUNT(users.id) unit' -> {CUBE.label_part_1} || ' per {users.label_part_2} unit' Three silent consequences: the literal's text changed, a measure reference appeared inside it, and `users` looked like a second dataset -- which also decides which cube the measure lands on. `quoted_runs`/`referenced_datasets` now confine every export-side rewrite to the code outside quoted runs, and `aggregate_spans` skips a name found inside one. Import deliberately does NOT do this, and now says so: because of the f-string compilation, a reference inside a literal is one Cube really resolves, so skipping it there would lose it. Both directions are pinned by tests. **A cross-dataset metric stopped being reported.** The check lived only in the calculated-measure fallback, so the two shapes a cross-dataset metric now normally takes -- a decomposed pair, or a recognized single aggregate -- reported nothing, even though decomposition produces *more* cross-cube references than the shape that did report. Moved to where the dataset set is computed, so it fires whatever shape the measure takes, and no test had ever asserted it. Also folds the three copies of the dotted-reference regex into one helper, and drops the parameters `_measure_from_expression` no longer needs. 298 tests, 96% line coverage (expressions.py 82% -> 96%). --- converters/cube/README.md | 9 ++ converters/cube/src/ossie_cube/_common.py | 74 +++++++++++++- converters/cube/src/ossie_cube/expressions.py | 9 ++ converters/cube/src/ossie_cube/osi_to_cube.py | 44 ++++----- converters/cube/tests/test_edge_cases.py | 98 +++++++++++++++++++ converters/cube/tests/test_osi_to_cube.py | 49 ++++++++++ 6 files changed, 258 insertions(+), 25 deletions(-) diff --git a/converters/cube/README.md b/converters/cube/README.md index d5833dc0..f3489dec 100644 --- a/converters/cube/README.md +++ b/converters/cube/README.md @@ -161,6 +161,15 @@ Ossie dialect enum has no `CUBE` entry -- so import emits `ANSI_SQL`, and export prefers `ANSI_SQL` with `--dialect` prepending a warehouse dialect (e.g. `SNOWFLAKE` for a Snowflake-backed Cube model). +**String literals** are handled asymmetrically, on purpose. Cube compiles a YAML +`sql` value as a Python f-string (`f""` in `YamlCompiler`), so `{CUBE}.col` +interpolates *anywhere* in the value -- SQL's own quotes mean nothing to it. So on +import a reference inside a literal is a real reference and is translated, while on +export nothing is rewritten inside a literal: emitting `{CUBE.col}` there would make +Cube replace the literal's own text with a column reference. The same rule decides +which dataset a metric belongs to, so a name mentioned only inside a literal does not +attribute the metric or make it look cross-dataset. + ## Fan-out This is the one place where Cube carries semantics an Ossie expression cannot, and diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py index 6ca0666a..200ef0fa 100644 --- a/converters/cube/src/ossie_cube/_common.py +++ b/converters/cube/src/ossie_cube/_common.py @@ -385,6 +385,73 @@ def repl(m): return out, changed +def quoted_runs(sql): + """Split SQL into (text, is_quoted) runs, delimiters included in the quoted run. + + Used by the **export** direction only, to keep a rewrite out of string literals + and delimited identifiers. Import deliberately does not do this: a Cube YAML + `sql` is compiled as a Python f-string (`f""` in YamlCompiler), so `{CUBE}` + interpolates anywhere in the value -- SQL's own quotes are ordinary characters to + it. Skipping quoted text on the way in would therefore *lose* a reference Cube + really does resolve. + + `'`, `"` and backtick all open a run; a run is closed by its own delimiter, and + an unterminated one runs to the end (reported quoted, so nothing in it is + rewritten). SQL's `''` doubling needs no special case: it reads as a close + immediately followed by an open, leaving an empty unquoted run between. + """ + runs, buf, quote = [], [], None + for ch in str(sql): + if quote: + buf.append(ch) + if ch == quote: + runs.append(("".join(buf), True)) + buf, quote = [], None + elif ch in "'\"`": + if buf: + runs.append(("".join(buf), False)) + buf, quote = [ch], ch + else: + buf.append(ch) + if buf: + runs.append(("".join(buf), quote is not None)) + return runs + + +def sub_outside_quotes(sql, transform): + """Apply `transform` to the parts of `sql` outside quoted runs.""" + return "".join(text if quoted else transform(text) + for text, quoted in quoted_runs(sql)) + + +def referenced_datasets(expr, known): + """The dataset names an Ossie expression references, ignoring quoted text. + + Decides which cube a measure lands on and whether it crosses cubes, so a name + that only appears inside a string literal must not count -- otherwise + `SUM(orders.amount) || ' per users.id unit'` reads as a two-dataset metric and + gets attributed to the base cube rather than to `orders`. + """ + found = set() + for text, quoted in quoted_runs(expr): + if quoted: + continue + found |= {m.group(1) for m in DOTTED_REF_RE.finditer(text) + if m.group(1) in known} + return found + + +def quoted_char_mask(sql): + """One flag per character of `sql`: True where it sits inside a quoted run. + + For a caller that needs offsets into the original string rather than a rewrite. + """ + mask = [] + for text, quoted in quoted_runs(sql): + mask.extend([quoted] * len(text)) + return mask + + def source_part_count(source): """How many identifier parts a dotted dataset `source` has, or None for a query. @@ -488,6 +555,11 @@ def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), exists only in Ossie -- Cube has neither a column nor a member by that name -- so a reference to it becomes the half's own SQL (`{CUBE}.lat`), requalified when it crosses cubes. + + A dotted token inside a string literal is left alone. This matters more than it + looks: Cube compiles a YAML `sql` as a Python f-string, so a `{...}` it emitted + into a literal would be interpolated at compile time and replace the literal's + own text with a column reference. """ escaped = str(expr).replace("{", "\\{").replace("}", "\\}") known = set(cube_names) @@ -510,7 +582,7 @@ def repl(m): # reference or an unrelated dotted token. Leave it alone. return m.group(0) - return DOTTED_REF_RE.sub(repl, escaped) + return sub_outside_quotes(escaped, lambda run: DOTTED_REF_RE.sub(repl, run)) # --- source --------------------------------------------------------------------- diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py index 68243c0f..12f70e44 100644 --- a/converters/cube/src/ossie_cube/expressions.py +++ b/converters/cube/src/ossie_cube/expressions.py @@ -38,6 +38,8 @@ import sqlglot import sqlglot.expressions as exp +from ._common import quoted_char_mask + # sqlglot node types for the aggregates this converter maps to a Cube measure type. # `Count` covers COUNT / COUNT(DISTINCT x); ApproxDistinct covers # APPROX_COUNT_DISTINCT. @@ -91,6 +93,10 @@ def aggregate_spans(expr): inside another span is not returned, so `SUM(x) / NULLIF(SUM(y), 0)` gives two and `SUM(SUM(x))` gives one. Returns [] when the expression does not parse, or is itself a single aggregate needing no decomposition. + + A name inside a string literal is not a call: `SUM(x) || ' per COUNT(y) unit'` + has one aggregate, not two. Taking the second would splice a measure reference + into the literal. """ text = str(expr) if parse(text) is None or is_single_aggregate(text): @@ -98,6 +104,7 @@ def aggregate_spans(expr): candidates = [] upper = text.upper() + quoted = quoted_char_mask(text) for name in _AGGREGATE_NAMES: at = 0 while True: @@ -106,6 +113,8 @@ def aggregate_spans(expr): break start, after = at, at + len(name) at = after + if quoted[start]: + continue # A call, not part of a longer identifier: boundary before, `(` after. if start and (text[start - 1].isalnum() or text[start - 1] == "_"): continue diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 889b3fce..762ae00f 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -36,7 +36,6 @@ from ._common import ( DATATYPE_TO_DIM_TYPE, - DOTTED_REF_RE, DEFAULT_DATATYPE_FOR_CUBE_TYPE, OSSIE_FUNC_TO_AGG, OSSIE_VERSION, @@ -53,6 +52,7 @@ pick_expression, primary_key_operand, read_stash, + referenced_datasets, require_str, sanitize_name, synonyms_of, @@ -660,15 +660,22 @@ def resolve_base(): "no ANSI_SQL or preferred-dialect expression; metric dropped") continue - referenced = { - m.group(1) for m in re.finditer( - r"(? 1: + # Cube resolves a cross-cube member reference by adding an implicit join, + # so the model needs a join path between these cubes -- which Ossie's + # expression does not state and this converter cannot verify. Reported for + # every shape the measure can take: it used to be raised only from the + # calculated-measure fallback, so a decomposed metric (the shape with the + # *most* cross-cube references) reported nothing at all. + issues.add(IssueType.APPROXIMATED, scope, + f"expression spans datasets {', '.join(sorted(referenced))}; " + f"Cube reaches the others from '{target}' through an implicit " + f"join, so verify a join path exists") + spans = [] if stash.get("sql") else aggregate_spans(expr) if len(spans) > 1: # A composite metric: give each aggregate its own measure on the cube its @@ -677,13 +684,12 @@ def resolve_base(): # seeing one opaque expression -- see _decompose_measure. public_sql = _decompose_measure( expr, spans, mname, target, measures_by_cube, members_by_cube, - inline_sql_by_cube, pk_by_cube, sanitized, name, scope, issues) + inline_sql_by_cube, pk_by_cube, sanitized, name) measure = {"name": mname, "sql": public_sql, "type": "number"} else: measure = _measure_from_expression( expr, target, mname, stash, members_by_cube.get(target, set()), - inline_sql_by_cube, pk_by_cube.get(target, []), sanitized, scope, - issues) + inline_sql_by_cube, pk_by_cube.get(target, []), sanitized) _apply_measure_metadata(metric, measure, stash) _place(measures_by_cube, target, measure, name) return measures_by_cube @@ -691,7 +697,7 @@ def resolve_base(): def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, members_by_cube, inline_sql_by_cube, pk_by_cube, sanitized, - model_name, scope, issues): + model_name): """Emit one `public: false` measure per aggregate; return the sql referencing them. Cube corrects for row multiplication per measure, keyed on the cube that measure @@ -712,8 +718,7 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, for start, end in spans: piece = expr[start:end] # Each aggregate lands on the cube its own operand references. - refs = {m.group(1) for m in DOTTED_REF_RE.finditer(piece) - if m.group(1) in sanitized} + refs = referenced_datasets(piece, sanitized) part_target = next(iter(refs)) if len(refs) == 1 else fallback index += 1 @@ -726,7 +731,7 @@ def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, part = _measure_from_expression( piece, part_target, part_name, {}, members_by_cube.get(part_target, set()), inline_sql_by_cube, - pk_by_cube.get(part_target, []), sanitized, scope, issues) + pk_by_cube.get(part_target, []), sanitized) part["public"] = False part["meta"] = {"ossie": {"part_of": mname}} _place(measures_by_cube, part_target, part, model_name) @@ -755,7 +760,7 @@ def _place(measures_by_cube, target, measure, model_name): def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_cube, - primary_key, sanitized, scope, issues): + primary_key, sanitized): """Turn an Ossie metric expression back into a structured Cube measure. `COUNT(DISTINCT )` is Cube's bare `type: count` -- @@ -796,15 +801,6 @@ def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_ measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( expr, target, members, sanitized, inline_sql=inline_sql_by_cube) measure["type"] = "number" - if len({ - ref for ref in re.findall( - r"(? 1: - issues.add(IssueType.APPROXIMATED, scope, - f"expression spans several datasets; emitted as a calculated " - f"measure on cube '{target}' -- verify the join path") return measure diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py index 0f83c4ac..ebcb59fa 100644 --- a/converters/cube/tests/test_edge_cases.py +++ b/converters/cube/tests/test_edge_cases.py @@ -1413,3 +1413,101 @@ def test_several_semantic_models_convert_the_first_with_an_issue(): # The other models are not preserved anywhere, so this is a drop. dropped = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT) assert any("only the first is converted" in i.detail for i in dropped) + + +# --- string literals ------------------------------------------------------------ +# +# The two directions are deliberately asymmetric, so both are pinned here. A Cube +# YAML `sql` is compiled as a Python f-string (`f""` in YamlCompiler), which +# interpolates `{...}` anywhere in the value -- SQL's own quotes mean nothing to it. +# So on import a reference inside a literal is a real reference, while on export a +# rewrite must stop at the quotes or it would destroy the literal's text. + +@pytest.mark.parametrize("sql,expected", [ + ("a = 'x'", [("a = ", False), ("'x'", True)]), + ("'x' = a", [("'x'", True), (" = a", False)]), + ("'it''s'", [("'it'", True), ("'s'", True)]), + ('"col" = `c`', [('"col"', True), (" = ", False), ("`c`", True)]), + ("a = 'unterminated", [("a = ", False), ("'unterminated", True)]), + ("plain", [("plain", False)]), +]) +def test_quoted_runs_splits_sql_into_code_and_quoted_text(sql, expected): + from ossie_cube._common import quoted_runs + assert quoted_runs(sql) == expected + + +@pytest.mark.parametrize("expr,expected", [ + ("SUM(orders.amount)", {"orders"}), + ("SUM(orders.amount) / COUNT(users.id)", {"orders", "users"}), + ("SUM(orders.amount) || ' per users.id unit'", {"orders"}), + ("'orders.amount'", set()), + ("SUM(ghost.amount)", set()), +]) +def test_referenced_datasets_ignores_quoted_text(expr, expected): + from ossie_cube._common import referenced_datasets + assert referenced_datasets(expr, {"orders", "users"}) == expected + + +def test_a_reference_inside_a_literal_is_still_translated_on_import(): + """Not an oversight: Cube would have interpolated it, so dropping it would lose a + reference the model really does resolve.""" + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " dimensions:\n" + " - name: note\n" + " sql: \"CONCAT({CUBE}.status, ' {CUBE}.status ')\"\n" + " type: string\n" + )) + ossie, _ = convert_cube_to_ossie(files) + field = by_name(by_name(model_of(ossie)["datasets"])["orders"]["fields"])["note"] + assert expr_of(field) == "CONCAT(status, ' status ')" + + +# --- aggregate span scanning ----------------------------------------------------- +# +# The scanner decides whether a metric is decomposed into one measure per aggregate, +# so its rejection paths matter as much as its matches: a false positive splices a +# measure reference into text that was never a call. + +@pytest.mark.parametrize("expr,expected", [ + # Two aggregates -- the case decomposition exists for. + ("SUM(a.x) / COUNT(b.y)", ["SUM(a.x)", "COUNT(b.y)"]), + # Only the outermost of a nested pair. + ("SUM(a.x) / NULLIF(SUM(b.y), 0)", ["SUM(a.x)", "SUM(b.y)"]), + # A closing paren inside a literal does not end the call. + ("SUM(a.x || ')') / COUNT(b.y)", ["SUM(a.x || ')')", "COUNT(b.y)"]), + # Part of a longer identifier, not a call. + ("MY_SUM(a.x) / 2", []), + ("SUMMARY(a.x) - MIN(b.y)", ["MIN(b.y)"]), + # A name with no argument list at all. + ("a.count / b.total", []), + # Whitespace between the name and its parens is still a call. + ("SUM (a.x) - MIN (b.y)", ["SUM (a.x)", "MIN (b.y)"]), + # Unbalanced parens: not a span, and not a crash. + ("SUM(a.x / MIN(b.y)", []), + # A single aggregate needs no decomposition. + ("SUM(a.x)", []), + # Unparseable input falls back to one opaque measure. + ("SUM(a.x) /// COUNT(", []), +]) +def test_aggregate_spans_only_matches_real_calls(expr, expected): + from ossie_cube.expressions import aggregate_spans + assert [expr[s:e] for s, e in aggregate_spans(expr)] == expected + + +@pytest.mark.parametrize("expr,expected", [ + ("SUM(a.x)", False), + # One self-contained term: the space is inside the parens, so inlining it into a + # larger expression needs no parentheses. + ("COUNT(DISTINCT a.x)", False), + ("SUM(a.x) / 2", True), + ("CASE WHEN a.x THEN 1 END", True), # a top-level space is structure + ("'a + b'", False), # operators inside a literal are text + ("'a b'", False), + ('"a b"', False), +]) +def test_has_top_level_operator_ignores_quoted_text(expr, expected): + from ossie_cube.expressions import has_top_level_operator + assert has_top_level_operator(expr) is expected diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py index 3fd94eca..f685b029 100644 --- a/converters/cube/tests/test_osi_to_cube.py +++ b/converters/cube/tests/test_osi_to_cube.py @@ -414,6 +414,55 @@ def test_a_ratio_is_split_into_one_measure_per_aggregate(): "sql": "{CUBE.aov_part_1} / {users.aov_part_2}"} +def test_a_dotted_token_inside_a_string_literal_is_left_alone(): + """Cube compiles a YAML `sql` as a Python f-string, so a `{...}` written into a + string literal is still interpolated -- it would replace the literal's own text + with a column reference. So the rewrite has to stop at the quotes.""" + files, _ = convert_ossie_to_cube(_ossie(_ORDERS, metrics=_metric( + "m", "CONCAT(CAST(SUM(orders.amount) AS VARCHAR), ' orders.amount ')"))) + assert _cubes(files)["orders"]["measures"][0]["sql"] == ( + "CONCAT(CAST(SUM({CUBE}.amount) AS VARCHAR), ' orders.amount ')") + + +def test_an_aggregate_name_inside_a_string_literal_is_not_an_aggregate(): + """Otherwise the literal is treated as a second aggregate and gets a measure + reference spliced into the middle of it.""" + files, issues = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL, _metric( + "label", "SUM(orders.amount) || ' per COUNT(users.id) unit'"))) + measures = _cubes(files)["orders"]["measures"] + # One measure, not a decomposed pair, and the literal survives verbatim. + assert [m["name"] for m in measures] == ["label"] + assert measures[0]["sql"] == ( + "SUM({CUBE}.amount) || ' per COUNT(users.id) unit'") + assert "measures" not in _cubes(files, "model/cubes/users.yml")["users"] + # `users` is named only inside the literal, so this is not a cross-cube metric. + assert not issues.of_type(IssueType.APPROXIMATED) + + +@pytest.mark.parametrize("shape,expr", [ + ("decomposed", "SUM(orders.amount) / COUNT(DISTINCT users.id)"), + ("single aggregate", "SUM(orders.amount - users.id)"), + ("calculated", "SUM(orders.amount) + users.id"), +]) +def test_a_cross_dataset_metric_is_reported_whatever_shape_it_takes(shape, expr): + """Cube reaches another cube's members through an implicit join, so the model + needs a join path this converter cannot verify. The report used to come only from + the calculated-measure fallback, which meant the decomposed shape -- the one with + the *most* cross-cube references -- reported nothing.""" + _, issues = convert_ossie_to_cube( + _ossie(_TWO_DATASETS, _REL, _metric("m", expr))) + reported = issues.of_type(IssueType.APPROXIMATED) + assert len(reported) == 1, shape + assert "orders, users" in reported[0].detail + assert "join path" in reported[0].detail + + +def test_a_single_dataset_metric_is_not_reported(): + _, issues = convert_ossie_to_cube(_ossie( + _TWO_DATASETS, _REL, _metric("m", "SUM(orders.amount)"))) + assert not issues.of_type(IssueType.APPROXIMATED) + + def test_a_split_ratio_comes_back_as_the_metric_it_was_split_from(): """The split is an implementation detail of the Cube side: the parts are marked generated, so import skips them and inlines their SQL back through the public From a3a5fbbb6a29e1c6fb80664bbcfc8ef9eb2d333e Mon Sep 17 00:00:00 2001 From: MikeNitsenko Date: Mon, 3 Aug 2026 20:00:07 +0500 Subject: [PATCH 25/25] Clean up the review findings that were not behaviour - `_convert_cube` had its docstring after the first statement, so `__doc__` was None. The statement it displaced is now passed in instead (see below). - `_MeasureResolver` built a `_dimensions` set for every cube and never read it. - Implemented the memoization its docstring already claimed. A measure referenced from several places was recomputed once per reference, recursively: a 16-deep chain of double references took 262,125 calls and 2.0s, now 49 calls and 0.9s. The docstring now also records what the cache cannot fix -- inlining is exponential in reference depth because Cube's own inlining is, and no limit is imposed since any threshold would reject a legitimate model. - The join stash tested three conditions, two of which could not be false (`from_cube` is `cname` at that point, and an unnormalized `many_to_one` implies a normalized one). Reduced to the one that matters, with the reason stated. - `_plain_members` ran once per measure; hoisted to once per cube and shared with the dimension stage, which was computing the same set again. - Two open-coded copies of `_restore_parked_extensions` replaced with the helper written for exactly that; `ds_scope` folded into the identical `scope`. - Noted why `sanitize_name` is called with an empty `taken` (the target cube is not known yet; `_place` rejects the collision later). tools/interop_matrix.py: a spoke that hangs no longer takes the run down with it (600s timeout), a missing `uv` reports itself instead of raising FileNotFoundError, and the uv-wording match that distinguishes SKIP from FAIL is named and explained rather than inline. 298 tests, 96% line coverage. Interop matrix unchanged. --- converters/cube/src/ossie_cube/cube_to_osi.py | 83 +++++++++++-------- converters/cube/src/ossie_cube/osi_to_cube.py | 3 + converters/cube/tools/interop_matrix.py | 38 +++++++-- 3 files changed, 85 insertions(+), 39 deletions(-) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py index a0fbcc6c..7e5bbedc 100644 --- a/converters/cube/src/ossie_cube/cube_to_osi.py +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -144,12 +144,16 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=False fanned_out = _fanned_out_datasets(relationships) pk_by_cube = {cname: _primary_key_of(cube, cname) for cname, cube in cubes.items()} + # Which members regenerate from a bare column name, worked out once per cube: + # both the measure and the dimension stage need the same answer. + plain_by_cube = {cname: _plain_members(cube, cname) + for cname, cube in cubes.items()} metrics, extra_measures = _convert_measures( - cubes, pk_by_cube, fanned_out, issues) + cubes, pk_by_cube, plain_by_cube, fanned_out, issues) model["datasets"] = [ - _convert_cube(cname, cube, extra_joins.get(cname), + _convert_cube(cname, cube, plain_by_cube[cname], extra_joins.get(cname), extra_measures.get(cname), issues) for cname, cube in cubes.items() ] @@ -184,10 +188,7 @@ def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=False # Foreign-vendor extensions a previous export parked on the mapped view are # restored after the stash is written, so the CUBE entry stays first. - parked_exts = ((mapped_view.get("meta") or {}).get("ossie") or {}).get( - "custom_extensions") - if parked_exts: - model.setdefault("custom_extensions", []).extend(parked_exts) + _restore_parked_extensions(model, mapped_view.get("meta")) return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}), issues @@ -465,8 +466,7 @@ def _primary_key_of(cube, cname): if dim.get("primary_key")] -def _convert_cube(cname, cube, extra_joins, extra_measures, issues): - plain = _plain_members(cube, cname) +def _convert_cube(cname, cube, plain, extra_joins, extra_measures, issues): """Build one Ossie dataset from a Cube cube.""" scope = f"cube '{cname}'" ds = {"name": cname} @@ -480,8 +480,7 @@ def _convert_cube(cname, cube, extra_joins, extra_measures, issues): # GSF converters all reject anything shorter -- so a model that converts # cleanly here still cannot reach them. Better to say so at the point the # Ossie document is produced than to have it fail three hops later. - ds_scope = f"cube '{cname}'" - issues.add(IssueType.SOURCE_NOT_FULLY_QUALIFIED, ds_scope, + issues.add(IssueType.SOURCE_NOT_FULLY_QUALIFIED, scope, f"source '{ds['source']}' has {parts} part(s); several Ossie " f"converters (Databricks, Snowflake, NVIDIA GSF) require a " f"3-part catalog.schema.table, so qualify the cube's `sql_table` " @@ -530,8 +529,7 @@ def _convert_cube(cname, cube, extra_joins, extra_measures, issues): # Foreign-vendor extensions parked by a previous export are restored after the # stash is written, so the CUBE entry stays first and both survive. - if parked.get("custom_extensions"): - ds.setdefault("custom_extensions", []).extend(parked["custom_extensions"]) + _restore_parked_extensions(ds, cube.get("meta")) return ds @@ -686,10 +684,10 @@ def _convert_joins(cubes, skipped_files, issues): # common case. Only an orientation Ossie cannot express on its own -- # one_to_many (flipped) or one_to_one (no many side) -- needs recording. stash = {} - if (rel_type != "many_to_one" or cname != from_cube - or raw_rel != "many_to_one"): - # The last clause keeps a legacy spelling (`belongsTo`) exact - # without costing the modern spelling a stash entry. + # Testing the *declared* spelling, not the normalized one: a legacy + # `belongsTo` normalizes to many_to_one but has to come back spelled the + # way it was written, while the modern spelling costs no stash entry. + if raw_rel != "many_to_one": stash["declared_on"] = cname stash["relationship"] = raw_rel if rel_type == "one_to_many": @@ -792,23 +790,26 @@ class _MeasureResolver: Kept as a class because a calculated measure (`type: number`, and the other types in `CALCULATED_MEASURE_TYPES`) can reference other measures, which Cube resolves by inlining their full aggregate SQL -- so producing one measure's - expression may require producing another's first. Results are memoized and - reference cycles are rejected rather than recursed into. + expression may require producing another's first. Each measure's expression is + computed once and cached; a reference cycle is rejected rather than recursed + into. + + Note that inlining is inherently exponential in reference depth -- a chain where + each measure names the previous one twice doubles the SQL at every step -- and + that is Cube's own behaviour, not this converter's choice. The cache makes the + work proportional to the output rather than to the output times the depth; it + cannot make the output smaller. No limit is imposed, since any threshold would + reject a legitimate model to guard against a hand-written pathological one. """ def __init__(self, cubes, pk_by_cube, issues): self._pk = pk_by_cube self._issues = issues self._raw = {} - self._dimensions = {} + self._cache = {} for cname, cube in cubes.items(): for m in _as_named_list(cube.get("measures"), f"cube '{cname}' measures"): self._raw[(cname, require_str(m, "name", f"cube '{cname}': measure"))] = m - self._dimensions[cname] = { - d["name"] - for d in _as_named_list(cube.get("dimensions"), - f"cube '{cname}' dimensions") - } def measures(self): return self._raw @@ -827,6 +828,8 @@ def expression(self, cname, mname, stack=()): if key in stack: chain = " -> ".join(f"{c}.{m}" for c, m in stack + (key,)) raise ConversionError(f"measure reference cycle: {chain}") + if key in self._cache: + return self._cache[key] measure = self._raw[key] scope = f"{cname}.{mname}" mtype = snake(measure.get("type") or "") @@ -840,7 +843,7 @@ def expression(self, cname, mname, stack=()): IssueType.MULTI_STAGE_MEASURE_PARKED, scope, f"multi_stage measure (type '{mtype}'); preserved in " f"custom_extensions only") - return None + return self._remember(key, None) sql = measure.get("sql") filter_exprs = [ self._translate(f["sql"], cname, stack + (key,)) @@ -853,14 +856,14 @@ def expression(self, cname, mname, stack=()): raise ConversionError( f"measure '{scope}': type '{mtype}' requires 'sql'") expr = self._translate(sql, cname, stack + (key,)) - return filtered_operand(expr, filter_exprs) + return self._remember(key, filtered_operand(expr, filter_exprs)) if mtype == "count": if sql is None: - return primary_key_count_expression( - cname, self._pk.get(cname) or [], filter_exprs) + return self._remember(key, primary_key_count_expression( + cname, self._pk.get(cname) or [], filter_exprs)) operand = filtered_operand( self._operand(cname, sql, stack + (key,)), filter_exprs) - return f"COUNT({operand})" + return self._remember(key, f"COUNT({operand})") func = AGG_TO_OSSIE_FUNC.get(mtype) if func is None: raise ConversionError( @@ -870,8 +873,20 @@ def expression(self, cname, mname, stack=()): f"measure '{scope}': type '{mtype}' requires 'sql'") operand = filtered_operand( self._operand(cname, sql, stack + (key,)), filter_exprs) - return (f"COUNT(DISTINCT {operand})" if func == "COUNT_DISTINCT" - else f"{func}({operand})") + return self._remember( + key, f"COUNT(DISTINCT {operand})" if func == "COUNT_DISTINCT" + else f"{func}({operand})") + + def _remember(self, key, expr): + """Cache one measure's expression. + + A calculated measure inlines each reference's full SQL, so a measure + referenced from several places was recomputed once per reference -- and + recursively, so a chain of them cost O(depth * 2**depth) instead of the + O(2**depth) the inlined output is inherently worth. + """ + self._cache[key] = expr + return expr def _translate(self, sql, cname, stack): """Translate a Cube SQL string, inlining any measure reference. @@ -931,7 +946,7 @@ def _is_generated_part(measure): return bool(((measure.get("meta") or {}).get("ossie") or {}).get("part_of")) -def _convert_measures(cubes, pk_by_cube, fanned_out, issues): +def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): """Hoist every cube's measures into Ossie model-level metrics. A metric name is the measure name when globally unique, else @@ -955,6 +970,7 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): extra_measures = {} seen = set() for cname, cube in cubes.items(): + plain = plain_by_cube[cname] for index, measure in enumerate( _as_named_list(cube.get("measures"), f"cube '{cname}' measures")): @@ -972,8 +988,7 @@ def _convert_measures(cubes, pk_by_cube, fanned_out, issues): f"colliding measures in Cube") seen.add(metric_name) metric = _convert_measure(cname, mname, metric_name, measure, resolver, - fanned_out, _plain_members(cube, cname), - issues) + fanned_out, plain, issues) if metric is not None: metrics.append(metric) else: diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py index 762ae00f..1e233ca5 100644 --- a/converters/cube/src/ossie_cube/osi_to_cube.py +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -641,6 +641,9 @@ def resolve_base(): mname_raw = require_str(metric, "name", "metric") scope = f"metric '{mname_raw}'" stash = read_stash(metric) + # An empty `taken` on purpose: a measure name only has to be unique within + # its own cube, and which cube this lands on is not known yet. `_place` + # rejects a collision once the target is decided. mname = stash.get("name") or sanitize_name(mname_raw, scope, set()) if "measure" in stash: diff --git a/converters/cube/tools/interop_matrix.py b/converters/cube/tools/interop_matrix.py index 2bc2ec3b..89939abd 100644 --- a/converters/cube/tools/interop_matrix.py +++ b/converters/cube/tools/interop_matrix.py @@ -86,14 +86,28 @@ def repo_root(): - for parent in [Path(__file__).resolve(), *Path(__file__).resolve().parents]: + for parent in Path(__file__).resolve().parents: if (parent / "converters").is_dir() and (parent / "core-spec").is_dir(): return parent sys.exit("cannot locate the repository root from this script's path") +# Resolving a converter's dependencies on a cold cache is the slow part; a spoke that +# has not finished by then is hung rather than working. Without a timeout one such +# spoke takes the whole run down with it and prints nothing. +_TIMEOUT_S = 600 + + def run(cwd, argv): - return subprocess.run(argv, cwd=cwd, capture_output=True, text=True) + """Run `argv` in `cwd`, or return a synthetic failure rather than raising.""" + try: + return subprocess.run(argv, cwd=cwd, capture_output=True, text=True, + timeout=_TIMEOUT_S) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess( + argv, 1, "", f"timed out after {_TIMEOUT_S}s") + except FileNotFoundError as e: + return subprocess.CompletedProcess(argv, 1, "", f"{argv[0]}: {e.strerror}") def count_warnings(stderr): @@ -121,6 +135,22 @@ def import_issues(stderr): return found +# uv's wording for "this converter's environment could not be built", which is not +# the converter rejecting the model. Matching on message text is unavoidable (uv exits +# 1 either way) and will drift, so a message that stops matching shows up as a FAIL +# with the reason in the note column rather than as a silent mislabel. +_ENV_FAILURE_MARKERS = ( + "No solution found", + "no such command", + "Failed to spawn", + "does not exist", +) + + +def _is_environment_failure(stderr): + return any(marker in stderr for marker in _ENV_FAILURE_MARKERS) + + def produced_output(dest, is_dir): if not dest.exists(): return False @@ -225,9 +255,7 @@ def main(): note = "" if r.returncode != 0: tail = (r.stderr.strip().splitlines() or [""])[-1] - # A missing environment is not the converter rejecting the model. - skipped = "No solution found" in r.stderr or "no such command" in tail - result = "SKIP" if skipped else "FAIL" + result = "SKIP" if _is_environment_failure(r.stderr) else "FAIL" note = tail[:40] failures += result == "FAIL" else: