diff --git a/.github/workflows/converter-cube-ci.yml b/.github/workflows/converter-cube-ci.yml new file mode 100644 index 00000000..a8cafb84 --- /dev/null +++ b/.github/workflows/converter-cube-ci.yml @@ -0,0 +1,63 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +name: Converters Cube CI + +on: + push: + branches: [ "main" ] + paths: + - 'converters/cube/**' + - '.github/workflows/converter-cube-ci.yml' + pull_request: + branches: [ "main" ] + paths: + - 'converters/cube/**' + - '.github/workflows/converter-cube-ci.yml' + +jobs: + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.12", "3.13", "3.14"] + + steps: + - name: Checkout project + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: ${{ matrix.python-version }} + + - name: Install uv + run: | + curl -LsSf https://astral.sh/uv/install.sh | sh + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Sync dependencies + working-directory: converters/cube + run: | + uv sync + + - name: Unit Tests + working-directory: converters/cube + run: | + uv run pytest diff --git a/converters/README.md b/converters/README.md index 5c9a4d54..ee0b6099 100644 --- a/converters/README.md +++ b/converters/README.md @@ -76,6 +76,7 @@ The Ossie specification currently defines extensions for the following vendors: | `OMNI` | Omni semantic model | | `WISDOM` | WisdomAI domain | | `NVIDIA_GSF` | NVIDIA Generative Semantic Fabric standalone YAML | +| `CUBE` | Cube data model | Each vendor may define custom extensions (via the `custom_extensions` field in the Ossie spec) to carry vendor-specific metadata that does not have an equivalent in the core specification. diff --git a/converters/cube/README.md b/converters/cube/README.md new file mode 100644 index 00000000..f3489dec --- /dev/null +++ b/converters/cube/README.md @@ -0,0 +1,413 @@ + + +# Apache Ossie <-> Cube converter + +Bidirectional, offline conversion between an [Apache Ossie](https://github.com/apache/ossie) +semantic model and a [Cube](https://cube.dev/docs/product/data-modeling/overview) +data model. No Cube deployment, API token, or network access required. + +A Cube data model is a *directory* of YAML files rather than a single document, so +this converter maps one Ossie YAML document to/from the Cube model layout: + +``` +model/cubes/.yml # one per Ossie dataset +model/views/.yml # the view the Ossie model maps to +``` + +Import accepts any layout: `cubes:` and `views:` may live in any `.yml`/`.yaml` +file at any depth, several per file, and original file paths are preserved through +a round trip. + +- **Import** (`ossie-cube import`): Cube files -> Ossie. Cube features Ossie has + no native field for are preserved in `custom_extensions[CUBE]`, so + **Cube -> Ossie -> Cube is lossless**. +- **Export** (`ossie-cube export`): Ossie -> Cube files. Ossie features with no + Cube slot are parked under `meta.ossie` rather than dropped -- Cube has a `meta` + field at every level -- so **Ossie -> Cube -> Ossie is lossless too**. + +Any input that breaks a [requirement](#requirements) **raises a +`ConversionError`** -- the converter never silently drops a field or produces an +invalid result. Losses it *can* absorb are returned as structured +[issues](#conversion-issues) rather than printed and forgotten. + +## Installation + +```bash +pip install apache-ossie-cube # once published to PyPI +# or, from a checkout of this directory: +pip install -e . +``` + +Runtime dependencies are `PyYAML` and `sqlglot` (already a runtime dependency of +the dbt and NVIDIA GSF converters, used here to locate the aggregate calls inside a +composite metric). Python 3.11+. + +## Usage + +### Command line + +```bash +ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] + [--strict-fanout] +ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] +``` + +`import` accepts a model directory (walked recursively), individual files, or any +mix of several — so converting part of a model does not mean assembling a directory +first: + +```bash +ossie-cube import -i model/ # the whole model +ossie-cube import -i model/cubes/orders.yml # one file +ossie-cube import -i model/cubes/orders.yml model/views/*.yml # a subset +``` + +Cube itself has a single model root (`CUBEJS_SCHEMA_PATH` is one path), so pointing +at that root is the idiomatic whole-project case. With several paths, files are keyed +relative to their common parent directory — which is what decides where `export` +writes them back — and the single-directory and single-file cases are keyed exactly +as they would be alone. + +With no `-o`, `import` writes the Ossie YAML to stdout; `export` always needs `-o` (a +directory). Issues always go to stderr, so stdout stays pipeable. `--view` picks +which view's name/description/AI context map onto the Ossie model when the input +holds several; `--name` overrides the model name. `--base-cube` picks the cube a +*generated* view is rooted at, and is only consulted for a hand-authored Ossie model +with no stashed views. + +**A view on its own is not a model.** A Cube view projects members from cubes and +defines none of its own, so passing only `views/sales.yml` is refused -- with an +error naming the cubes it references, so you know which files to add. Include the +cube files (or point `-i` at the model directory). + +### Python API + +```python +from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube + +ossie_yaml, issues = convert_cube_to_ossie(files) # {relative filename: YAML str} +files, issues = convert_ossie_to_cube(ossie_yaml) # -> {relative filename: YAML str} +for issue in issues: + print(issue) +``` + +## Mapping + +Each row maps in both directions; the **Notes** flag where a behavior is specific +to **import** (Cube -> Ossie) or **export** (Ossie -> Cube). + +| Apache Ossie | Cube | Notes | +|---|---|---| +| `semantic_model` | a **view** | Cube users are view-first, and Cube's agent reads `meta.ai_context` only from views and members -- so the view, not any cube, is the model boundary. | +| `semantic_model.name` | view name | Import: the mapped view's name (override with `--name`). | +| `model.description` / `ai_context.instructions` | view `description` / `meta.ai_context` | Import: taken from the sole view, or `--view`. | +| dataset | `cubes[]` entry in `model/cubes/.yml` | Import: a non-canonical original path is stashed and restored on export. | +| `dataset.source` (dotted) | `sql_table` | Passed through verbatim; Cube interpolates it straight into `FROM`, so no catalog/schema split is needed. | +| `dataset.source` (`SELECT ...`) | `sql` | Cube requires exactly one of `sql` / `sql_table`. | +| `dataset.description` | cube `description` | | +| `dataset.ai_context` | cube `meta.ai_context` | Preserved for the round trip, but **inert in Cube** -- its agent ignores cube-level `ai_context`. Recorded as an issue. | +| `dataset.primary_key` | dimension(s) with `primary_key: true` | Composite = several. Export marks a dimension only when it is **scalar** — a single source column — since `primary_key: true` declares that dimension's own `sql` to be the key; a computed dimension or a merged `geo` one would declare the wrong thing even if its name matches. Anything left uncovered becomes a `public: false` scalar dimension, suffixed (`id_pk`) if the obvious name is taken. | +| `dataset.unique_keys` | `meta.ossie.unique_keys` | No native Cube slot; parked rather than dropped. | +| field | `dimensions[]` entry | Export: a name that is not a valid Cube identifier is sanitized; a case-insensitive collision is an error, never a silent merge. | +| `field.expression` | dimension `sql` | Dataset-scoped, so `{CUBE}.col` <-> `col`. Export emits `{CUBE}.column` for a raw column and `{CUBE.member}` for a declared member, and never spells the cube's own name (which would break under `extends`). | +| `field.datatype` | dimension `type` (**required**) | `String`->`string`, `Boolean`->`boolean`, `Date`/`Time`/`DateTime`/`DateTimeTz`->`time`, `Integer`/`Decimal`/`Float`->`number`, `Opaque`->`string`. Import maps back, choosing `Decimal` for `number` -- Cube collapses three Ossie types into one, so any single answer is a guess, and a stated datatype is what another converter can act on. Export parks the exact one in `meta.ossie.datatype`, which import prefers when present, so `Integer` and `Float` still survive a round trip. | +| `field.dimension.is_time` | `type: time` | Import sets `is_time: true` for a time dimension. | +| `field.label` / `description` | dimension `title` / `description` | | +| `field.ai_context.instructions` | dimension `meta.ai_context` | Cube's documented AI-only context field. | +| — | `type: geo` dimension | An Ossie field holds one expression and a geo dimension has two, so it **splits** into `_latitude` / `_longitude` (`Float`). Reconstruction data rides on the latitude half. See [Geo dimensions](#geo-dimensions). | +| relationship | `joins[]` on a cube | `many_to_one` on cube A -> `from: A`(many), `to: B`(one). `one_to_many` is flipped so Ossie's `from` is the many side; the declared side and type are stashed so export restores the original. | +| `from_columns` / `to_columns` | join `sql` | Only an AND-chain of equalities between two member references maps. Anything else (non-equi, range, literal, third cube) is preserved verbatim in the stash. | +| metric | `measures[]` on the cube its expression references | Import hoists cube-scoped measures to the model level, qualifying a colliding name as `__` and stashing the original name and owning cube. | +| `SUM`/`AVG`/`MIN`/`MAX(x)` | `type: sum`/`avg`/`min`/`max` + `sql` | | +| `COUNT(DISTINCT x)` | `type: count_distinct` | | +| `APPROX_COUNT_DISTINCT(x)` | `type: count_distinct_approx` | Cube resolves the warehouse-specific function itself. | +| `COUNT(DISTINCT )` | bare `type: count` | See [Fan-out](#fan-out) -- the primary key is load-bearing here. | +| several aggregates in one expression | one `public: false` measure per aggregate + a `type: number` measure referencing them | Each part is declared on the cube its own operand reads, so Cube corrects row multiplication per aggregate rather than once for the whole expression. The parts carry `meta.ossie.part_of`, and import skips them and inlines their SQL back through the references -- recovering the original expression exactly. | +| anything else | `type: number` (calculated) | A `{other_measure}` reference is **inlined**, because that is what Cube itself does; Ossie has no metric-to-metric reference. | +| — | measure `filters` | Folded into `CASE WHEN … THEN … END` inside the aggregate, exactly as Cube's own `applyMeasureFilters` renders it. | +| `metric.datatype` | — | Import emits `Integer` for the count family, whose result type Cube does know, and omits it otherwise. | +| `metric.description` / `ai_context` | measure `description` / `meta.ai_context` | | +| `custom_extensions[CUBE]` | everything Cube-only | Import stashes; export restores -- keeping `Cube -> Ossie -> Cube` lossless. | +| foreign-vendor `custom_extensions` | `meta.ossie.custom_extensions` | Parked so a multi-vendor Ossie model survives the round trip. | + +**Stashed on import** (and restored on export): the views verbatim (minus the +natively mapped description/AI context), the mapped view's identity, original file +paths, cube extras (`title`, `sql_alias`, `data_source`, `public`, `refresh_key`, +`segments`, `pre_aggregations`, `hierarchies`, `access_policy`, `calendar`, ...), +dimension extras (`format`, `currency`, `granularities`, `case`, `sub_query`, +`order`, `aliases`, `meta`, ...), measure extras and any non-reconstructible +measure, joins with no Ossie form, Jinja-templated members, and files with no +Ossie form (`.js`/`.ts` models, non-model YAML). + +**Expression dialects**: Cube SQL is the SQL of the model's data source, and the +Ossie dialect enum has no `CUBE` entry -- so import emits `ANSI_SQL`, and export +prefers `ANSI_SQL` with `--dialect` prepending a warehouse dialect (e.g. +`SNOWFLAKE` for a Snowflake-backed Cube model). + +**String literals** are handled asymmetrically, on purpose. Cube compiles a YAML +`sql` value as a Python f-string (`f""` in `YamlCompiler`), so `{CUBE}.col` +interpolates *anywhere* in the value -- SQL's own quotes mean nothing to it. So on +import a reference inside a literal is a real reference and is translated, while on +export nothing is rewritten inside a literal: emitting `{CUBE.col}` there would make +Cube replace the literal's own text with a column reference. The same rule decides +which dataset a metric belongs to, so a name mentioned only inside a literal does not +attribute the metric or make it look cross-dataset. + +## Fan-out + +This is the one place where Cube carries semantics an Ossie expression cannot, and +it is handled deliberately rather than papered over. + +When a cube sits on the multiplied side of a join, Cube does **not** aggregate over +the flattened join. It builds `SELECT DISTINCT FROM `, joins +that key set back to the measure's own cube, and aggregates there -- so each source +row is counted once. If the measures themselves span cubes that fan out, Cube +refuses the query outright. Correctness comes from a *runtime rewrite keyed on +declared primary keys*, and a static SQL string has no way to inherit it. + +So the converter emits the fan-out-safe form wherever one exists, and refuses to +emit a silently-wrong one: + +| Cube measure | Ossie expression | Safe under fan-out? | +|---|---|---| +| bare `count` | `COUNT(DISTINCT )` | **Yes, exactly.** Cube renders `count(pk)` normally and `count(distinct pk)` when multiplied; `COUNT(DISTINCT pk)` equals both. A composite key is concatenated with `CAST` + `CONCAT`, as Cube does. | +| `count_distinct` | `COUNT(DISTINCT x)` | Yes, inherently | +| `count_distinct_approx` | `APPROX_COUNT_DISTINCT(x)` | Yes, inherently | +| `min` / `max` | `MIN(x)` / `MAX(x)` | Yes -- idempotent under duplication | +| `sum`, `avg`, `count` + `sql` | `SUM(x)`, `AVG(x)`, `COUNT(x)` | **No** | + +Only the last row is at risk, and only when its own cube is the `to` (one) side of +a relationship in the model. The converter computes that from the Ossie graph and +**records a `FANOUT_UNSAFE_METRIC` issue** naming the metric, the dataset and the +relationship responsible -- refusing a whole model over one such metric would leave +the spoke on the other side with nothing. Pass `--strict-fanout` to refuse instead, +mirroring Cube's own refusal. + +The issue is reported to the caller, not written into the Ossie model: the spec has +no additivity declaration to write it into (see below), and a `custom_extensions` +entry would only give every other converter something to warn about and discard. + +Because a bare `count` maps through the primary key, a cube carrying one **must** +declare `primary_key: true` on a dimension; its absence is an error, not a +different number. + +Going the other way, an Ossie metric combining several aggregates is **decomposed** +rather than emitted as one calculated measure, so Cube's correction applies to each +aggregate on its own cube: + +```yaml +# Ossie # Cube +SUM(store_sales.amount) store_sales: clv_part_1 (sum, public: false) + / COUNT(DISTINCT customer.id) customer: clv_part_2 (count, public: false) + store_sales: clv = {CUBE.clv_part_1} + / {customer.clv_part_2} +``` + +A single aggregate reading two datasets cannot be split this way and still lands on +one cube. + +> Ossie has no additivity or grain declaration to record this properly -- dbt's +> `non_additive_dimension` is the nearest precedent, and this repo's dbt converter +> already loses the same information. Worth raising on `dev@`. + +## Geo dimensions + +A Cube `type: geo` dimension carries two SQL expressions where an Ossie field carries one, so it splits on import: + +```yaml +# Cube # Ossie +- name: home - name: home_latitude (expression: lat) + type: geo - name: home_longitude (expression: lon) + latitude: { sql: "{CUBE}.lat" } + longitude: { sql: "{CUBE}.lon" } +``` + +Export merges the halves back into the single geo dimension, so the round trip is exact. + +The half names exist **only in Ossie** — Cube has neither a column nor a member called `home_latitude`. So when an Ossie metric or field expression references a half, export substitutes the half's own SQL rather than emitting a reference Cube cannot resolve: + +``` +AVG(users.home_latitude) -> sql: AVG({CUBE}.lat) +AVG(users.home_latitude) - MIN(orders.amt) -> sql: AVG({users}.lat) - MIN({CUBE.amt}) +``` + +`{CUBE}` means "the cube this is declared on", so an inlined snippet is requalified to name its original cube when it crosses into another cube's SQL. + +One documented normalization follows: after a round trip such a metric names the column the half actually reads (`users.lat`) rather than the Ossie-only field name (`users.home_latitude`). Same reference, and it is the form Cube can express. + +## Onward conversion + +Ossie is a hub, so the useful question is not only whether `Cube → Ossie → Cube` +round-trips but whether the Ossie model then reaches the other spokes. Two things +matter in practice. + +**Keep Cube-only detail out of `custom_extensions`.** Converters that do not read +foreign extensions warn about and discard every one, so anything placed there is +noise to them. This converter therefore stashes only what is genuinely Cube-specific +— segments, pre-aggregations, hierarchies, view curation, geo reconstruction — and +maps everything else natively. On the TPC-DS model that is 7 stash entries rather +than 41, and 2 Databricks warnings rather than 32. + +**Qualify your `sql_table`.** Cube accepts `orders` or `public.orders`, but the +Databricks, Snowflake and NVIDIA GSF converters all require a three-part +`catalog.schema.table` and reject anything shorter: + +``` +Error: Dataset 'orders': source 'public.orders' must be a 3-part catalog.schema.table +Error: Dataset 'orders' source must resolve to database.schema.table +Error: Source 'public.orders' must be a fully qualified db.schema.table or a subquery +``` + +Import reports this as `SOURCE_NOT_FULLY_QUALIFIED` rather than guessing a catalog +name, so it surfaces where the Ossie document is produced instead of three hops later. + +### Measuring it + +Both claims above are measurements, so they are reproducible: + +```bash +uv run tools/interop_matrix.py # the committed TPC-DS fixture +uv run tools/interop_matrix.py path/to/cube/model # any Cube model directory +uv run tools/interop_matrix.py --spokes omni --keep # one spoke, keep its output +``` + +It converts a Cube model to Ossie, checks that intermediate against the repo's own +`validation/validate.py`, then hands it to every other converter and reports what +each made of it: + +``` +model: converters/cube/tests/fixtures/tpcds_cube +Ossie: 539 lines, 7 CUBE stash entries +issues: 5x CUBE_LEVEL_AI_CONTEXT_INERT +spec: valid (validation/validate.py) + +spoke result warns foreign note +---------------------------------------------------------------------------- +databricks OK 22 2 +dbt FAIL 0 0 AttributeError: 'PydanticSemanticManifes +gooddata OK 0 0 +gsf OK 0 0 +honeydew OK 0 0 +omni OK 15 7 +orionbelt OK 2 0 +snowflake OK 7 7 +wisdom OK 47 7 +polaris -- Java converter, needs Maven +salesforce -- Java converter, needs Maven +``` + +`foreign` counts warnings that name a `custom_extensions` vendor — the cost this +converter imposes on the others by stashing, and the number to watch when deciding +whether something belongs in a stash at all. The dbt `FAIL` is unrelated to this +converter: its CLI crashes on every input, including this repo's own examples +([#296](https://github.com/apache/ossie/issues/296)). + +Each spoke runs in its own `uv` environment, so the first run resolves that +converter's dependencies; the script is stdlib-only and needs none of its own. Nothing +about it is Cube-specific except the first hop — if it is useful repo-wide it belongs +somewhere like `compliance/`, which is a question for `dev@`. + +## Conversion issues + +`convert_cube_to_ossie` returns `(yaml, IssueLog)`. Each issue carries a type, the +element it concerns, and a detail string. + +| Issue type | Meaning | +|---|---| +| `FANOUT_UNSAFE_METRIC` | A non-idempotent aggregate on a dataset the graph fans out; see [Fan-out](#fan-out) | +| `MULTI_STAGE_MEASURE_PARKED` | A `multi_stage` measure (`group_by`/`reduce_by`/`time_shift`/`rank`) renders as a window function over another grain, so it gets no `metrics` entry — the original is preserved verbatim in the dataset's stash and restored on export | +| `CUBE_LEVEL_AI_CONTEXT_INERT` | Cube's agent ignores cube-level `meta.ai_context` | +| `GEO_DIMENSION_SPLIT` | A `type: geo` dimension became two Ossie fields | +| `TEMPLATED_FILE_SKIPPED` | Jinja templating anywhere in a file, or a `.js`/`.ts` model file. Detected per file, as Cube's own tooling does, so the file is preserved whole rather than half-converted | +| `NO_USABLE_DIALECT` | Export: no `ANSI_SQL` or preferred-dialect expression | +| `SOURCE_NOT_FULLY_QUALIFIED` | A `sql_table` shorter than `catalog.schema.table`. Valid Cube and nothing is lost, but the Databricks, Snowflake and NVIDIA GSF converters reject such a source, so the model cannot convert onward — see [Onward conversion](#onward-conversion) | +| `PARKED_IN_META` | Preserved in the stash or under `meta.ossie` — invisible to Cube, but intact through a round trip | +| `DROPPED_NO_CUBE_EQUIVALENT` | **Gone from the output.** Cube has nowhere to hold it and it cannot be parked: relationship `ai_context` (a Cube join entry has no `meta`), a `dimension.is_time` role or opt-out that Cube expresses only through `type`, and the second and later `semantic_model` entries | +| `APPROXIMATED` | Emitted, but not an exact equivalent: a value Cube requires and Ossie does not carry (so the converter chose one), or a construct rendered in the nearest form Cube has | + +These three are kept distinct on purpose. A caller gating on issue types has to be +able to tell "preserved but unreadable by Cube" from "actually lost" from "emitted, +but asserting slightly more than the input did". + +## Requirements + +Conversion raises a `ConversionError` (rather than guessing or emitting something +invalid) when an input breaks one of these: + +- a cube has neither or both of `sql` / `sql_table` (Cube requires exactly one); +- a cube uses `extends` -- resolving it means reproducing Cube's definition-merge + semantics exactly, so it is refused rather than half-applied; +- a bare `type: count` measure's cube declares no primary key; +- a join names a cube that is not in the model, or an unknown `relationship`; +- a measure has an unknown `type`, or a measure reference cycle; +- two cubes, two views, or two derived metric names collide; +- a dimension has an unknown `type`, or a `geo` dimension is missing + `latitude.sql` / `longitude.sql`; +- the model carries foreign-vendor `custom_extensions` but no view is mapped, so + there is nowhere to park them (re-import with `--view `); model-level + metadata rides on the view representing the model, and picking one arbitrarily + would not survive a re-import; +- there are no convertible cubes at all; the input YAML is malformed. + +## Notes and limitations + +- **YAML data models only.** `.js`/`.ts` models and Jinja-templated YAML are + preserved verbatim for the round trip but no cube inside them is converted -- + matching what Cube's own `CubeSchemaConverter` does for the Rollup Designer. +- **camelCase is normalized.** Cube accepts `sqlTable` and `sql_table` alike; + import normalizes to snake_case and export always emits snake_case, so a + camelCase source file comes back snake_cased. +- A filter or computed operand written with bare column names (rather than + `{CUBE}.col`) cannot be qualified into `dataset.column` form, so it is emitted + as-is. Cube's own idiom uses the reference form, which converts fully. +- View curation (`prefix`, `alias`, `includes`/`excludes`, `folders`, + `default_filters`, `view_group`) is stash-and-restore only; Ossie field names + are always *cube* member names, so prefixed view members never leak into them. +- `type: switch` dimensions, `hierarchies`, `pre_aggregations`, `access_policy`, + and multiple `data_source`s have no Ossie semantics and round-trip via the stash. + +## Development + +```bash +uv sync +uv run pytest +``` + +Example-based unit tests per direction, CLI +behavior tests, fixture round-trip tests (including the +[TPC-DS model](../../examples/tpcds_semantic_model.yaml) the converter guide asks +for as a baseline), core-spec JSON Schema validation of every emitted Ossie +document, and Hypothesis property-based round-trip tests over generated Cube +models -- which fall back to a seeded sweep when `hypothesis` is unavailable, so +the properties still run. + +`tools/interop_matrix.py` checks the other half of the job — whether the Ossie this +converter emits is any use to the other spokes. It is not part of `pytest`, because +it drives the other converters' environments rather than this one's. See +[Measuring it](#measuring-it). + +## Future effort + +Both the Apache Ossie specification and Cube's data model are still evolving. As +either side adds or changes fields, this converter will be updated to track them. +Known next steps: offline `extends` resolution, `.js`/`.ts` model support (which +needs Cube's own transpiler, so most likely a Cube-side exporter feeding this +converter), and a first-class Ossie representation for measure additivity so the +fan-out caveat can be recorded in the model instead of an issue log. diff --git a/converters/cube/pyproject.toml b/converters/cube/pyproject.toml new file mode 100644 index 00000000..72a087d4 --- /dev/null +++ b/converters/cube/pyproject.toml @@ -0,0 +1,74 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "apache-ossie-cube" +version = "0.2.0.dev0" +description = "Bidirectional converter between Apache Ossie semantic models and Cube data models" +authors = [{ name = "Apache Software Foundation", email = "dev@ossie.apache.org" }] +requires-python = ">=3.11" +readme = "README.md" +license = "Apache-2.0" +keywords = [ + "Apache Ossie", + "Ossie", + "Cube", +] +classifiers = [ + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", +] +dependencies = [ + "PyYAML>=6.0", + # Expression handling: locating the aggregate calls inside a composite metric so + # each can become its own Cube measure. Already a runtime dependency of the dbt + # and NVIDIA GSF converters, which use it for the same purpose. + "sqlglot>=20.0", +] + +[dependency-groups] +dev = [ + "pytest>=8.0", + "hypothesis>=6.0", + # So the core-spec schema validation in test_roundtrip.py runs rather than + # skipping; the converter itself needs neither. + "jsonschema>=4.0", +] + +[project.scripts] +ossie-cube = "ossie_cube.cli:main" + +[project.urls] +homepage = "https://ossie.apache.org/" +repository = "https://github.com/apache/ossie/" + +[tool.hatch.build.targets.wheel] +packages = ["src/ossie_cube"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["src"] + +[tool.uv] +required-version = ">=0.9.0" +default-groups = [ + "dev", +] diff --git a/converters/cube/src/ossie_cube/__init__.py b/converters/cube/src/ossie_cube/__init__.py new file mode 100644 index 00000000..037f4162 --- /dev/null +++ b/converters/cube/src/ossie_cube/__init__.py @@ -0,0 +1,40 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Bidirectional converter between Apache Ossie semantic models and Cube data +models. Pure offline transforms: Ossie YAML string <-> {relative filename: YAML +string}. + + from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube + + ossie_yaml, issues = convert_cube_to_ossie(files) + files, issues = convert_ossie_to_cube(ossie_yaml) +""" + +from ._common import ConversionError +from .converter_issues import ConverterIssue, IssueLog, IssueType +from .cube_to_osi import convert_cube_to_ossie +from .osi_to_cube import convert_ossie_to_cube + +__all__ = [ + "ConversionError", + "ConverterIssue", + "IssueLog", + "IssueType", + "convert_cube_to_ossie", + "convert_ossie_to_cube", +] diff --git a/converters/cube/src/ossie_cube/_common.py b/converters/cube/src/ossie_cube/_common.py new file mode 100644 index 00000000..200ef0fa --- /dev/null +++ b/converters/cube/src/ossie_cube/_common.py @@ -0,0 +1,762 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared helpers for the Apache Ossie <-> Cube converters. + +Both directions are pure offline YAML transforms. The cross-cutting concerns +live here: version constants, the `custom_extensions` stash protocol, Cube +identifier rules, key-spelling normalization, the type/aggregate mapping tables, +and the member-reference translation between Cube's f-string SQL and the plain +column references Ossie expressions use. +""" + +import json +import re + +import yaml + +# Ossie semantic model spec version this converter targets (see core-spec). +OSSIE_VERSION = "0.2.0.dev0" + +# Vendor id used for the `custom_extensions` stash. +VENDOR = "CUBE" + +# Cube SQL is the SQL of the model's data source, so there is no CUBE entry in +# the Ossie dialect enum. Import emits ANSI_SQL; export prefers ANSI_SQL and lets +# the caller prepend a warehouse dialect the actual data source would accept. +DIALECT_ANSI = "ANSI_SQL" + +# Bump when the shape of a stashed `data` blob changes. +STASH_VERSION = 1 + +# Cube's default data model directory layout (`CUBEJS_SCHEMA_PATH` defaults to +# `model`, and `cube create` scaffolds these two subdirectories). +CUBE_DIR = "model/cubes" +VIEW_DIR = "model/views" + +# A valid Cube identifier -- `identifierRegex` in Cube's CubeValidator. +_CUBE_NAME_RE = re.compile(r"^[_a-zA-Z][_a-zA-Z0-9]*$") + +# A bare SQL identifier (single column reference), e.g. `c_name`. +_IDENTIFIER_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + +# `cube.member` -- a dotted reference an Ossie expression uses to point into a +# dataset. Guarded so `a.b.c` and `1.5` do not match. +DOTTED_REF_RE = re.compile( + r"(? Cube -> Ossie` lossless for models carrying several vendors. + """ + return [ + ext + for ext in (obj or {}).get("custom_extensions") or [] + if ext.get("vendor_name") != VENDOR + ] + + +# --- expressions ---------------------------------------------------------------- + +def pick_expression(ossie_expression, preferred=None): + """Choose the SQL string for an Ossie expression. + + Preference order: the caller-chosen warehouse dialect (Cube passes SQL + through to the data source, so e.g. SNOWFLAKE SQL is valid on a + Snowflake-backed Cube model), then ANSI_SQL. Returns None if neither is + present (the caller records an issue and skips). + """ + dialects = { + d.get("dialect"): d.get("expression") + for d in (ossie_expression or {}).get("dialects") or [] + } + expr = None + if preferred: + expr = dialects.get(preferred) + if expr is None: + expr = dialects.get(DIALECT_ANSI) + if expr is not None and not isinstance(expr, str): + raise ConversionError( + f"expression must be a string, got {type(expr).__name__}") + return expr + + +def synonyms_of(ai_context): + """Extract the synonyms list from an Ossie ai_context (object form only).""" + if isinstance(ai_context, dict): + return list(ai_context.get("synonyms") or []) + return [] + + +def examples_of(ai_context): + if isinstance(ai_context, dict): + return list(ai_context.get("examples") or []) + return [] + + +def instructions_of(ai_context): + """The free-text part of an Ossie ai_context: the string itself, or the + object form's `instructions`.""" + if isinstance(ai_context, str) and ai_context.strip(): + return ai_context + if isinstance(ai_context, dict): + text = ai_context.get("instructions") + if isinstance(text, str) and text.strip(): + return text + return None + + +def cube_sql_to_ossie(sql, own_cube, resolve_ref=None, self_prefix=None): + """Translate Cube member references in a SQL string to the plain references + Ossie expressions use. Returns (translated, changed). + + - `{CUBE}.col` / `{TABLE}.col` -> `col` (a raw column of the own cube) + - `{CUBE.member}` -> `member` (own-cube member reference) + - `{member}` -> `member` (same, unqualified) + - `{other.member}` -> `other.member` + - `{own_cube.member}` -> `member` + + Ossie has no field-vs-column distinction, so both flavors flatten to names. + `\\{` / `\\}` (Cube's escape for a literal brace) survive as plain braces. + + `self_prefix`, when given, qualifies own-cube references with it instead of + reducing them to a bare name -- so `{CUBE}.col` becomes `orders.col`. Ossie + field expressions are dataset-scoped and want the bare form, but model-level + metric expressions address columns as `dataset.column`, so measure conversion + passes the owning cube's name here. + + `resolve_ref`, when given, is called with each raw reference body before the + rules above are applied; returning a string uses it verbatim instead, and + returning None falls through. Measure conversion uses this to inline a + `{other_measure}` reference, which Cube resolves to that measure's own + aggregate SQL and Ossie has no reference form for. + """ + if not isinstance(sql, str): + sql = str(sql) + changed = False + protected = sql.replace("\\{", _ESC_OPEN).replace("\\}", _ESC_CLOSE) + + def repl(m): + nonlocal changed + body = m.group(1).strip() + if resolve_ref is not None: + override = resolve_ref(body) + if override is not None: + changed = True + return override + changed = True + head, _, rest = body.partition(".") + if not rest: + # A lone `{name}`: either `{CUBE}`/`{TABLE}`, the cube's own name + # spelled out, or an unqualified member reference. The first two are + # an alias that a trailing `.column` attaches to, so they are marked + # for removal along with that dot; a member name is an own-cube + # reference. + if body in _SELF_REFS or (own_cube and body == own_cube): + return _SELF_MARK + return f"{self_prefix}.{body}" if self_prefix else body + if head in _SELF_REFS or (own_cube and head == own_cube): + return f"{self_prefix}.{rest}" if self_prefix else rest + return body + + out = _CUBE_REF_RE.sub(repl, protected) + # `{CUBE}.column` -- the alias marker plus the dot the column hangs off. + out = out.replace(f"{_SELF_MARK}.", f"{self_prefix}." if self_prefix else "") + out = out.replace(_SELF_MARK, "") + out = out.replace(_ESC_OPEN, "{").replace(_ESC_CLOSE, "}") + return out, changed + + +def quoted_runs(sql): + """Split SQL into (text, is_quoted) runs, delimiters included in the quoted run. + + Used by the **export** direction only, to keep a rewrite out of string literals + and delimited identifiers. Import deliberately does not do this: a Cube YAML + `sql` is compiled as a Python f-string (`f""` in YamlCompiler), so `{CUBE}` + interpolates anywhere in the value -- SQL's own quotes are ordinary characters to + it. Skipping quoted text on the way in would therefore *lose* a reference Cube + really does resolve. + + `'`, `"` and backtick all open a run; a run is closed by its own delimiter, and + an unterminated one runs to the end (reported quoted, so nothing in it is + rewritten). SQL's `''` doubling needs no special case: it reads as a close + immediately followed by an open, leaving an empty unquoted run between. + """ + runs, buf, quote = [], [], None + for ch in str(sql): + if quote: + buf.append(ch) + if ch == quote: + runs.append(("".join(buf), True)) + buf, quote = [], None + elif ch in "'\"`": + if buf: + runs.append(("".join(buf), False)) + buf, quote = [ch], ch + else: + buf.append(ch) + if buf: + runs.append(("".join(buf), quote is not None)) + return runs + + +def sub_outside_quotes(sql, transform): + """Apply `transform` to the parts of `sql` outside quoted runs.""" + return "".join(text if quoted else transform(text) + for text, quoted in quoted_runs(sql)) + + +def referenced_datasets(expr, known): + """The dataset names an Ossie expression references, ignoring quoted text. + + Decides which cube a measure lands on and whether it crosses cubes, so a name + that only appears inside a string literal must not count -- otherwise + `SUM(orders.amount) || ' per users.id unit'` reads as a two-dataset metric and + gets attributed to the base cube rather than to `orders`. + """ + found = set() + for text, quoted in quoted_runs(expr): + if quoted: + continue + found |= {m.group(1) for m in DOTTED_REF_RE.finditer(text) + if m.group(1) in known} + return found + + +def quoted_char_mask(sql): + """One flag per character of `sql`: True where it sits inside a quoted run. + + For a caller that needs offsets into the original string rather than a rewrite. + """ + mask = [] + for text, quoted in quoted_runs(sql): + mask.extend([quoted] * len(text)) + return mask + + +def source_part_count(source): + """How many identifier parts a dotted dataset `source` has, or None for a query. + + Dots inside double quotes or backticks belong to a quoted identifier, not to the + path -- `"My.Catalog".public.t` is three parts, not four. + """ + s = str(source).strip() + if re.match(r"(?i)(select|with)\b", s): + return None + parts, quote = 1, None + for ch in s: + if quote: + if ch == quote: + quote = None + elif ch in '"`': + quote = ch + elif ch == ".": + parts += 1 + return parts + + +def sql_is_reversible(sql, plain_members=(), own_cube=None): + """True if translating this Cube SQL to Ossie and back reproduces it. + + `{CUBE}.column` / `{TABLE}.column` -- a raw physical column of the owning cube -- + always survives, because Ossie expressions address columns and the exporter + re-emits them in that form. + + A *member* reference (`{CUBE.member}`, `{member}`) survives only when the member + is **plain**: its own `sql` is just the same-named column, so the reference and + the raw column are the same thing. Otherwise Cube inlines the member's own SQL, + which a bare column name would not reproduce, and the original spelling has to be + kept. + + A **cross-cube** reference never survives: `{other.member}` is what makes Cube + add the implicit join, and the raw `{other}.column` form does not, so the two are + not interchangeable. + """ + if not isinstance(sql, str): + sql = str(sql) + plain = set(plain_members) + protected = sql.replace("\\{", "").replace("\\}", "") + for m in _CUBE_REF_RE.finditer(protected): + body = m.group(1).strip() + head, _, rest = body.partition(".") + if not rest: + if body in _SELF_REFS or (own_cube and body == own_cube): + # A bare alias only makes sense followed by `.column`. + if not protected[m.end():].startswith("."): + return False + continue + # `{member}` -- an unqualified own-cube member reference. + if body not in plain: + return False + continue + if head in _SELF_REFS or (own_cube and head == own_cube): + if rest not in plain: + return False + continue + return False # cross-cube reference; carries join semantics + return True + + +def requalify_self_refs(sql, cube_name): + """Rewrite `{CUBE}` / `{TABLE}` in a Cube SQL snippet to name `cube_name`. + + Needed when a snippet written for one cube is inlined into another cube's SQL: + `{CUBE}` means "the cube this is declared on", so it changes meaning on the + move, while `{orders}.col` is explicit and does not. + """ + return re.sub( + r"\$?\{\s*(?:CUBE|TABLE)\s*(\.\s*[A-Za-z_][A-Za-z0-9_]*\s*)?\}", + lambda m: "{" + cube_name + (m.group(1).strip() if m.group(1) else "") + "}", + str(sql), + ) + + +def ossie_expr_to_cube_sql(expr, own_cube, own_members=(), cube_names=(), + inline_sql=None): + """Rewrite an Ossie expression into Cube member-reference form. + + Only *dotted* `cube.name` references are rewritten -- a bare identifier stays + bare, because in Ossie it is a physical column of the owning dataset and + rewriting it to `{CUBE.name}` would make a member's own `sql` self-referential. + + A dotted reference resolves to whichever form Cube expects: + - `own_cube.member` where `member` is declared -> `{CUBE.member}` + (compile-time checked, and inlines the member's own SQL) + - `own_cube.column` where it is not -> `{CUBE}.column` + (a raw physical column, passed through to the database) + - `other_cube.member` -> `{other_cube.member}` + (which is also what triggers the implicit join a cross-dataset metric needs) + + The own cube is always referenced as `{CUBE}` rather than by name, so the + model keeps working when the cube is extended. Literal braces in the incoming + expression are escaped. + + `inline_sql` maps `{cube: {field: cube_sql}}` for Ossie fields that have no + addressable Cube counterpart, and whose SQL therefore has to be substituted + inline. The case that needs it is a split `geo` dimension: `location_latitude` + exists only in Ossie -- Cube has neither a column nor a member by that name -- + so a reference to it becomes the half's own SQL (`{CUBE}.lat`), requalified when + it crosses cubes. + + A dotted token inside a string literal is left alone. This matters more than it + looks: Cube compiles a YAML `sql` as a Python f-string, so a `{...}` it emitted + into a literal would be interpolated at compile time and replace the literal's + own text with a column reference. + """ + escaped = str(expr).replace("{", "\\{").replace("}", "\\}") + known = set(cube_names) + members = set(own_members) + inline = inline_sql or {} + + def repl(m): + head, name = m.group(1), m.group(2) + substitute = (inline.get(head) or {}).get(name) + if substitute is not None: + # Already-Cube SQL, so it bypasses the escaping above; `{CUBE}` inside + # it means `head`, which only stays true while head is the own cube. + return (str(substitute) if head == own_cube + else requalify_self_refs(substitute, head)) + if head == own_cube: + return "{CUBE." + name + "}" if name in members else "{CUBE}." + name + if head in known: + return "{" + head + "." + name + "}" + # Not a dataset in this model -- a genuine schema-qualified table + # reference or an unrelated dotted token. Leave it alone. + return m.group(0) + + return sub_outside_quotes(escaped, lambda run: DOTTED_REF_RE.sub(repl, run)) + + +# --- source --------------------------------------------------------------------- + +def parse_source(source, dataset_name): + """Classify an Ossie dataset `source` for placement on a Cube cube. + + Returns ("sql", sql_text) for a SELECT/WITH subquery source, or + ("sql_table", table_ref) for a table reference. Cube's `sql_table` takes the + reference verbatim (it is interpolated straight into FROM), so no splitting + into catalog/schema/table is needed -- unlike Omni, Cube has no separate + `schema` key, which also means a bare one-part table name is fine. + """ + if not source or not str(source).strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = str(source).strip() + if re.match(r"(?i)(select|with)\b", s): + return ("sql", s) + return ("sql_table", s) + + +def join_source(cube, cube_name): + """Rebuild an Ossie dataset `source` string from a Cube cube dict. + + Cube's schema requires exactly one of `sql` / `sql_table` (an `xor` in + CubeValidator), so anything else is rejected rather than guessed at. + """ + sql = cube.get("sql") + table = cube.get("sql_table") + if sql is not None and table is not None: + raise ConversionError( + f"Cube '{cube_name}': has both 'sql' and 'sql_table'; Cube allows " + f"exactly one") + if table is not None: + return str(table).strip() + if sql is not None: + return str(sql).strip() + raise ConversionError( + f"Cube '{cube_name}': has neither 'sql' nor 'sql_table' (an `extends`-only " + f"cube?); Ossie datasets require a source") + + +# --- type mapping --------------------------------------------------------------- + +# Cube dimension `type` -> Ossie `datatype`. `number` is deliberately absent: +# Cube collapses Integer/Decimal/Float into one type, and Ossie says to omit +# `datatype` when it is unknown rather than assert a precision the model does not +# have. (Cube's SQL API reports `number` as Double, but that is a wire-protocol +# floor, not a claim about the column.) `geo` is absent because such a dimension +# is split into two numeric fields. +DIM_TYPE_TO_DATATYPE = { + "string": "String", + "boolean": "Boolean", + "time": "DateTime", + "switch": "String", + # Cube collapses Integer/Decimal/Float into one type, so no mapping back is + # exact. `Decimal` is chosen over omitting a datatype because a downstream + # converter can use it: exact base-10 is the safe reading for the money and + # quantity columns `number` overwhelmingly holds, and asserting it beats + # emitting nothing plus a Cube-only extension no other spoke reads. When the + # model came from Ossie in the first place, the precise datatype is recovered + # from `meta.ossie.datatype` instead of guessed. + "number": "Decimal", +} + +# The datatype each Cube type maps back to by default. Export parks the original in +# `meta.ossie.datatype` only when it is *not* the default -- Cube cannot hold the +# distinction, and `meta.ossie` is Cube-side, so this keeps Ossie -> Cube -> Ossie +# exact without putting anything in `custom_extensions`. +DEFAULT_DATATYPE_FOR_CUBE_TYPE = dict(DIM_TYPE_TO_DATATYPE) + +# Ossie `datatype` -> Cube dimension `type`, which is required on every +# dimension. Lossy in the numeric and temporal directions by construction. +DATATYPE_TO_DIM_TYPE = { + "String": "string", + "Integer": "number", + "Decimal": "number", + "Float": "number", + "Boolean": "boolean", + "Date": "time", + "Time": "time", + "DateTime": "time", + "DateTimeTz": "time", + "Opaque": "string", +} + +# Cube measure `type` -> the Ossie aggregate function that reproduces it. +# `count` is absent: it maps through the cube's primary key, see +# primary_key_count_expression(). +AGG_TO_OSSIE_FUNC = { + "sum": "SUM", + "avg": "AVG", + "min": "MIN", + "max": "MAX", + "count_distinct": "COUNT_DISTINCT", + "count_distinct_approx": "APPROX_COUNT_DISTINCT", +} + +OSSIE_FUNC_TO_AGG = { + "SUM": "sum", + "AVG": "avg", + "MIN": "min", + "MAX": "max", + "COUNT_DISTINCT": "count_distinct", + "APPROX_COUNT_DISTINCT": "count_distinct_approx", +} + +# Cube measure types whose aggregation is written out in the `sql` itself +# (CubeSymbols.isCalculatedMeasureType). Their sql is emitted verbatim. +CALCULATED_MEASURE_TYPES = frozenset({"number", "string", "boolean", "time"}) + +# Aggregates whose value is unaffected by duplicate input rows, so a static Ossie +# expression stays correct even when the relationship graph fans the dataset out. +# `count` belongs here only in its bare form, which maps to COUNT(DISTINCT ). +FANOUT_SAFE_AGGS = frozenset({ + "count_distinct", "count_distinct_approx", "min", "max", +}) + +# Aggregates that over-count under row multiplication. Cube corrects for these at +# query time by deduplicating on the primary key; an Ossie expression cannot. +FANOUT_UNSAFE_AGGS = frozenset({"sum", "avg"}) + +# The Ossie result datatype Cube itself declares for each aggregate. Only the +# count family is listed: those are exactly the aggregates whose result type does +# not depend on the operand. +AGG_TO_RESULT_DATATYPE = { + "count": "Integer", + "count_distinct": "Integer", + "count_distinct_approx": "Integer", +} + + +def primary_key_operand(cube_name, primary_keys): + """The single scalar expression standing for a cube's primary key. + + A composite key is concatenated the same way Cube does it (CAST + CONCAT, in + `primaryKeyCount`); both are REQUIRED functions in the Ossie expression + language, so the result stays portable. + """ + if not primary_keys: + raise ConversionError( + f"Cube '{cube_name}': a bare `type: count` measure needs the cube's " + f"primary key to convert safely, but no dimension declares " + f"`primary_key: true`") + if len(primary_keys) == 1: + return f"{cube_name}.{primary_keys[0]}" + parts = ", ".join(f"CAST({cube_name}.{pk} AS VARCHAR)" for pk in primary_keys) + return f"CONCAT({parts})" + + +def primary_key_count_expression(cube_name, primary_keys, filter_exprs=()): + """The Ossie expression for Cube's bare `type: count` measure. + + Cube renders such a measure as `count()` normally and + `count(distinct )` when the cube sits on the multiplied side of a join + (BaseQuery `primaryKeyCount`). `COUNT(DISTINCT )` equals both -- a primary + key is unique, so the DISTINCT is free when there is no fan-out and + load-bearing when there is -- making it the one static form that is correct in + every join context. + """ + operand = filtered_operand(primary_key_operand(cube_name, primary_keys), + filter_exprs) + return f"COUNT(DISTINCT {operand})" + + +def filtered_operand(operand, filter_sqls): + """Fold Cube measure `filters` into the operand, the way Cube itself does. + + Cube's `applyMeasureFilters` wraps the operand as + `CASE WHEN THEN END` inside the aggregate, + which is the filtered-aggregation idiom the Ossie expression language + endorses. The `ELSE` is omitted, matching Cube. + """ + if not filter_sqls: + return operand + where = " AND ".join(f"({f})" for f in filter_sqls) + return f"CASE WHEN {where} THEN {operand} END" diff --git a/converters/cube/src/ossie_cube/cli.py b/converters/cube/src/ossie_cube/cli.py new file mode 100644 index 00000000..d665be0c --- /dev/null +++ b/converters/cube/src/ossie_cube/cli.py @@ -0,0 +1,190 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line interface for the Apache Ossie <-> Cube converter. + + ossie-cube import -i model/ [-o model.yaml] [--name my_model] [--view sales] + ossie-cube import -i cubes/orders.yml cubes/users.yml views/sales.yml + ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] + +`import` converts a Cube data model (any `.yml` holding `cubes:` / `views:`) into an +Apache Ossie semantic model; with no `-o` the Ossie YAML goes to stdout. It accepts +a model directory, individual files, or a mix of several -- so converting part of a +model does not mean assembling a directory first. `export` does the reverse and +always needs `-o` (a directory). Conversions that could not carry something across +print an issue list to stderr. + +A metric whose value a static Ossie expression cannot keep correct under row +multiplication is converted with a `FANOUT_UNSAFE_METRIC` issue naming the metric, +the dataset and the relationship responsible -- a hub-and-spoke converter that +refuses a whole model over one such metric is not much use to the spoke on the +other side. Pass `--strict-fanout` to refuse instead, mirroring Cube's own refusal +to answer such a query. +""" + +import argparse +import os +import sys + +from ._common import ConversionError +from .cube_to_osi import convert_cube_to_ossie +from .osi_to_cube import convert_ossie_to_cube + + +def _build_parser(): + parser = argparse.ArgumentParser( + prog="ossie-cube", description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command") + sub.required = True + + imp = sub.add_parser( + "import", help="Cube data model directory -> Apache Ossie semantic model YAML") + imp.add_argument("-i", "--input", required=True, nargs="+", + metavar="PATH", + help="Cube model directories and/or files. A directory is " + "walked recursively; several paths are merged, keyed " + "relative to their common parent (globs work)") + imp.add_argument("-o", "--output", + help="output Ossie YAML file (default: stdout)") + imp.add_argument("--name", + help="Ossie model name (default: the mapped view's name)") + imp.add_argument("--view", + help="view whose name/description/AI context map onto the " + "Ossie model (default: the sole view, if there is one)") + imp.add_argument("--strict-fanout", dest="strict_fanout", + action="store_true", default=False, + help="refuse the conversion when a metric is fan-out-unsafe, " + "instead of converting it and recording an issue") + + exp = sub.add_parser( + "export", help="Apache Ossie semantic model -> Cube data model directory") + exp.add_argument("-i", "--input", required=True, help="Ossie YAML file") + exp.add_argument("-o", "--output", required=True, + help="output directory for the Cube model files") + exp.add_argument("-d", "--dialect", + help="preferred Ossie expression dialect (e.g. SNOWFLAKE); " + "ANSI_SQL is always the fallback") + exp.add_argument("-b", "--base-cube", + help="dataset a generated view is rooted at (only used for a " + "model with no stashed views; default: the FK-sink dataset)") + return parser + + +def _read_model_input(paths): + """Collect a Cube model as {relative path: text} from one or more paths. + + Cube itself has a single model root (`CUBEJS_SCHEMA_PATH`, one string), so + pointing at a model directory is the idiomatic whole-project case. But + converting part of a model -- two cubes out of fifty, or files that live in + different trees -- is a real workflow, so several paths merge into one model + rather than forcing the caller to assemble a directory first. + + Keys are relative to the deepest directory containing every input, which + leaves the single-directory and single-file cases keyed exactly as before. + Directories are walked recursively, collecting everything rather than only + YAML: a `.js` data model has no Ossie form, but the converter preserves it so a + round trip does not lose the file. Hidden files and directories (including + `node_modules`) are skipped. + """ + resolved = [os.path.abspath(p) for p in paths] + for path, original in zip(resolved, paths): + if not os.path.exists(path): + raise ConversionError(f"'{original}' is not a file or directory") + + # The anchor keys every file. Using the inputs' common parent means one + # directory anchors to itself and one file to its own directory, so those + # cases are unchanged; several inputs stay distinguishable from each other. + containers = [p if os.path.isdir(p) else os.path.dirname(p) for p in resolved] + try: + anchor = os.path.commonpath(containers) + except ValueError: + # No shared prefix at all (different drives on Windows); fall back to bare + # file names, which are still unique or else reported as a collision below. + anchor = None + + files = {} + for path in resolved: + if os.path.isfile(path): + _collect_file(files, path, anchor) + continue + for dirpath, dirnames, filenames in os.walk(path): + dirnames[:] = [d for d in sorted(dirnames) + if not d.startswith(".") and d != "node_modules"] + for fname in sorted(filenames): + if not fname.startswith("."): + _collect_file(files, os.path.join(dirpath, fname), anchor) + if not files: + raise ConversionError( + f"{', '.join(repr(p) for p in paths)} holds no files") + return dict(sorted(files.items())) + + +def _collect_file(files, path, anchor): + rel = (os.path.basename(path) if anchor is None + else os.path.relpath(path, anchor)).replace(os.sep, "/") + if rel in files: + raise ConversionError( + f"two inputs both resolve to '{rel}'; pass their common parent " + f"directory instead, or rename one") + with open(path) as fh: + files[rel] = fh.read() + + +def _report(issues): + if not len(issues): + return + print(f"{len(issues)} conversion issue(s):", file=sys.stderr) + for issue in issues: + print(f" {issue}", file=sys.stderr) + + +def main(argv=None): + args = _build_parser().parse_args(argv) + try: + if args.command == "export": + with open(args.input) as fh: + ossie_yaml = fh.read() + files, issues = convert_ossie_to_cube( + ossie_yaml, dialect=args.dialect, base_cube=args.base_cube) + for rel, text in files.items(): + dest = os.path.join(args.output, *rel.split("/")) + os.makedirs(os.path.dirname(dest) or ".", exist_ok=True) + with open(dest, "w") as fh: + fh.write(text) + print(f"Wrote {len(files)} file(s) to {args.output}", file=sys.stderr) + _report(issues) + return 0 + + files = _read_model_input(args.input) + out, issues = convert_cube_to_ossie( + files, model_name=args.name, view=args.view, + strict_fanout=args.strict_fanout) + if args.output: + with open(args.output, "w") as fh: + fh.write(out) + else: + sys.stdout.write(out) + _report(issues) + except (ConversionError, OSError) as e: + print(f"Error: {e}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/cube/src/ossie_cube/converter_issues.py b/converters/cube/src/ossie_cube/converter_issues.py new file mode 100644 index 00000000..d0285cd7 --- /dev/null +++ b/converters/cube/src/ossie_cube/converter_issues.py @@ -0,0 +1,133 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Structured record of what a conversion could not carry across. + +A bare `warnings.warn` is fine for "this label was dropped", but Cube carries +semantics that an Apache Ossie expression string genuinely cannot hold -- most +importantly the row-multiplication correction Cube applies at query time (see +`FANOUT_UNSAFE_METRIC`). Those need to reach the caller as data, not as text on +stderr, so a pipeline can gate on them. Same approach as the osi-dbt converter's +`ConverterIssue`. +""" + +from dataclasses import dataclass, field +from enum import Enum + + +class IssueType(Enum): + """Identifies the kind of information loss that occurred during conversion.""" + + # A non-idempotent aggregate (sum/avg, or count over an expression) on a + # dataset that the relationship graph can fan out. Cube corrects for this at + # query time by deduplicating on the primary key; a static Ossie expression + # cannot, so a downstream consumer may over-count. See README "Fan-out". + FANOUT_UNSAFE_METRIC = "FANOUT_UNSAFE_METRIC" + + # A `multi_stage` measure (group_by / reduce_by / time_shift / rank). These + # render as window functions over a grain other than the query's, which an Ossie + # expression has no form for -- so the measure gets no `metrics` entry, and the + # original is preserved verbatim in the owning dataset's stash instead. + MULTI_STAGE_MEASURE_PARKED = "MULTI_STAGE_MEASURE_PARKED" + + # A cube-level `meta.ai_context`. Cube's own agent only consumes ai_context + # on views and on individual members, so this value is inert in Cube; it is + # preserved so the round trip stays lossless. + CUBE_LEVEL_AI_CONTEXT_INERT = "CUBE_LEVEL_AI_CONTEXT_INERT" + + # A `type: geo` dimension, split into two Ossie fields (latitude/longitude) + # because an Ossie field holds a single expression. + GEO_DIMENSION_SPLIT = "GEO_DIMENSION_SPLIT" + + # A file with no static form -- Jinja templating anywhere in it, or a `.js` / + # `.ts` data model needing Cube's transpiler. Detected per file (as Cube's own + # CubeSchemaConverter does), so the whole file is preserved verbatim in the + # stash rather than half-converted. + TEMPLATED_FILE_SKIPPED = "TEMPLATED_FILE_SKIPPED" + + # An Ossie field or metric with no usable expression dialect (export). + NO_USABLE_DIALECT = "NO_USABLE_DIALECT" + + # A dataset `source` that is a valid Cube `sql_table` but not a three-part + # `catalog.schema.table`. Nothing is lost and Cube is happy, but several other + # Ossie converters reject such a source outright, so the model will not travel + # past this hub. Reported so that is discovered here rather than downstream. + SOURCE_NOT_FULLY_QUALIFIED = "SOURCE_NOT_FULLY_QUALIFIED" + + # An Ossie construct Cube has no slot for, parked under `meta.ossie` -- so the + # value survives the round trip even though Cube itself cannot read it. + PARKED_IN_META = "PARKED_IN_META" + + # A value Cube has nowhere to hold *and* that cannot be parked, so it is gone + # from the output. Distinct from PARKED_IN_META on purpose: a caller gating on + # issue types has to be able to tell "preserved but invisible to Cube" from + # "actually lost". + DROPPED_NO_CUBE_EQUIVALENT = "DROPPED_NO_CUBE_EQUIVALENT" + + # Something *was* emitted, but it is not an exact equivalent: a value Cube + # requires and Ossie does not carry (so the converter had to choose one), or a + # construct rendered in the nearest form Cube has. Nothing is lost and nothing + # is hidden -- but the output asserts a little more than the input did, so it + # is worth a look. + APPROXIMATED = "APPROXIMATED" + + +@dataclass(frozen=True) +class ConverterIssue: + """One instance of information loss, addressed to a named element.""" + + issue_type: IssueType + element_name: str + detail: str = "" + + def __str__(self): + suffix = f": {self.detail}" if self.detail else "" + return f"[{self.issue_type.value}] {self.element_name}{suffix}" + + +@dataclass +class IssueLog: + """Collects issues during a conversion. + + `strict_types` names the issue types that should abort the conversion instead of + being recorded. Nothing is in there by default: a converter that refuses a whole + model over one metric leaves the spoke on the other side with nothing. Passing + `--strict-fanout` adds `FANOUT_UNSAFE_METRIC`, mirroring Cube's own refusal to + answer a query whose measures reference cubes that lead to row multiplication. + """ + + issues: list = field(default_factory=list) + strict_types: frozenset = frozenset() + + def add(self, issue_type, element_name, detail=""): + issue = ConverterIssue(issue_type, element_name, detail) + if issue_type in self.strict_types: + # Imported here to avoid a circular import at module load. + from ._common import ConversionError + + raise ConversionError(f"{issue} (refused under strict mode)") + self.issues.append(issue) + return issue + + def of_type(self, issue_type): + return [i for i in self.issues if i.issue_type is issue_type] + + def __len__(self): + return len(self.issues) + + def __iter__(self): + return iter(self.issues) diff --git a/converters/cube/src/ossie_cube/cube_to_osi.py b/converters/cube/src/ossie_cube/cube_to_osi.py new file mode 100644 index 00000000..7e5bbedc --- /dev/null +++ b/converters/cube/src/ossie_cube/cube_to_osi.py @@ -0,0 +1,1066 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert a Cube data model to an Apache Ossie semantic model. + +Pure offline conversion -- no Cube deployment required. Accepts a Cube model +directory as {relative filename: YAML string}: any `.yml`/`.yaml` file holding +top-level `cubes:` and/or `views:`. Cubes become Ossie datasets, cube joins +become relationships, cube measures are hoisted to model-level metrics, and the +mapped view supplies the model's name, description, and AI context. + +Cube features Ossie has no native field for (segments, pre-aggregations, +hierarchies, folders, view curation, formats, access policies, ...) are preserved +in `custom_extensions[CUBE]` so that converting back reproduces the original +files. See README.md. + +Usage (CLI): + ossie-cube import -i model/ [-o model.yaml] [--name NAME] [--view VIEW] +""" + +import re + +from ._common import ( + AGG_TO_OSSIE_FUNC, + AGG_TO_RESULT_DATATYPE, + CALCULATED_MEASURE_TYPES, + DIALECT_ANSI, + DIM_TYPE_TO_DATATYPE, + DOTTED_REF_RE, + FANOUT_UNSAFE_AGGS, + JINJA_RE, + OSSIE_VERSION, + ConversionError, + cube_file, + cube_sql_to_ossie, + dump_yaml, + filtered_operand, + is_simple_identifier, + join_source, + load_yaml, + primary_key_count_expression, + require_str, + snake, + snake_keys, + source_part_count, + sql_is_reversible, + view_file, + read_stash, + write_stash, +) +from .converter_issues import IssueLog, IssueType +from .expressions import has_top_level_operator + +# Cube keys the converter maps natively at the cube level; everything else is +# stashed verbatim in the dataset's `cube_extras` and restored on export. +_CUBE_NATIVE_KEYS = frozenset({ + "name", "sql", "sql_table", "description", "dimensions", "measures", + "joins", "meta", +}) + +# Dimension keys mapped natively; the rest stash flat on the field. +_DIM_NATIVE_KEYS = frozenset({ + "name", "sql", "type", "primary_key", "title", "description", "meta", + "latitude", "longitude", +}) + +# Measure keys an Ossie metric represents natively. Any other key forces the +# full-measure stash, because export could not rebuild the measure without it. +_MEASURE_NATIVE_KEYS = frozenset({ + "name", "sql", "type", "filters", "title", "description", "meta", +}) + +# `relationship` values, normalized. Cube accepts the legacy `belongsTo` / +# `hasMany` / `hasOne` spellings alongside the modern ones, in either case style. +_RELATIONSHIP_ALIASES = { + "belongs_to": "many_to_one", + "many_to_one": "many_to_one", + "has_many": "one_to_many", + "one_to_many": "one_to_many", + "has_one": "one_to_one", + "one_to_one": "one_to_one", +} + +_AND_SPLIT_RE = re.compile(r"\s+AND\s+", re.IGNORECASE) + + +def convert_cube_to_ossie(files, model_name=None, view=None, strict_fanout=False): + """Convert Cube model files ({relative filename: YAML str}) to Ossie YAML. + + Returns (ossie_yaml_str, IssueLog). `model_name` overrides the Ossie model + name (default: the mapped view's name, else 'cube_model'). `view` names the + view whose name/description/AI context map onto the Ossie model when the + directory holds more than one. + + A metric whose value a static Ossie expression cannot keep correct under row + multiplication is converted with a FANOUT_UNSAFE_METRIC issue; `strict_fanout` + refuses it instead -- see README "Fan-out". + """ + if not isinstance(files, dict) or not files: + raise ConversionError("expected a non-empty mapping of {filename: YAML}") + + strict = {IssueType.FANOUT_UNSAFE_METRIC} if strict_fanout else set() + issues = IssueLog(strict_types=frozenset(strict)) + + cubes, cube_paths, views, view_paths, extra_files = _collect(files, issues) + if not cubes: + raise ConversionError(_no_cubes_message(views)) + + # The mapped view supplies the Ossie model's identity. Cube users are + # view-first, and Cube's own agent reads `meta.ai_context` only from views and + # individual members -- so the view, not any cube, is the model boundary. + mapped_name = _pick_view(views, view, issues) + mapped_view = views.get(mapped_name) or {} + cubes = _order_by_view(cubes, mapped_view) + + model = {"name": model_name or mapped_name or "cube_model"} + if mapped_view.get("description"): + model["description"] = mapped_view["description"] + ai = _ai_context_from_meta(mapped_view.get("meta")) + if ai: + model["ai_context"] = ai + + # Anything a cube's stash has to carry is worked out before the dataset is + # built: joins with no Ossie form, and measures with no static Ossie + # expression. Primary keys are read straight off the dimensions so this + # ordering does not depend on the datasets existing yet. + relationships, extra_joins = _convert_joins(cubes, sorted(extra_files), issues) + if relationships: + model["relationships"] = relationships + fanned_out = _fanned_out_datasets(relationships) + pk_by_cube = {cname: _primary_key_of(cube, cname) + for cname, cube in cubes.items()} + # Which members regenerate from a bare column name, worked out once per cube: + # both the measure and the dimension stage need the same answer. + plain_by_cube = {cname: _plain_members(cube, cname) + for cname, cube in cubes.items()} + + metrics, extra_measures = _convert_measures( + cubes, pk_by_cube, plain_by_cube, fanned_out, issues) + + model["datasets"] = [ + _convert_cube(cname, cube, plain_by_cube[cname], extra_joins.get(cname), + extra_measures.get(cname), issues) + for cname, cube in cubes.items() + ] + if metrics: + model["metrics"] = metrics + + # Model-level stash: the views verbatim (minus natively mapped properties), + # the mapped view's identity, non-canonical file paths, and any file with no + # Ossie form. `views` is stashed even when empty, so a lossless re-export does + # not invent a view the original model never had. + stash = {"views": {}} + for vname, vdict in views.items(): + vdict = dict(vdict) + if vname == mapped_name: + vdict.pop("description", None) + leftover = _meta_without_ai_context(vdict.get("meta")) + vdict.pop("meta", None) + if leftover: + vdict["meta"] = leftover + stash["views"][vname] = vdict + off_layout_views = {v: p for v, p in view_paths.items() if p != view_file(v)} + if off_layout_views: + stash["view_files"] = off_layout_views + off_layout_cubes = {c: p for c, p in cube_paths.items() if p != cube_file(c)} + if off_layout_cubes: + stash["cube_files"] = off_layout_cubes + if mapped_name is not None: + stash["mapped_view"] = mapped_name + if extra_files: + stash["extra_files"] = extra_files + write_stash(model, stash) + + # Foreign-vendor extensions a previous export parked on the mapped view are + # restored after the stash is written, so the CUBE entry stays first. + _restore_parked_extensions(model, mapped_view.get("meta")) + + return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}), issues + + +# --- collection ----------------------------------------------------------------- + +def _collect(files, issues): + """Partition the input files into cubes, views, and everything else.""" + cubes, views = {}, {} + cube_paths, view_paths = {}, {} + extra_files = {} + for fname in sorted(files): + text = files[fname] + if not fname.lower().endswith((".yml", ".yaml")): + # A `.js`/`.ts` data model needs Cube's own transpiler and a `.py` one + # is Jinja-driven. Preserved verbatim so the round trip keeps the file, + # but no cube inside it is converted. + issues.add(IssueType.TEMPLATED_FILE_SKIPPED, fname, + "not a YAML data model; preserved in custom_extensions only") + extra_files[fname] = text + continue + if JINJA_RE.search(text): + issues.add(IssueType.TEMPLATED_FILE_SKIPPED, fname, + "uses Jinja templating, which has no static form; " + "preserved in custom_extensions only") + extra_files[fname] = text + continue + parsed = load_yaml(text, fname) + if not isinstance(parsed, dict) or not ("cubes" in parsed or "views" in parsed): + issues.add(IssueType.PARKED_IN_META, fname, + "no top-level `cubes:` or `views:`; preserved in " + "custom_extensions only") + extra_files[fname] = text + continue + for entry in _as_named_list(parsed.get("cubes"), f"'{fname}' cubes"): + name = require_str(entry, "name", f"'{fname}': cube") + if name in cubes: + raise ConversionError( + f"cube '{name}' is defined twice " + f"('{cube_paths[name]}' and '{fname}')") + if "extends" in entry: + # Resolving `extends` means reproducing Cube's definition-merge + # semantics exactly; refused rather than half-applied. + raise ConversionError( + f"cube '{name}' uses `extends`, which this converter does not " + f"resolve yet; flatten the cube or exclude the file") + cubes[name] = entry + cube_paths[name] = fname + for entry in _as_named_list(parsed.get("views"), f"'{fname}' views"): + name = require_str(entry, "name", f"'{fname}': view") + if name in views: + raise ConversionError( + f"view '{name}' is defined twice " + f"('{view_paths[name]}' and '{fname}')") + views[name] = entry + view_paths[name] = fname + return cubes, cube_paths, views, view_paths, extra_files + + +def _as_named_list(value, what): + """Normalize a Cube collection to a list of dicts carrying `name`. + + YAML data models write `cubes:` / `dimensions:` / `joins:` as lists whose + entries carry a `name`; the JavaScript form (and Cube's post-transpile schema) + uses a mapping keyed by name. Both are accepted, and keys are normalized to + snake_case so the mapping code only has to know one spelling. + """ + if value is None: + return [] + if isinstance(value, list): + out = [] + for entry in value: + if not isinstance(entry, dict): + raise ConversionError( + f"{what}: expected a mapping, got {type(entry).__name__}") + out.append(snake_keys(entry)) + return out + if isinstance(value, dict): + out = [] + for name, entry in value.items(): + entry = snake_keys(entry or {}) + entry.setdefault("name", name) + out.append(entry) + return out + raise ConversionError( + f"{what}: expected a list or mapping, got {type(value).__name__}") + + +def _cubes_referenced_by(view): + """The cube names a view's `cubes:` entries address, in order. + + Every segment of a `join_path` names a cube (`orders.users.addresses` reaches + three), so all of them count as referenced. + """ + names = [] + for entry in view.get("cubes") or []: + if not isinstance(entry, dict): + continue + path = entry.get("join_path") + if not isinstance(path, str) or not path: + continue + for segment in path.split("."): + if segment and segment not in names: + names.append(segment) + return names + + +def _no_cubes_message(views): + """Explain *why* there is nothing to convert. + + Being handed only view files is an easy mistake -- a Cube view looks like a + complete model, and it is what a view-first user thinks of as "the model". But a + view only projects members from cubes and defines none of its own, so it cannot + become an Ossie semantic model on its own. Naming the cubes it references turns + the error into instructions. + """ + if not views: + return ("no convertible cubes found (a `.yml` file with a top-level " + "`cubes:` list); nothing to convert") + referenced = [] + for view in views.values(): + for name in _cubes_referenced_by(view): + if name not in referenced: + referenced.append(name) + which = ", ".join(f"'{v}'" for v in sorted(views)) + needed = ( + f" It references {', '.join(repr(c) for c in referenced)}, so include the " + f"file(s) defining those cubes." + if referenced else + " Include the files defining the cubes it draws from." + ) + return ( + f"found only view(s) {which} and no cubes. A Cube view projects members " + f"from cubes rather than defining any, so it has no Ossie dataset to " + f"convert on its own.{needed}" + ) + + +def _order_by_view(cubes, mapped_view): + """Order the datasets the way the mapped view presents them. + + The view is the model boundary, so its `cubes:` order is the order a Cube user + sees -- and carrying it over means the Ossie dataset order is meaningful rather + than an artifact of how the files happened to be named. A cube the view does + not include keeps its file position, after the ones it does. + """ + ranks = {} + for entry in mapped_view.get("cubes") or []: + if not isinstance(entry, dict): + continue + path = entry.get("join_path") + if not isinstance(path, str) or not path: + continue + leaf = path.split(".")[-1] + ranks.setdefault(leaf, len(ranks)) + if not ranks: + return cubes + order = sorted(cubes, key=lambda name: (ranks.get(name, len(ranks)),)) + return {name: cubes[name] for name in order} + + +def _pick_view(views, requested, issues): + if requested is not None: + if requested not in views: + raise ConversionError( + f"requested view '{requested}' not found; views present: " + f"{sorted(views) or 'none'}") + return requested + if len(views) == 1: + return next(iter(views)) + if len(views) > 1: + issues.add(IssueType.PARKED_IN_META, "model", + f"{len(views)} views found and none chosen with --view; view " + f"metadata is preserved in custom_extensions only") + return None + + +# --- ai_context ----------------------------------------------------------------- + +def _ai_context_from_meta(meta): + """Build an Ossie `ai_context` from a Cube `meta`. + + `meta.ai_context` is Cube's documented AI-only context field. A structured + copy parked by a previous export under `meta.ossie.ai_context` wins, since it + carries the synonyms/examples lists that the prose form flattens. + """ + if not isinstance(meta, dict): + return None + parked = (meta.get("ossie") or {}).get("ai_context") + if parked: + return parked + text = meta.get("ai_context") + if isinstance(text, str) and text.strip(): + # Kept verbatim rather than stripped: a folded block scalar carries a + # trailing newline, and normalizing it away here would make the round trip + # lossy for the sake of cosmetics. + return {"instructions": text} + return None + + +def _meta_without_ai_context(meta): + """The part of a Cube `meta` with no Ossie home, for the stash. + + `meta.ossie` is this converter's own parking spot; its contents are restored + into native Ossie fields, so it never rides in the stash. + """ + if not isinstance(meta, dict): + return {} + return {k: v for k, v in meta.items() if k not in ("ai_context", "ossie")} + + +def _fanned_out_datasets(relationships): + """{dataset: relationship name} for datasets a join can multiply rows of. + + A dataset on the `to` (one) side of a many-to-one join is fanned out by rows from + the `from` (many) side. A **one-to-one** join multiplies neither side, so it is + excluded -- otherwise a perfectly safe `sum` on either side would be refused + under strict fan-out mode. The cardinality comes from the stash Cube's join left + behind, in normalized form, so `one_to_one` and the legacy `has_one` both count. + + A hand-authored Ossie relationship carries no Cube cardinality, and Ossie's own + `from`/`to` says only many/one -- so it keeps the conservative assumption. + """ + out = {} + for rel in relationships: + declared = read_stash(rel).get("relationship") + if declared and _RELATIONSHIP_ALIASES.get(snake(declared)) == "one_to_one": + continue + out[rel["to"]] = rel["name"] + return out + + +def _restore_parked_extensions(obj, meta): + """Reattach foreign-vendor extensions a previous export parked under + `meta.ossie.custom_extensions`. + + Called after `write_stash`, so the CUBE entry stays first and the restored + foreign entries follow -- the ordering datasets already used. Without this the + parked entries are stripped by `_meta_without_ai_context` and never come back, + which would make `Ossie -> Cube -> Ossie` lose them. + """ + parked = ((meta or {}).get("ossie") or {}).get("custom_extensions") + if parked: + obj.setdefault("custom_extensions", []).extend(parked) + + +# --- cubes ---------------------------------------------------------------------- + +def _plain_members(cube, cname): + """Dimension names whose `sql` is just the same-named column. + + For those, `{CUBE.member}`, `{CUBE}.member` and a bare `member` all mean the + same thing, so the spelling carries no information worth stashing. Any other + member inlines its own SQL when referenced, which a column name would not + reproduce. + """ + plain = set() + for dim in _as_named_list(cube.get("dimensions"), f"cube '{cname}' dimensions"): + name = dim.get("name") + sql = dim.get("sql") + if name and (sql is None or str(sql).strip() == name): + plain.add(name) + return plain + + +def _primary_key_of(cube, cname): + """The names of a cube's `primary_key: true` dimensions. + + Read directly off the dimensions so the stages that need it -- measures, and the + fan-out check -- do not have to wait for the dataset to be built. + """ + return [require_str(dim, "name", f"cube '{cname}': dimension") + for dim in _as_named_list(cube.get("dimensions"), + f"cube '{cname}' dimensions") + if dim.get("primary_key")] + + +def _convert_cube(cname, cube, plain, extra_joins, extra_measures, issues): + """Build one Ossie dataset from a Cube cube.""" + scope = f"cube '{cname}'" + ds = {"name": cname} + stash = {} + + ds["source"] = join_source(cube, cname) + parts = source_part_count(ds["source"]) + if parts is not None and parts < 3: + # Cube accepts a one- or two-part `sql_table`, but the Ossie spec describes + # `source` as `database.schema.table` and the Databricks, Snowflake and NVIDIA + # GSF converters all reject anything shorter -- so a model that converts + # cleanly here still cannot reach them. Better to say so at the point the + # Ossie document is produced than to have it fail three hops later. + issues.add(IssueType.SOURCE_NOT_FULLY_QUALIFIED, scope, + f"source '{ds['source']}' has {parts} part(s); several Ossie " + f"converters (Databricks, Snowflake, NVIDIA GSF) require a " + f"3-part catalog.schema.table, so qualify the cube's `sql_table` " + f"if the model needs to convert onward") + if cube.get("description"): + ds["description"] = cube["description"] + + meta = cube.get("meta") if isinstance(cube.get("meta"), dict) else {} + parked = meta.get("ossie") or {} + ai = _ai_context_from_meta(meta) + if ai: + ds["ai_context"] = ai + if meta.get("ai_context"): + issues.add(IssueType.CUBE_LEVEL_AI_CONTEXT_INERT, scope, + "Cube's agent reads ai_context only on views and members, " + "so a cube-level value has no effect in Cube") + if parked.get("unique_keys"): + ds["unique_keys"] = [list(k) for k in parked["unique_keys"]] + + fields = [] + for dim in _as_named_list(cube.get("dimensions"), f"{scope} dimensions"): + dname = require_str(dim, "name", f"{scope}: dimension") + fields.extend(_convert_dimension(cname, dname, dim, plain, issues)) + if fields: + ds["fields"] = fields + primary_key = _primary_key_of(cube, cname) + if primary_key: + ds["primary_key"] = primary_key + if extra_joins: + stash["extra_joins"] = extra_joins + if extra_measures: + # Measures with no static Ossie expression (multi-stage ones) ride here with + # their original positions, so export can put them back among the measures it + # rebuilds from metrics. Without this they would be lost outright: `measures` + # is a natively-mapped key, so `cube_extras` does not carry it. + stash["extra_measures"] = extra_measures + + extras = {snake(k): v for k, v in cube.items() + if snake(k) not in _CUBE_NATIVE_KEYS} + leftover_meta = _meta_without_ai_context(cube.get("meta")) + if leftover_meta: + extras["meta"] = leftover_meta + if extras: + stash["cube_extras"] = extras + write_stash(ds, stash) + + # Foreign-vendor extensions parked by a previous export are restored after the + # stash is written, so the CUBE entry stays first and both survive. + _restore_parked_extensions(ds, cube.get("meta")) + return ds + + +def _convert_dimension(cname, dname, dim, plain, issues): + """Build the Ossie field(s) for one Cube dimension. + + Returns a list because a `type: geo` dimension carries two SQL expressions + (latitude and longitude) where an Ossie field holds one, so it splits into two + fields. Every other dimension yields exactly one. + """ + dtype = snake(dim.get("type") or "string") + if dtype == "geo": + return _convert_geo_dimension(cname, dname, dim, issues) + + stash = {} + sql = dim.get("sql") + if sql is None: + # No `sql` means the same-named physical column. + expr = dname + else: + expr, _ = cube_sql_to_ossie(sql, cname) + if not sql_is_reversible(sql, plain, cname): + # Only a *member* reference needs the original spelling kept: Cube + # inlines the referenced member's own SQL, which a bare column name in + # the Ossie expression would not reproduce. A plain `{CUBE}.column` (or + # a bare column) regenerates faithfully, so nothing is stashed -- which + # is the common case, and stashing it only added noise for every other + # converter reading the model. + stash["sql"] = sql + + field = { + "name": dname, + "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, + } + datatype = DIM_TYPE_TO_DATATYPE.get(dtype) + if not datatype: + raise ConversionError( + f"cube '{cname}': dimension '{dname}' has unknown type '{dtype}'") + # A precise datatype parked by a previous export wins over the default the Cube + # type maps to, since Cube itself cannot hold the distinction. + parked_dt = ((dim.get("meta") or {}).get("ossie") or {}).get("datatype") + field["datatype"] = parked_dt or datatype + if dtype == "time": + field["dimension"] = {"is_time": True} + if dim.get("title"): + field["label"] = dim["title"] + if dim.get("description"): + field["description"] = dim["description"] + ai = _ai_context_from_meta(dim.get("meta")) + if ai: + field["ai_context"] = ai + + for key, value in dim.items(): + skey = snake(key) + if skey not in _DIM_NATIVE_KEYS: + stash[skey] = value + leftover_meta = _meta_without_ai_context(dim.get("meta")) + if leftover_meta: + stash["meta"] = leftover_meta + write_stash(field, stash) + # Foreign-vendor extensions a previous export parked under the dimension's + # `meta.ossie` are restored after the stash is written, so the CUBE entry stays + # first -- the same ordering datasets use. + _restore_parked_extensions(field, dim.get("meta")) + return [field] + + +def _convert_geo_dimension(cname, dname, dim, issues): + """Split a `type: geo` dimension into a latitude and a longitude field. + + The reconstruction data rides on the latitude half (`geo.host` holds the + dimension's other keys), so export can rebuild the single geo dimension. + """ + issues.add(IssueType.GEO_DIMENSION_SPLIT, f"{cname}.{dname}", + f"split into '{dname}_latitude' and '{dname}_longitude'; an Ossie " + f"field holds a single expression") + host_extras = { + snake(k): v for k, v in dim.items() + if snake(k) not in ("name", "type", "latitude", "longitude") + } + out = [] + for part in ("latitude", "longitude"): + sub = (dim.get(part) or {}).get("sql") + if sub is None: + raise ConversionError( + f"cube '{cname}': geo dimension '{dname}' is missing '{part}.sql'") + expr, _ = cube_sql_to_ossie(sub, cname) + field = { + "name": f"{dname}_{part}", + "expression": { + "dialects": [{"dialect": DIALECT_ANSI, "expression": expr}] + }, + "datatype": "Float", + } + geo = {"of": dname, "part": part, "sql": sub} + if part == "latitude" and host_extras: + geo["host"] = host_extras + write_stash(field, {"geo": geo}) + out.append(field) + return out + + +# --- joins ---------------------------------------------------------------------- + +def _convert_joins(cubes, skipped_files, issues): + """Turn every cube's `joins` into Ossie relationships. + + Ossie's `from` is always the many side. A `many_to_one` join declared on cube + A points A(many) -> B(one) directly; a `one_to_many` join is flipped, and the + declared side and type are stashed so export restores the original. + + `skipped_files` names the input files that held no convertible cube, so a join + pointing into one of them explains itself rather than just reporting a missing + cube. + + Returns (relationships, {cube name: [unconvertible join, ...]}). + """ + relationships = [] + extra_joins = {} + taken = set() + for cname, cube in cubes.items(): + for index, join in enumerate( + _as_named_list(cube.get("joins"), f"cube '{cname}' joins")): + target = require_str(join, "name", f"cube '{cname}': join") + what = f"join '{cname}' -> '{target}'" + if target not in cubes: + hint = "" + if skipped_files: + hint = (f"; note that no cube was converted from " + f"{', '.join(repr(f) for f in skipped_files)} -- if " + f"'{target}' is defined there, that is why") + raise ConversionError( + f"{what}: '{target}' is not a cube in this model{hint}") + raw_rel = snake(require_str(join, "relationship", what)) + rel_type = _RELATIONSHIP_ALIASES.get(raw_rel) + if rel_type is None: + raise ConversionError( + f"{what}: unknown relationship '{join['relationship']}'") + sql = require_str(join, "sql", what) + + pairs = _decompose_join_sql(sql, cname, target, what, issues) + if pairs is None: + extra_joins.setdefault(cname, []).append( + {"index": index, "join": join}) + continue + + from_cube, to_cube = cname, target + from_cols = [p[0] for p in pairs] + to_cols = [p[1] for p in pairs] + # A `many_to_one` join declared on the many side is exactly what Ossie's + # `from`(many) -> `to`(one) already says, so nothing is stashed for the + # common case. Only an orientation Ossie cannot express on its own -- + # one_to_many (flipped) or one_to_one (no many side) -- needs recording. + stash = {} + # Testing the *declared* spelling, not the normalized one: a legacy + # `belongsTo` normalizes to many_to_one but has to come back spelled the + # way it was written, while the modern spelling costs no stash entry. + if raw_rel != "many_to_one": + stash["declared_on"] = cname + stash["relationship"] = raw_rel + if rel_type == "one_to_many": + from_cube, to_cube = to_cube, from_cube + from_cols, to_cols = to_cols, from_cols + elif rel_type == "one_to_one": + # Neither side multiplies, so Ossie's many/one orientation is not + # meaningful; the declared orientation is kept. + issues.add(IssueType.PARKED_IN_META, what, + "one_to_one has no Ossie orientation; the declared " + "orientation is kept and the type preserved") + if sql != _rebuild_join_sql(target, pairs): + stash["sql"] = sql + for key, value in join.items(): + if snake(key) not in ("name", "sql", "relationship"): + stash[snake(key)] = value + + # Ossie relationship names are unique per model; several joins between + # one cube pair would generate the same `_to_`, so repeats + # are suffixed. Export never reads the name, so this stays lossless. + name = f"{from_cube}_to_{to_cube}" + base, k = name, 2 + while name in taken: + name, k = f"{base}_{k}", k + 1 + taken.add(name) + + rel = {"name": name, "from": from_cube, "to": to_cube, + "from_columns": from_cols, "to_columns": to_cols} + write_stash(rel, stash) + relationships.append(rel) + return relationships, extra_joins + + +def _decompose_join_sql(sql, own_cube, target, what, issues): + """Split a Cube join `sql` into (own_column, target_column) pairs. + + Only an AND-chain of equalities between one own-cube reference and one + target-cube reference has an Ossie relationship form. Anything else -- a + range/non-equi condition, a comparison against a literal, a third cube -- + returns None, and the caller preserves the join in the stash instead. + """ + pairs = [] + for clause in _AND_SPLIT_RE.split(sql): + sides = clause.split("=") + if len(sides) != 2: + issues.add(IssueType.PARKED_IN_META, what, + f"join clause '{clause.strip()}' is not a single equality; " + f"preserved in custom_extensions only") + return None + left = _ref_target(sides[0], own_cube, target) + right = _ref_target(sides[1], own_cube, target) + if left is None or right is None: + issues.add(IssueType.PARKED_IN_META, what, + f"join clause '{clause.strip()}' is not between two member " + f"references; preserved in custom_extensions only") + return None + (lcube, lcol), (rcube, rcol) = left, right + if lcube == own_cube and rcube == target: + pairs.append((lcol, rcol)) + elif lcube == target and rcube == own_cube: + pairs.append((rcol, lcol)) + else: + issues.add(IssueType.PARKED_IN_META, what, + f"join clause '{clause.strip()}' references cubes other than " + f"'{own_cube}'/'{target}'; preserved in custom_extensions only") + return None + return pairs or None + + +def _ref_target(side, own_cube, target): + """Resolve one side of a join equality to (cube_name, column), or None.""" + translated, _ = cube_sql_to_ossie(side, own_cube) + translated = translated.strip() + if is_simple_identifier(translated): + # A bare name came from `{CUBE}.col`, `{CUBE.col}`, or `{col}` -- all of + # which address the cube the join is declared on. + return (own_cube, translated) + m = DOTTED_REF_RE.fullmatch(translated) + if m and m.group(1) in (own_cube, target): + return (m.group(1), m.group(2)) + return None + + +def _rebuild_join_sql(target, pairs): + """The canonical form export emits, used to decide whether the original has to + be stashed. The own side is always `{CUBE}` so the join keeps working when the + cube is extended, and both sides use the alias-dot raw-column form because + Ossie's from_columns/to_columns name columns, not members.""" + return " AND ".join( + "{CUBE}." + own + " = {" + target + "}." + other + for own, other in pairs + ) + + +# --- measures ------------------------------------------------------------------- + +class _MeasureResolver: + """Computes the Ossie expression for a Cube measure. + + Kept as a class because a calculated measure (`type: number`, and the other + types in `CALCULATED_MEASURE_TYPES`) can reference other measures, which Cube + resolves by inlining their full aggregate SQL -- so producing one measure's + expression may require producing another's first. Each measure's expression is + computed once and cached; a reference cycle is rejected rather than recursed + into. + + Note that inlining is inherently exponential in reference depth -- a chain where + each measure names the previous one twice doubles the SQL at every step -- and + that is Cube's own behaviour, not this converter's choice. The cache makes the + work proportional to the output rather than to the output times the depth; it + cannot make the output smaller. No limit is imposed, since any threshold would + reject a legitimate model to guard against a hand-written pathological one. + """ + + def __init__(self, cubes, pk_by_cube, issues): + self._pk = pk_by_cube + self._issues = issues + self._raw = {} + self._cache = {} + for cname, cube in cubes.items(): + for m in _as_named_list(cube.get("measures"), f"cube '{cname}' measures"): + self._raw[(cname, require_str(m, "name", f"cube '{cname}': measure"))] = m + + def measures(self): + return self._raw + + def is_measure(self, cube, name): + return (cube, name) in self._raw + + def aggregate_of(self, cname, mname): + """The normalized Cube `type` of a measure.""" + return snake(self._raw[(cname, mname)].get("type") or "") + + def expression(self, cname, mname, stack=()): + """The Ossie expression reproducing this measure, or None when the measure + has no static form (multi-stage, Jinja-templated).""" + key = (cname, mname) + if key in stack: + chain = " -> ".join(f"{c}.{m}" for c, m in stack + (key,)) + raise ConversionError(f"measure reference cycle: {chain}") + if key in self._cache: + return self._cache[key] + measure = self._raw[key] + scope = f"{cname}.{mname}" + mtype = snake(measure.get("type") or "") + if not mtype: + raise ConversionError(f"measure '{scope}': missing required 'type'") + + if measure.get("multi_stage"): + # group_by / reduce_by / time_shift / rank render as window functions + # over a grain other than the query's; Ossie has no form for that. + self._issues.add( + IssueType.MULTI_STAGE_MEASURE_PARKED, scope, + f"multi_stage measure (type '{mtype}'); preserved in " + f"custom_extensions only") + return self._remember(key, None) + sql = measure.get("sql") + filter_exprs = [ + self._translate(f["sql"], cname, stack + (key,)) + for f in (measure.get("filters") or []) + if isinstance(f, dict) and f.get("sql") + ] + + if mtype in CALCULATED_MEASURE_TYPES: + if sql is None: + raise ConversionError( + f"measure '{scope}': type '{mtype}' requires 'sql'") + expr = self._translate(sql, cname, stack + (key,)) + return self._remember(key, filtered_operand(expr, filter_exprs)) + if mtype == "count": + if sql is None: + return self._remember(key, primary_key_count_expression( + cname, self._pk.get(cname) or [], filter_exprs)) + operand = filtered_operand( + self._operand(cname, sql, stack + (key,)), filter_exprs) + return self._remember(key, f"COUNT({operand})") + func = AGG_TO_OSSIE_FUNC.get(mtype) + if func is None: + raise ConversionError( + f"measure '{scope}': unknown aggregate type '{mtype}'") + if sql is None: + raise ConversionError( + f"measure '{scope}': type '{mtype}' requires 'sql'") + operand = filtered_operand( + self._operand(cname, sql, stack + (key,)), filter_exprs) + return self._remember( + key, f"COUNT(DISTINCT {operand})" if func == "COUNT_DISTINCT" + else f"{func}({operand})") + + def _remember(self, key, expr): + """Cache one measure's expression. + + A calculated measure inlines each reference's full SQL, so a measure + referenced from several places was recomputed once per reference -- and + recursively, so a chain of them cost O(depth * 2**depth) instead of the + O(2**depth) the inlined output is inherently worth. + """ + self._cache[key] = expr + return expr + + def _translate(self, sql, cname, stack): + """Translate a Cube SQL string, inlining any measure reference. + + `self_prefix` is the owning cube: Ossie metrics are model-level, so a + column reads as `dataset.column` here, unlike in a dataset-scoped field + expression. + """ + out, _ = cube_sql_to_ossie( + sql, cname, resolve_ref=lambda body: self._inline(body, cname, stack), + self_prefix=cname) + return out + + def _inline(self, body, cname, stack): + """Resolve one `{...}` body when it names a measure, else fall through. + + Cube inlines a measure reference to that measure's own aggregate SQL + (`isCalculatedMeasureType` emits the sql as-is), so `{revenue} / {count}` + becomes a complete ratio expression -- which is exactly the shape Ossie + metrics use. Parenthesized to keep the referenced measure's precedence. + """ + head, _, rest = body.partition(".") + if rest: + target_cube = cname if head in ("CUBE", "TABLE") else head + target_name = rest + else: + target_cube, target_name = cname, body + if not self.is_measure(target_cube, target_name): + return None + inner = self.expression(target_cube, target_name, stack) + if inner is None: + raise ConversionError( + f"measure '{cname}': references '{target_cube}.{target_name}', " + f"which has no static Ossie form") + # A lone `SUM(x)` needs no parentheses; only a term with its own top-level + # operators does. Keeping them off means a decomposed metric inlines back to + # exactly the expression it was split from. + return f"({inner})" if has_top_level_operator(inner) else inner + + def _operand(self, cname, sql, stack): + """Translate an aggregate's operand into an Ossie reference. + + A same-cube member or bare column becomes `cube.name` -- the qualified form + Ossie model-level metrics use. A computed operand keeps its own qualifiers + and is emitted as-is; the owning cube rides in the stash either way, so + export still puts the measure back on the right cube. + """ + translated = self._translate(sql, cname, stack).strip() + if is_simple_identifier(translated): + return f"{cname}.{translated}" + return translated + + +def _is_generated_part(measure): + """True for a `public: false` measure a previous export created to hold one + aggregate of a composite metric (marked `meta.ossie.part_of`).""" + return bool(((measure.get("meta") or {}).get("ossie") or {}).get("part_of")) + + +def _convert_measures(cubes, pk_by_cube, plain_by_cube, fanned_out, issues): + """Hoist every cube's measures into Ossie model-level metrics. + + A metric name is the measure name when globally unique, else + `__`; the original name and owning cube are stashed so export + puts the measure back where it came from. + + Returns (metrics, {cube: [{"index": i, "measure": ...}]}). The second value holds + measures with no static Ossie expression -- a multi-stage measure renders as a + window function over another grain -- which have no `metrics` entry and would + otherwise vanish. They ride on the owning dataset's stash with their positions, + the same protocol unconvertible joins use. + """ + resolver = _MeasureResolver(cubes, pk_by_cube, issues) + + counts = {} + for (cname, mname), measure in resolver.measures().items(): + if not _is_generated_part(measure): + counts[mname] = counts.get(mname, 0) + 1 + + metrics = [] + extra_measures = {} + seen = set() + for cname, cube in cubes.items(): + plain = plain_by_cube[cname] + for index, measure in enumerate( + _as_named_list(cube.get("measures"), + f"cube '{cname}' measures")): + mname = measure["name"] + if _is_generated_part(measure): + # Emitted by a previous export to split a composite metric across + # cubes. It has no Ossie metric of its own -- the public measure's + # references inline back to the whole expression -- and export + # regenerates it, so it is not stashed either. + continue + metric_name = mname if counts[mname] == 1 else f"{cname}__{mname}" + if metric_name in seen: + raise ConversionError( + f"metric name '{metric_name}' derived twice; rename the " + f"colliding measures in Cube") + seen.add(metric_name) + metric = _convert_measure(cname, mname, metric_name, measure, resolver, + fanned_out, plain, issues) + if metric is not None: + metrics.append(metric) + else: + extra_measures.setdefault(cname, []).append( + {"index": index, "measure": measure}) + return metrics, extra_measures + + +def _convert_measure(cname, mname, metric_name, measure, resolver, fanned_out, + plain, issues): + scope = f"{cname}.{mname}" + expr = resolver.expression(cname, mname) + if expr is None: + # No static form; the resolver already recorded why. + return None + mtype = resolver.aggregate_of(cname, mname) + sql = measure.get("sql") + + # Reconstructible = export can rebuild this measure from the Ossie expression + # alone. A calculated measure never is: export would re-parse its expression + # into a structured measure, and the inlined references cannot be un-inlined. + # Neither is a filtered one -- recovering `filters` would mean parsing the + # folded CASE back apart, so the original rides along instead. + reconstructible = ( + {snake(k) for k in measure} <= _MEASURE_NATIVE_KEYS + and mtype not in CALCULATED_MEASURE_TYPES + and not measure.get("filters") + ) + + # Fan-out: a non-idempotent aggregate on a dataset the graph can multiply. + # Cube fixes this at query time by deduplicating on the primary key; a static + # expression cannot, so the caller has to be told. + unsafe = mtype in FANOUT_UNSAFE_AGGS or (mtype == "count" and sql is not None) + if unsafe and cname in fanned_out: + issues.add( + IssueType.FANOUT_UNSAFE_METRIC, scope, + f"'{mtype}' over dataset '{cname}', which relationship " + f"'{fanned_out[cname]}' fans out; Cube deduplicates on the primary key " + f"at query time but a static Ossie expression cannot, so a consumer " + f"joining through that relationship may over-count") + + metric = { + "name": metric_name, + "expression": {"dialects": [{"dialect": DIALECT_ANSI, "expression": expr}]}, + } + datatype = AGG_TO_RESULT_DATATYPE.get(mtype) + if datatype: + metric["datatype"] = datatype + if measure.get("description"): + metric["description"] = measure["description"] + ai = _ai_context_from_meta(measure.get("meta")) + if ai: + metric["ai_context"] = ai + + stash = {"cube": cname} + if not reconstructible: + stash["measure"] = { + snake(k): v for k, v in measure.items() + if snake(k) not in ("description", "meta") + } + elif sql is not None and not sql_is_reversible(sql, plain, cname): + # Only a reference export cannot regenerate needs the original spelling: a + # non-plain member (whose own SQL is inlined) or a cross-cube reference + # (which is what adds the implicit join). + stash["sql"] = sql + if metric_name != mname: + stash["name"] = mname + if measure.get("title"): + stash["title"] = measure["title"] + leftover_meta = _meta_without_ai_context(measure.get("meta")) + if leftover_meta: + stash["meta"] = leftover_meta + write_stash(metric, stash) + _restore_parked_extensions(metric, measure.get("meta")) + return metric diff --git a/converters/cube/src/ossie_cube/expressions.py b/converters/cube/src/ossie_cube/expressions.py new file mode 100644 index 00000000..12f70e44 --- /dev/null +++ b/converters/cube/src/ossie_cube/expressions.py @@ -0,0 +1,185 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Reading the structure of an Ossie metric expression. + +Cube expects a measure to *be* an aggregation -- `type: sum` over a column -- and +falls back to a calculated `type: number` measure whose sql carries the whole +aggregate. A composite Ossie metric such as + + SUM(store_sales.amount) / COUNT(DISTINCT customer.id) + +can be emitted either way, and the difference matters: as one calculated measure +Cube sees a single opaque expression, whereas as two `public: false` measures on +their own cubes plus a ratio referencing them, **Cube applies its row-multiplication +correction to each aggregate independently**. So decomposition is a correctness +improvement for cross-dataset metrics, not a formatting choice. + +Locating the aggregate calls is done with sqlglot rather than a regex, since an +expression can nest them (`SUM(x) / NULLIF(SUM(y), 0)`) and string matching cannot +tell a top-level call from one inside another argument. sqlglot is already a runtime +dependency of the dbt and NVIDIA GSF converters for the same purpose. +""" + +import sqlglot +import sqlglot.expressions as exp + +from ._common import quoted_char_mask + +# sqlglot node types for the aggregates this converter maps to a Cube measure type. +# `Count` covers COUNT / COUNT(DISTINCT x); ApproxDistinct covers +# APPROX_COUNT_DISTINCT. +_AGGREGATE_NODES = ( + exp.Sum, exp.Avg, exp.Min, exp.Max, exp.Count, exp.ApproxDistinct, +) + + +def parse(expr): + """Parse an Ossie expression, or None when sqlglot cannot. + + An unparseable expression is not an error: the converter falls back to treating + it as one opaque calculated measure, which is what it did for everything before. + """ + try: + return sqlglot.parse_one(str(expr).strip()) + except Exception: + return None + + +def is_single_aggregate(expr): + """True if the whole expression is exactly one aggregate call. + + Those already map to a structured Cube measure (`type: sum` + `sql`), so they + are never decomposed. + """ + tree = parse(expr) + return tree is not None and isinstance(tree, _AGGREGATE_NODES) + + +# The aggregate call names this converter maps to a Cube measure type. Scanned for +# in the source text: sqlglot renames some when it renders (`APPROX_COUNT_DISTINCT` +# comes back as `APPROX_DISTINCT`), and two calls of the same name render +# identically, so node text cannot be used to find them in the original string. +_AGGREGATE_NAMES = ( + "APPROX_COUNT_DISTINCT", "APPROX_DISTINCT", + "COUNT", "SUM", "AVG", "MIN", "MAX", +) + + +def aggregate_spans(expr): + """The outermost aggregate calls in `expr`, as (start, end) offsets. + + Offsets index the original string so a caller can substitute each span in place. + That matters because the surrounding text may carry Cube `{...}` references, + which sqlglot would not reproduce verbatim if the expression were re-rendered. + + Spans are found by scanning for an aggregate name followed by a balanced + parenthesis group, then confirmed with sqlglot -- which is also what rules out a + malformed expression. Nesting is resolved on the offsets themselves: a span + inside another span is not returned, so `SUM(x) / NULLIF(SUM(y), 0)` gives two + and `SUM(SUM(x))` gives one. Returns [] when the expression does not parse, or is + itself a single aggregate needing no decomposition. + + A name inside a string literal is not a call: `SUM(x) || ' per COUNT(y) unit'` + has one aggregate, not two. Taking the second would splice a measure reference + into the literal. + """ + text = str(expr) + if parse(text) is None or is_single_aggregate(text): + return [] + + candidates = [] + upper = text.upper() + quoted = quoted_char_mask(text) + for name in _AGGREGATE_NAMES: + at = 0 + while True: + at = upper.find(name, at) + if at < 0: + break + start, after = at, at + len(name) + at = after + if quoted[start]: + continue + # A call, not part of a longer identifier: boundary before, `(` after. + if start and (text[start - 1].isalnum() or text[start - 1] == "_"): + continue + probe = after + while probe < len(text) and text[probe].isspace(): + probe += 1 + if probe >= len(text) or text[probe] != "(": + continue + close = _match_paren(text, probe) + if close is None: + continue + end = close + 1 + # Confirm the slice really is an aggregate and not, say, a UDF that + # happens to share a prefix. + node = parse(text[start:end]) + if isinstance(node, _AGGREGATE_NODES): + candidates.append((start, end)) + + # Drop any span contained within another: only the outermost becomes a measure. + candidates.sort() + out = [] + for start, end in candidates: + if any(s <= start and end <= e for s, e in out): + continue + out.append((start, end)) + return out + + +def _match_paren(text, open_at): + """Index of the `)` closing the `(` at `open_at`, honouring quotes.""" + depth, quote = 0, None + for i in range(open_at, len(text)): + ch = text[i] + if quote: + if ch == quote: + quote = None + elif ch in "'\"": + quote = ch + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth == 0: + return i + return None + + +def has_top_level_operator(expr): + """True if `expr` is not a single self-contained term. + + Used to decide whether inlining it back into a larger expression needs + parentheses: a lone `SUM(x)` does not, `SUM(x) / 2` does. + """ + depth, quote = 0, None + for ch in str(expr): + if quote: + if ch == quote: + quote = None + elif ch in "'\"": + quote = ch + elif ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + elif depth == 0 and (ch in "+-*/%<>=|&" or ch.isspace()): + # Whitespace at depth 0 also implies structure (`CASE WHEN ...`). + return True + return False diff --git a/converters/cube/src/ossie_cube/osi_to_cube.py b/converters/cube/src/ossie_cube/osi_to_cube.py new file mode 100644 index 00000000..1e233ca5 --- /dev/null +++ b/converters/cube/src/ossie_cube/osi_to_cube.py @@ -0,0 +1,954 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Convert an Apache Ossie semantic model to a Cube data model. + +Pure offline conversion. Produces the Cube model-directory layout: one +`model/cubes/.yml` per dataset and a `model/views/.yml` for the model +itself, plus -- when a prior import stashed them -- the original file paths and +every Cube-only construct restored verbatim. + +Ossie features Cube has no field for (`unique_keys`, foreign-vendor +`custom_extensions`, the structured form of `ai_context`) are parked under +`meta.ossie` rather than dropped, since Cube has a `meta` field at every level. +That keeps `Ossie -> Cube -> Ossie` lossless as well. + +Usage (CLI): + ossie-cube export -i model.yaml -o model/ [--dialect SNOWFLAKE] [--base-cube orders] +""" + +import re +from collections import deque + +from ._common import ( + DATATYPE_TO_DIM_TYPE, + DEFAULT_DATATYPE_FOR_CUBE_TYPE, + OSSIE_FUNC_TO_AGG, + OSSIE_VERSION, + ConversionError, + cube_file, + dump_yaml, + examples_of, + foreign_vendor_extensions, + instructions_of, + is_simple_identifier, + load_yaml, + ossie_expr_to_cube_sql, + parse_source, + pick_expression, + primary_key_operand, + read_stash, + referenced_datasets, + require_str, + sanitize_name, + synonyms_of, + view_file, +) +from .converter_issues import IssueLog, IssueType +from .expressions import aggregate_spans + +# An aggregate call the exporter can turn back into a structured Cube measure. +_AGG_CALL_RE = re.compile( + r"^\s*(SUM|AVG|MIN|MAX|COUNT|APPROX_COUNT_DISTINCT)\s*\((.*)\)\s*$", + re.IGNORECASE | re.DOTALL, +) +_DISTINCT_RE = re.compile(r"^DISTINCT\s+(.+)$", re.IGNORECASE | re.DOTALL) + +# The order Cube's own YAML documentation and generators use, so exported files +# read the way a hand-authored model does. +_CUBE_KEY_ORDER = [ + "name", "sql_table", "sql", "title", "description", "meta", "joins", + "dimensions", "measures", "segments", +] +_DIM_KEY_ORDER = [ + "name", "sql", "type", "primary_key", "title", "description", "meta", +] +_MEASURE_KEY_ORDER = [ + "name", "sql", "type", "filters", "title", "description", "meta", +] + + +def convert_ossie_to_cube(ossie_yaml_str, dialect=None, base_cube=None): + """Parse Ossie YAML and return Cube model files as {relative filename: YAML str}. + + Returns (files, IssueLog). `dialect` prepends a warehouse dialect (e.g. + SNOWFLAKE) to the expression preference order; ANSI_SQL is always the fallback. + `base_cube` names the dataset a generated view is rooted at, and is only + consulted for a hand-authored Ossie model with no stashed views. + """ + root = load_yaml(ossie_yaml_str, "Ossie model") + if not isinstance(root, dict): + raise ConversionError("Invalid Ossie YAML: expected a mapping at the root") + version = str(root.get("version", "")) + if version != OSSIE_VERSION: + raise ConversionError( + f"Unsupported Ossie version '{version}'. Supported: {OSSIE_VERSION}") + models = root.get("semantic_model") + if not isinstance(models, list) or not models: + raise ConversionError("'semantic_model' must be a non-empty list") + + issues = IssueLog() + if len(models) > 1: + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, "model", + f"{len(models)} semantic models found; only the first is " + f"converted and the rest are not preserved anywhere") + return _convert_model(models[0], dialect, base_cube, issues) + + +def _convert_model(model, dialect, base_cube, issues): + name = model.get("name", "") + dataset_list = model.get("datasets") or [] + if not dataset_list: + raise ConversionError(f"Model '{name}' has no datasets") + + # Dataset -> cube names. A collision (including a case-insensitive duplicate, + # which sanitizes identically) fails loudly rather than merging. + cube_names = {} + taken = set() + for ds in dataset_list: + ds_name = require_str(ds, "name", f"Model '{name}': dataset") + cube_names[ds_name] = sanitize_name( + ds_name, f"Model '{name}': dataset", taken) + taken.add(cube_names[ds_name].lower()) + datasets = {ds["name"]: ds for ds in dataset_list} + + relationships = model.get("relationships") or [] + for rel in relationships: + scope = f"Model '{name}': relationship '{rel.get('name', '')}'" + if (require_str(rel, "from", scope) not in datasets + or require_str(rel, "to", scope) not in datasets): + raise ConversionError(f"{scope} references an unknown dataset") + + model_stash = read_stash(model) + + # Per-cube facts the join and measure stages need. + # Field -> dimension names are resolved once here and reused by every stage. + # Sanitizing per stage would let a collision go undetected in one place and be + # rejected in another, and would disagree about which members a cube actually + # has -- which decides `{CUBE.member}` vs `{CUBE}.column` and where a measure + # lands. + dim_names_by_cube = {} + members_by_cube = {} + inline_sql_by_cube = {} + pk_by_cube = {} + for ds_name, ds in datasets.items(): + cname = cube_names[ds_name] + dim_names_by_cube[cname], inline_sql_by_cube[cname] = ( + _resolve_dimension_names(ds, f"Model '{name}': dataset '{ds_name}'")) + # Not every member: only those the `{CUBE.member}` form is required for. + members_by_cube[cname] = _reference_members( + ds, dim_names_by_cube[cname], dialect) + pk_by_cube[cname] = [str(c) for c in (ds.get("primary_key") or [])] + + joins_by_cube = _build_joins(relationships, cube_names, issues) + measures_by_cube = _build_measures( + model, cube_names, members_by_cube, inline_sql_by_cube, pk_by_cube, + datasets, relationships, base_cube, dialect, issues) + + # Cubes, grouped by the file they belong in: several datasets can share one + # stashed original path, in which case they go back into the same file. + stashed_paths = model_stash.get("cube_files") or {} + files_content = {} + for ds_name, ds in datasets.items(): + cname = cube_names[ds_name] + cube = _build_cube(ds, cname, dim_names_by_cube[cname], + inline_sql_by_cube[cname], members_by_cube[cname], + joins_by_cube.get(cname), measures_by_cube.get(cname), + dialect, issues) + path = stashed_paths.get(cname) or cube_file(cname) + files_content.setdefault(path, {}).setdefault("cubes", []).append(cube) + + for vpath, views in _build_views(model, model_stash, cube_names, relationships, + datasets, base_cube).items(): + files_content.setdefault(vpath, {}).setdefault("views", []).extend(views) + + files = {path: dump_yaml(content) for path, content in files_content.items()} + + # Files a prior import could not convert (`.js` models, Jinja-templated YAML, + # non-model YAML) restore verbatim. + for fname, text in (model_stash.get("extra_files") or {}).items(): + files[fname] = text + return files, issues + + +# --- ai_context ----------------------------------------------------------------- + +def _ai_context_to_meta(ai_context): + """Split an Ossie `ai_context` into (Cube prose, parked original). + + Cube's `meta.ai_context` is free text, so the instructions go there verbatim + and any synonyms are appended as prose -- which is how Cube's own + documentation expresses them ("Common acronyms: LC = Lucky Charms"). The + structured original is parked under `meta.ossie.ai_context` whenever the prose + alone would not restore it, so the Ossie round trip stays exact. + """ + if not ai_context: + return None, None + instructions = instructions_of(ai_context) + synonyms = synonyms_of(ai_context) + examples = examples_of(ai_context) + + parts = [instructions] if instructions else [] + if synonyms: + parts.append("Also known as: " + ", ".join(str(s) for s in synonyms) + ".") + if examples: + parts.append("Example questions: " + + " ".join(str(e) for e in examples)) + prose = "\n".join(parts) if parts else None + + # Import reads a bare prose value back as {"instructions": prose}. Anything + # else -- a plain string, synonyms, examples, extra keys -- needs the original. + round_trips = (isinstance(ai_context, dict) + and set(ai_context) == {"instructions"} + and ai_context.get("instructions") == prose) + return prose, (None if round_trips else ai_context) + + +def _build_meta(ai_context, stashed_meta, parked_extra): + """Assemble a Cube `meta` from the Ossie AI context, a stashed original meta, + and anything Ossie-only that needs parking.""" + prose, parked_ai = _ai_context_to_meta(ai_context) + meta = {} + if prose: + meta["ai_context"] = prose + for key, value in (stashed_meta or {}).items(): + meta[key] = value + parked = dict(parked_extra or {}) + if parked_ai is not None: + parked["ai_context"] = parked_ai + if parked: + meta["ossie"] = parked + return meta + + +def _ordered(obj, order): + """Re-key a dict so the well-known Cube keys come first, in their documented + order, with anything restored from the stash following.""" + out = {k: obj[k] for k in order if k in obj} + for key, value in obj.items(): + if key not in out: + out[key] = value + return out + + +# --- cubes ---------------------------------------------------------------------- + +def _build_cube(ds, cname, dim_names, inline_sql, ref_members, joins, measures, + dialect, issues): + ds_name = ds["name"] + scope = f"dataset '{ds_name}'" + stash = read_stash(ds) + cube = {"name": cname} + + kind, value = parse_source(ds.get("source"), ds_name) + cube[kind] = value + if ds.get("description"): + cube["description"] = ds["description"] + + parked = {} + if ds.get("unique_keys"): + parked["unique_keys"] = [list(k) for k in ds["unique_keys"]] + issues.add(IssueType.PARKED_IN_META, scope, + "unique_keys have no Cube field; parked under meta.ossie") + foreign = foreign_vendor_extensions(ds) + if foreign: + parked["custom_extensions"] = foreign + cube_extras = dict(stash.get("cube_extras") or {}) + stashed_meta = cube_extras.pop("meta", None) + meta = _build_meta(ds.get("ai_context"), stashed_meta, parked) + if meta: + cube["meta"] = meta + if "ai_context" in meta: + issues.add(IssueType.CUBE_LEVEL_AI_CONTEXT_INERT, scope, + "Cube's agent reads ai_context only on views and members, " + "so this cube-level value has no effect in Cube") + + dimensions, by_name_scalar, by_column = _build_dimensions( + ds, cname, dim_names, inline_sql, ref_members, dialect, issues) + # Resolve each `primary_key` entry to the dimension Cube should mark. A + # dimension only qualifies when it is *scalar* -- backed by a single source + # column -- because `primary_key: true` in Cube declares that dimension's own + # sql to be the key. A computed dimension would declare the wrong expression, + # and a merged geo dimension has no single sql at all, so neither counts even + # when its name matches. Anything left uncovered gets a private dimension. + pk_names = [] + taken = {d["name"].lower() for d in dimensions} + for entry in (ds.get("primary_key") or []): + entry = str(entry) + # Import records the *dimension name*, so the name match is checked first; + # a hand-authored model naming the source column resolves by column. + match = by_name_scalar.get(entry) or by_column.get(entry) + if match: + pk_names.append(match) + continue + name = _unique_pk_dimension_name(entry, taken) + taken.add(name.lower()) + detail = (f"primary key '{entry}' is not backed by a scalar dimension; " + f"emitted as a non-public dimension with type 'string' (Cube " + f"requires a type and Ossie carries none here)") + if name != entry: + detail += f", named '{name}' to avoid colliding with the existing member" + issues.add(IssueType.APPROXIMATED, scope, detail) + dimensions.append({"name": name, "sql": entry, "type": "string", + "primary_key": True, "public": False}) + pk_names.append(name) + for dim in dimensions: + if dim["name"] in pk_names: + dim["primary_key"] = True + + if dimensions: + cube["dimensions"] = [_ordered(d, _DIM_KEY_ORDER) for d in dimensions] + + joins = list(joins or []) + # Joins a prior import could not represent go back at their original indices. + for item in sorted(stash.get("extra_joins") or [], key=lambda x: x.get("index", 0)): + joins.insert(min(item.get("index", 0), len(joins)), item["join"]) + if joins: + cube["joins"] = joins + measures = [_ordered(m, _MEASURE_KEY_ORDER) for m in (measures or [])] + # Measures a prior import could not express in Ossie (multi-stage ones) go back + # at their original indices, interleaved with the ones rebuilt from metrics. + for item in sorted(stash.get("extra_measures") or [], + key=lambda x: x.get("index", 0)): + measures.insert(min(item.get("index", 0), len(measures)), item["measure"]) + if measures: + cube["measures"] = measures + + for key, value in cube_extras.items(): + cube[key] = value + return _ordered(cube, _CUBE_KEY_ORDER) + + +def _reference_members(ds, dim_names, dialect): + """Members that must be addressed as `{CUBE.member}` rather than `{CUBE}.column`. + + Only a member whose expression is something other than its own same-named column + needs the reference form, because that form makes Cube inline the member's SQL. A + plain member is identical either way, and the raw-column form is what survives a + round trip without stashing the spelling. + """ + needed = set() + for field in (ds.get("fields") or []): + fname = field.get("name") + dname = dim_names.get(fname) + if not dname: + continue + expr = pick_expression(field.get("expression"), dialect) + if expr is None or not is_simple_identifier(expr) or expr.strip() != dname: + needed.add(dname) + return needed + + +def _resolve_dimension_names(ds, scope): + """Map each of a dataset's fields to the Cube dimension name it becomes. + + Sanitization and collision detection happen here and nowhere else, so every + stage agrees on the result. Two subtleties the mapping has to get right: + + - A collision is an error, not a silent merge. Sanitizing with a fresh `taken` + set per field would hide one. + - The two halves of a split `geo` dimension map back to the *single* dimension + they merge into, so `location_latitude` resolves to `location`. + + Returns (names, inline_sql). `inline_sql` holds the fields whose name exists + only in Ossie -- the two halves of a split geo dimension -- mapped to the Cube + SQL a reference to them must be replaced by, since Cube has neither a column nor + a member of that name. + """ + names, inline_sql = {}, {} + taken = set() + geo_halves = {} # base -> {part: field name}, for validating the pair + for field in (ds.get("fields") or []): + fname = require_str(field, "name", f"{scope}: field") + geo = read_stash(field).get("geo") + if geo: + base, part = geo.get("of"), geo.get("part") + if part not in ("latitude", "longitude"): + raise ConversionError( + f"{scope}: field '{fname}' has a geo part '{part}'; expected " + f"'latitude' or 'longitude'") + if not base: + raise ConversionError( + f"{scope}: field '{fname}' has a geo stash with no 'of'") + seen = geo_halves.setdefault(base, {}) + if part in seen: + raise ConversionError( + f"{scope}: fields '{seen[part]}' and '{fname}' both claim the " + f"{part} of geo dimension '{base}'") + if not seen and base.lower() in taken: + # The base is the name of the merged Cube dimension, so it cannot + # also be an ordinary dimension -- that would emit two members of + # the same name. Order must not decide whether this is caught, so + # it is checked here rather than left to sanitize_name. + raise ConversionError( + f"{scope}: geo dimension '{base}' collides with another field " + f"of that name; rename one in the Ossie model.") + seen[part] = fname + taken.add(base.lower()) + names[fname] = base + inline_sql[fname] = geo["sql"] + continue + dname = sanitize_name(fname, f"{scope}: field", taken) + taken.add(dname.lower()) + names[fname] = dname + for base, seen in geo_halves.items(): + missing = {"latitude", "longitude"} - set(seen) + if missing: + raise ConversionError( + f"{scope}: geo dimension '{base}' is missing its " + f"{' and '.join(sorted(missing))} half") + return names, inline_sql + + +def _build_dimensions(ds, cname, dim_names, inline_sql, ref_members, dialect, + issues): + """Build a cube's dimensions from an Ossie dataset's fields. + + Returns (dimensions, by_name_scalar, by_column) -- the two maps are what + primary-key resolution matches against, and both hold only *scalar* dimensions + (those whose expression is a single source column). A computed dimension and a + merged geo dimension are deliberately absent from both: Cube's + `primary_key: true` declares that dimension's own sql to be the key, so marking + either would declare something other than the column Ossie named. Fields + carrying a `geo` stash are re-merged into the single Cube dimension they were + split from. + Dimension names come from `dim_names` (see `_resolve_dimension_names`) rather + than being sanitized again here. + """ + ds_name = ds["name"] + by_name_scalar, by_column = {}, {} + # Built by target dimension name rather than by list position: a geo dimension + # is assembled from two fields that may appear in either order and need not be + # adjacent, so an insertion index computed mid-loop is not a safe way to hold + # its place. `order` records first appearance of each target name, which is + # well defined however the halves are arranged. + order, built, geo_parts = [], {}, {} + for field in (ds.get("fields") or []): + fname = require_str(field, "name", f"dataset '{ds_name}': field") + stash = read_stash(field) + dname = dim_names[fname] + if dname not in order: + order.append(dname) + if "geo" in stash: + geo = stash["geo"] + slot = geo_parts.setdefault(dname, {}) + slot[geo["part"]] = geo["sql"] + if "host" in geo: + slot["host"] = geo["host"] + continue + + expr = pick_expression(field.get("expression"), dialect) + if expr is None: + issues.add(IssueType.NO_USABLE_DIALECT, f"{ds_name}.{fname}", + "no ANSI_SQL or preferred-dialect expression; field dropped") + continue + + dim = {"name": dname} + if "sql" in stash: + # The exact Cube spelling a prior import saw. + dim["sql"] = stash["sql"] + else: + dim["sql"] = ossie_expr_to_cube_sql( + expr, cname, ref_members, (), inline_sql={cname: inline_sql}) + dim["type"] = _dimension_type(field, stash, f"{ds_name}.{fname}", issues) + if field.get("label"): + dim["title"] = field["label"] + if field.get("description"): + dim["description"] = field["description"] + parked = {} + foreign = foreign_vendor_extensions(field) + if foreign: + parked["custom_extensions"] = foreign + # Cube's `type` is coarser than Ossie's `datatype` (Integer/Decimal/Float all + # become `number`), so the precise one is parked whenever importing would not + # recover it. `meta.ossie` is Cube-side, so this costs the Ossie document + # nothing -- unlike a custom_extension, which every other spoke would warn + # about and discard. + dt = field.get("datatype") + if dt and DEFAULT_DATATYPE_FOR_CUBE_TYPE.get(dim["type"]) != dt: + parked["datatype"] = dt + extras = {k: v for k, v in stash.items() if k not in ("sql", "type", "meta")} + meta = _build_meta(field.get("ai_context"), stash.get("meta"), parked) + if meta: + dim["meta"] = meta + for key, value in extras.items(): + dim[key] = value + + built[dname] = dim + if is_simple_identifier(expr): + # Scalar: this dimension is exactly one source column, so Cube can mark + # it as the key. Reachable by its own name and by that column's name. + by_name_scalar[dname] = dname + by_column.setdefault(expr.strip(), dname) + + # Both halves are guaranteed present by _resolve_dimension_names, which + # validates the pair before anything is built. + for base, slot in geo_parts.items(): + dim = {"name": base, "type": "geo", + "latitude": {"sql": slot["latitude"]}, + "longitude": {"sql": slot["longitude"]}} + for key, value in (slot.get("host") or {}).items(): + dim[key] = value + built[base] = dim + + # A name in `order` with nothing built is a field dropped for want of a usable + # dialect; it simply does not appear. + return [built[n] for n in order if n in built], by_name_scalar, by_column + + +def _unique_pk_dimension_name(entry, taken): + """A valid, unused Cube identifier for a synthesized primary-key dimension. + + The obvious name is the primary-key entry itself, but a computed or geo + dimension may already own it -- in which case emitting a second dimension of + that name would produce an invalid cube, and overwriting the existing one would + lose a member. So a suffix is added until the name is free. + """ + base = sanitize_name(entry, "primary key", set()) + if base.lower() not in taken: + return base + for n in range(1, 100): + candidate = f"{base}_pk" if n == 1 else f"{base}_pk_{n}" + if candidate.lower() not in taken: + return candidate + raise ConversionError( + f"cannot find a free dimension name for primary key '{entry}'; rename the " + f"colliding members in the Ossie model.") + + +def _dimension_type(field, stash, scope, issues): + """Choose the Cube `type`, which every dimension must declare.""" + if "type" in stash: + # An older stash from before datatypes were mapped natively. + return stash["type"] + datatype = field.get("datatype") + explicit_is_time = (field.get("dimension") or {}).get("is_time") + if datatype: + ctype = DATATYPE_TO_DIM_TYPE.get(datatype) + if ctype is None: + raise ConversionError(f"{scope}: unknown datatype '{datatype}'") + if explicit_is_time is True and ctype != "time": + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, scope, + f"is_time is true but datatype '{datatype}' maps to Cube " + f"type '{ctype}'; Cube marks time dimensions by type, so " + f"the temporal role is not carried") + elif explicit_is_time is False and ctype == "time": + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, scope, + f"is_time is false but datatype '{datatype}' maps to Cube " + f"type 'time', which Cube always treats as a time dimension; " + f"the opt-out is not carried") + return ctype + if explicit_is_time: + return "time" + issues.add(IssueType.APPROXIMATED, scope, + "no datatype; emitted as Cube type 'string', which Cube requires") + return "string" + + +# --- joins ---------------------------------------------------------------------- + +def _build_joins(relationships, cube_names, issues): + """Group Ossie relationships into per-cube `joins` lists. + + A stashed `declared_on`/`relationship` restores the original declaring side and + type. A hand-authored relationship is declared on its `from` (many) cube as + `many_to_one`, which is the orientation Ossie already guarantees. + """ + joins_by_cube = {} + for rel in relationships: + rname = rel.get("name", "") + from_cols = rel.get("from_columns") or [] + to_cols = rel.get("to_columns") or [] + if not isinstance(from_cols, list) or not isinstance(to_cols, list) \ + or not from_cols or not to_cols: + raise ConversionError( + f"Relationship '{rname}': from_columns and to_columns are required " + f"lists") + if len(from_cols) != len(to_cols): + raise ConversionError( + f"Relationship '{rname}': from_columns ({len(from_cols)}) and " + f"to_columns ({len(to_cols)}) must have the same length") + + stash = read_stash(rel) + from_cube = cube_names[rel["from"]] + to_cube = cube_names[rel["to"]] + declared_on = stash.get("declared_on") + relationship = stash.get("relationship", "many_to_one") + + if declared_on == to_cube: + # The import flipped a one_to_many (or kept a one_to_one) declared on + # the other side; flip back to the original orientation. + own, other = to_cube, from_cube + own_cols, other_cols = to_cols, from_cols + else: + own, other = from_cube, to_cube + own_cols, other_cols = from_cols, to_cols + + join = {"name": other, "relationship": relationship} + if "sql" in stash: + join["sql"] = stash["sql"] + else: + join["sql"] = " AND ".join( + "{CUBE}." + str(a) + " = {" + other + "}." + str(b) + for a, b in zip(own_cols, other_cols)) + for key, value in stash.items(): + if key not in ("declared_on", "relationship", "sql"): + join[key] = value + if rel.get("ai_context"): + # A Cube join entry takes only name/sql/relationship -- no `meta` -- so + # unlike every other level there is nowhere to park this. + issues.add(IssueType.DROPPED_NO_CUBE_EQUIVALENT, + f"relationship '{rname}'", + "a Cube join carries no metadata field, so relationship " + "ai_context has nowhere to go and is dropped") + joins_by_cube.setdefault(own, []).append( + _ordered(join, ["name", "sql", "relationship"])) + return joins_by_cube + + +# --- measures ------------------------------------------------------------------- + +def _build_measures(model, cube_names, members_by_cube, inline_sql_by_cube, + pk_by_cube, datasets, relationships, base_cube, dialect, + issues): + """Group Ossie metrics into per-cube `measures` lists.""" + name = model.get("name", "") + sanitized = set(cube_names.values()) + base_cache = [] + + def resolve_base(): + if not base_cache: + base_cache.append(cube_names[_pick_base_cube( + name, datasets, relationships, base_cube)]) + return base_cache[0] + + measures_by_cube = {} + for metric in (model.get("metrics") or []): + mname_raw = require_str(metric, "name", "metric") + scope = f"metric '{mname_raw}'" + stash = read_stash(metric) + # An empty `taken` on purpose: a measure name only has to be unique within + # its own cube, and which cube this lands on is not known yet. `_place` + # rejects a collision once the target is decided. + mname = stash.get("name") or sanitize_name(mname_raw, scope, set()) + + if "measure" in stash: + # A prior import stashed the original measure (a filtered, calculated, + # or otherwise non-reconstructible one); restore it verbatim and + # re-inject the natively mapped metadata. + measure = dict(stash["measure"]) + measure["name"] = mname + _apply_measure_metadata(metric, measure, stash) + target = stash.get("cube") or resolve_base() + _place(measures_by_cube, target, measure, name) + continue + + expr = pick_expression(metric.get("expression"), dialect) + if expr is None: + issues.add(IssueType.NO_USABLE_DIALECT, scope, + "no ANSI_SQL or preferred-dialect expression; metric dropped") + continue + + referenced = referenced_datasets(expr, sanitized) + target = stash.get("cube") or ( + next(iter(referenced)) if len(referenced) == 1 else resolve_base()) + + if len(referenced) > 1: + # Cube resolves a cross-cube member reference by adding an implicit join, + # so the model needs a join path between these cubes -- which Ossie's + # expression does not state and this converter cannot verify. Reported for + # every shape the measure can take: it used to be raised only from the + # calculated-measure fallback, so a decomposed metric (the shape with the + # *most* cross-cube references) reported nothing at all. + issues.add(IssueType.APPROXIMATED, scope, + f"expression spans datasets {', '.join(sorted(referenced))}; " + f"Cube reaches the others from '{target}' through an implicit " + f"join, so verify a join path exists") + + spans = [] if stash.get("sql") else aggregate_spans(expr) + if len(spans) > 1: + # A composite metric: give each aggregate its own measure on the cube its + # operand belongs to, and let the public measure reference them. Cube then + # applies its row-multiplication correction per aggregate instead of + # seeing one opaque expression -- see _decompose_measure. + public_sql = _decompose_measure( + expr, spans, mname, target, measures_by_cube, members_by_cube, + inline_sql_by_cube, pk_by_cube, sanitized, name) + measure = {"name": mname, "sql": public_sql, "type": "number"} + else: + measure = _measure_from_expression( + expr, target, mname, stash, members_by_cube.get(target, set()), + inline_sql_by_cube, pk_by_cube.get(target, []), sanitized) + _apply_measure_metadata(metric, measure, stash) + _place(measures_by_cube, target, measure, name) + return measures_by_cube + + +def _decompose_measure(expr, spans, mname, fallback, measures_by_cube, + members_by_cube, inline_sql_by_cube, pk_by_cube, sanitized, + model_name): + """Emit one `public: false` measure per aggregate; return the sql referencing them. + + Cube corrects for row multiplication per measure, keyed on the cube that measure + sits on. A cross-dataset ratio emitted as a single calculated measure gets one + correction for the whole expression; split into a measure per aggregate, each on + the cube its operand comes from, each aggregate is corrected on its own terms. + That is why this is a correctness change and not a formatting one. + + Each part carries `meta.ossie.part_of` so import knows it is generated and skips + it, recovering the original expression by inlining the references instead. + """ + # A part name has to be free on whichever cube it lands on, and a Cube member name + # is unique across dimensions and measures alike -- so the check is over both, and + # over every cube rather than the one part it happens to land on. + taken = {m["name"].lower() for ms in measures_by_cube.values() for m in ms} + taken |= {n.lower() for ns in members_by_cube.values() for n in ns} + out, cursor, index = [], 0, 0 + for start, end in spans: + piece = expr[start:end] + # Each aggregate lands on the cube its own operand references. + refs = referenced_datasets(piece, sanitized) + part_target = next(iter(refs)) if len(refs) == 1 else fallback + + index += 1 + part_name = f"{mname}_part_{index}" + while part_name.lower() in taken: + index += 1 + part_name = f"{mname}_part_{index}" + taken.add(part_name.lower()) + + part = _measure_from_expression( + piece, part_target, part_name, {}, + members_by_cube.get(part_target, set()), inline_sql_by_cube, + pk_by_cube.get(part_target, []), sanitized) + part["public"] = False + part["meta"] = {"ossie": {"part_of": mname}} + _place(measures_by_cube, part_target, part, model_name) + + out.append(ossie_expr_to_cube_sql( + expr[cursor:start], fallback, members_by_cube.get(fallback, set()), + sanitized, inline_sql=inline_sql_by_cube)) + # `{CUBE.x}` for a part on the same cube as the public measure: an explicit + # name pins the reference to this cube and breaks if it is extended. + qualifier = "CUBE" if part_target == fallback else part_target + out.append("{" + f"{qualifier}.{part_name}" + "}") + cursor = end + out.append(ossie_expr_to_cube_sql( + expr[cursor:], fallback, members_by_cube.get(fallback, set()), sanitized, + inline_sql=inline_sql_by_cube)) + return "".join(out) + + +def _place(measures_by_cube, target, measure, model_name): + bucket = measures_by_cube.setdefault(target, []) + if any(m["name"].lower() == measure["name"].lower() for m in bucket): + raise ConversionError( + f"Model '{model_name}': two metrics map to measure " + f"'{measure['name']}' on cube '{target}'; rename one in the Ossie model.") + bucket.append(measure) + + +def _measure_from_expression(expr, target, mname, stash, members, inline_sql_by_cube, + primary_key, sanitized): + """Turn an Ossie metric expression back into a structured Cube measure. + + `COUNT(DISTINCT )` is Cube's bare `type: count` -- + which is how import renders it, precisely because that form stays correct + whether or not the cube is fanned out. A recognized aggregate over a single + operand becomes the matching `type` plus `sql`; anything else becomes a + calculated `type: number` measure carrying the whole expression. + """ + measure = {"name": mname} + m = _AGG_CALL_RE.match(expr) + if m and _balanced(m.group(2)): + func, inner = m.group(1).upper(), m.group(2).strip() + distinct = _DISTINCT_RE.match(inner) + if func == "COUNT" and distinct: + inner = distinct.group(1).strip() + if primary_key and inner == primary_key_operand(target, primary_key): + measure["type"] = "count" + return measure + func = "COUNT_DISTINCT" + # `COUNT(*)` deliberately falls through to the calculated measure below. + # A bare Cube `type: count` is this converter's representation of + # `COUNT(DISTINCT )` -- handled above -- so emitting one here + # would round-trip back as a different expression, and on a dataset with no + # primary key it would produce a measure the importer refuses. Cube renders + # `type: number` with `count(*)` natively (BaseQuery special-cases exactly + # that pair), so the expression survives intact either way. + if not (func == "COUNT" and inner == "*"): + agg = OSSIE_FUNC_TO_AGG.get(func) or ("count" if func == "COUNT" else None) + if agg is not None: + measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( + inner, target, members, sanitized, + inline_sql=inline_sql_by_cube) + measure["type"] = agg + return measure + + # A ratio, a window expression, or a multi-dataset aggregate: Cube expresses + # these as a calculated measure whose sql carries the aggregation. + measure["sql"] = stash.get("sql") or ossie_expr_to_cube_sql( + expr, target, members, sanitized, inline_sql=inline_sql_by_cube) + measure["type"] = "number" + return measure + + +def _apply_measure_metadata(metric, measure, stash): + if stash.get("title"): + measure["title"] = stash["title"] + if metric.get("description"): + measure["description"] = metric["description"] + parked = {} + foreign = foreign_vendor_extensions(metric) + if foreign: + parked["custom_extensions"] = foreign + meta = _build_meta(metric.get("ai_context"), stash.get("meta"), parked) + if meta: + measure["meta"] = meta + + +def _balanced(s): + depth = 0 + for ch in s: + if ch == "(": + depth += 1 + elif ch == ")": + depth -= 1 + if depth < 0: + return False + return depth == 0 + + +# --- views ---------------------------------------------------------------------- + +def _build_views(model, model_stash, cube_names, relationships, datasets, + base_cube): + """Return {file path: [view dict, ...]}. + + A list per path, not a single view: several views can share one YAML file, and + keying one view per path silently kept only the last. + + Stashed views restore verbatim, with the natively mapped description and AI + context re-injected on the mapped one. The `views` stash key being *present* -- + even empty -- means the original Cube model's view set is known, so a view is + only generated for hand-authored Ossie. + """ + # The model's foreign-vendor extensions have no Cube field, so they ride on the + # view that represents the model -- the mapped one, or the generated one. + parked = {} + foreign = foreign_vendor_extensions(model) + if foreign: + parked["custom_extensions"] = foreign + + out = {} + if "views" in model_stash: + mapped = model_stash.get("mapped_view") + paths = model_stash.get("view_files") or {} + if foreign and mapped is None: + # The model's own metadata rides on the view that represents it, and + # there isn't one: the source Cube model had several views and none was + # chosen. Dropping the extensions would be silent data loss, and + # picking a view arbitrarily would not survive a re-import (only the + # mapped view's parked extensions are restored). So this is refused + # with the fix in the message. + vendors = ", ".join( + sorted({str(e.get("vendor_name")) for e in foreign})) + raise ConversionError( + f"Model carries custom_extensions for {vendors}, which have no Cube " + f"field and ride on the view representing the model -- but no view " + f"is mapped, so there is nowhere to put them without losing them. " + f"Re-import naming the view the model maps to (`--view `), or " + f"remove the foreign-vendor extensions.") + for vname, view in (model_stash["views"] or {}).items(): + view = dict(view) + if vname == mapped: + if model.get("description"): + view["description"] = model["description"] + meta = _build_meta(model.get("ai_context"), view.get("meta"), parked) + if meta: + view["meta"] = meta + path = paths.get(vname) or view_file(vname) + out.setdefault(path, []).append(view) + return out + + vname = sanitize_name(model.get("name", "model"), "Model", set()) + view = {"name": vname} + if model.get("description"): + view["description"] = model["description"] + meta = _build_meta(model.get("ai_context"), None, parked) + if meta: + view["meta"] = meta + view["cubes"] = _view_cubes( + cube_names, relationships, + cube_names[_pick_base_cube(model.get("name", ""), datasets, + relationships, base_cube)]) + out[view_file(vname)] = [view] + return out + + +def _view_cubes(cube_names, relationships, base): + """Build a generated view's `cubes:` list: the base cube plus every cube + reachable from it, each addressed by its full `join_path`.""" + adjacency = {} + for rel in relationships: + a, b = cube_names[rel["from"]], cube_names[rel["to"]] + adjacency.setdefault(a, []).append(b) + adjacency.setdefault(b, []).append(a) + + entries = [{"join_path": base, "includes": "*"}] + paths = {base: base} + queue = deque([base]) + while queue: + current = queue.popleft() + for neighbor in adjacency.get(current, []): + if neighbor in paths: + continue + paths[neighbor] = f"{paths[current]}.{neighbor}" + entries.append({"join_path": paths[neighbor], "includes": "*"}) + queue.append(neighbor) + # A cube no relationship reaches cannot be addressed by a join path, so it is + # simply not part of the generated view; it is still exported and joinable. + return entries + + +def _pick_base_cube(model_name, datasets, relationships, hint): + """Choose the cube a generated view is rooted at: an explicit hint, else the + dataset that is never a relationship `to` (the FK sink of a many-to-one star).""" + if hint is not None: + if hint not in datasets: + raise ConversionError( + f"Model '{model_name}': requested base cube '{hint}' is not a dataset") + return hint + if len(datasets) == 1: + return next(iter(datasets)) + if not relationships: + raise ConversionError( + f"Model '{model_name}': {len(datasets)} datasets but no relationships; " + f"name the view's base cube with --base-cube.") + incoming = {name: 0 for name in datasets} + for rel in relationships: + incoming[rel["to"]] += 1 + roots = [n for n in datasets if incoming[n] == 0] + if not roots: + raise ConversionError( + f"Model '{model_name}': every dataset is a relationship target (the " + f"graph has a cycle); name the view's base cube with --base-cube.") + if len(roots) > 1: + raise ConversionError( + f"Model '{model_name}': multiple candidate base cubes {sorted(roots)}; " + f"name the view's base cube with --base-cube.") + return roots[0] diff --git a/converters/cube/tests/_roundtrip_helpers.py b/converters/cube/tests/_roundtrip_helpers.py new file mode 100644 index 00000000..2343f683 --- /dev/null +++ b/converters/cube/tests/_roundtrip_helpers.py @@ -0,0 +1,213 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared model builders and round-trip assertions for property-based tests. + +This module is deliberately free of any third-party test dependency (no +hypothesis, no pytest) so the generation and assertion logic can run two ways: + + - driven by Hypothesis strategies (see test_roundtrip_properties.py), and + - driven by a plain seeded `random.Random` (RandomRnd below), which is how the + logic is exercised when hypothesis is not installed. + +Both drivers implement the small `Rnd` interface (chance/count/pick/text); the +builders depend only on that interface, so the generated model space is identical +either way. + +The builders generate within the *round-trippable subset* -- the shapes the +converter reproduces exactly. Known normalizations are avoided by construction: + + - names are generated already valid as Cube identifiers, so the sanitizer never + renames anything; + - the topology is a star with a single fact, so there is one unambiguous FK sink + and no cycle; + - every cube declares a primary key, which a bare `type: count` needs; + - `sum`/`avg` measures are only placed on the fact cube, which is never the + `to` side of a join -- a non-idempotent aggregate on a fanned-out cube is + refused by design, and that refusal has its own targeted tests; + - a view lists every cube, so dataset ordering is pinned by the view rather + than by file names. + +Name fuzzing (collisions, reserved words) and the fan-out refusal are left to the +targeted unit tests, which assert the converter *rejects* or *reports* those. +""" + +import random +import string + +from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube +from ossie_cube._common import dump_yaml, load_yaml + +# Aggregates whose value survives duplicate rows, so they are safe on any cube. +IDEMPOTENT_AGGS = ["count_distinct", "count_distinct_approx", "min", "max"] +# Aggregates only placed on the fact cube; see the module docstring. +FACT_ONLY_AGGS = ["sum", "avg"] + +DIM_TYPES = ["string", "number", "boolean", "time"] + + +class RandomRnd: + """The `Rnd` interface backed by a seeded `random.Random`.""" + + def __init__(self, seed): + self.r = random.Random(seed) + + def chance(self, p=0.5): + return self.r.random() < p + + def count(self, lo, hi): + return self.r.randint(lo, hi) + + def pick(self, seq): + return self.r.choice(list(seq)) + + def text(self): + # Alphanumeric with optional interior spaces; no leading/trailing space and + # no YAML-special characters, so the value survives a dump/load cycle + # verbatim. + alnum = string.ascii_letters + string.digits + words = [] + for _ in range(self.r.randint(1, 3)): + words.append("".join( + self.r.choice(alnum) for _ in range(self.r.randint(1, 6)))) + return " ".join(words) + + +def build_cube_model(rnd): + """Generate a Cube model as {relative filename: YAML str}.""" + dim_count = rnd.count(1, 3) + dim_names = [f"dim_{i}" for i in range(dim_count)] + fact = "fact" + + cubes = {} + cubes[fact] = _build_cube(rnd, fact, is_fact=True, dim_names=dim_names) + for name in dim_names: + cubes[name] = _build_cube(rnd, name, is_fact=False, dim_names=()) + + files = {} + for name, cube in cubes.items(): + files[f"model/cubes/{name}.yml"] = dump_yaml({"cubes": [cube]}) + + view = {"name": "main"} + if rnd.chance(0.6): + view["description"] = rnd.text() + if rnd.chance(0.6): + view["meta"] = {"ai_context": rnd.text()} + view["cubes"] = ( + [{"join_path": fact, "includes": "*"}] + + [{"join_path": f"{fact}.{d}", "includes": "*"} for d in dim_names] + ) + files["model/views/main.yml"] = dump_yaml({"views": [view]}) + return files + + +def _build_cube(rnd, name, is_fact, dim_names): + cube = {"name": name} + if rnd.chance(0.3): + cube["sql"] = f"SELECT * FROM raw.{name}" + else: + cube["sql_table"] = f"public.{name}" + if rnd.chance(0.5): + cube["description"] = rnd.text() + + if is_fact and dim_names: + cube["joins"] = [ + {"name": d, "sql": "{CUBE}." + f"{d}_id" + " = {" + f"{d}.id" + "}", + "relationship": "many_to_one"} + for d in dim_names + ] + + dimensions = [{"name": "id", "sql": "id", "type": "number", + "primary_key": True}] + for d in dim_names: + dimensions.append({"name": f"{d}_id", "sql": f"{d}_id", "type": "number"}) + for i in range(rnd.count(0, 3)): + dimensions.append(_build_dimension(rnd, f"attr_{i}")) + if rnd.chance(0.25): + dimensions.append({ + "name": "place", "type": "geo", + "latitude": {"sql": "{CUBE}.lat"}, + "longitude": {"sql": "{CUBE}.lon"}, + }) + cube["dimensions"] = dimensions + + # Every cube carries a bare `count`, which collides across cubes and so + # exercises the `__` qualification on import. + measures = [{"name": "count", "type": "count"}] + aggs = IDEMPOTENT_AGGS + (FACT_ONLY_AGGS if is_fact else []) + for i in range(rnd.count(0, 2)): + measure = {"name": f"m_{i}", "sql": "{CUBE}.value", "type": rnd.pick(aggs)} + if rnd.chance(0.4): + measure["description"] = rnd.text() + if rnd.chance(0.3): + measure["meta"] = {"ai_context": rnd.text()} + if rnd.chance(0.25): + measure["format"] = "currency" + measures.append(measure) + cube["measures"] = measures + return cube + + +def _build_dimension(rnd, name): + dtype = rnd.pick(DIM_TYPES) + dim = {"name": name, "type": dtype} + if rnd.chance(0.3): + # A computed expression, which import translates and stashes verbatim. + dim["sql"] = "LOWER({CUBE}." + name + ")" if dtype == "string" \ + else "{CUBE}." + name + else: + dim["sql"] = name + if rnd.chance(0.4): + dim["title"] = rnd.text() + if rnd.chance(0.4): + dim["description"] = rnd.text() + if rnd.chance(0.3): + dim["meta"] = {"ai_context": rnd.text()} + if rnd.chance(0.2): + dim["format"] = "percent" if dtype == "number" else None + if dim["format"] is None: + del dim["format"] + return dim + + +def _parse_files(files): + # Same documented normalization the fixture tests use; see _util.canon_sql. + from _util import canon_sql + return {name: canon_sql(load_yaml(text, name)) + for name, text in files.items()} + + +def assert_cube_roundtrip_is_lossless(files): + """Cube -> Ossie -> Cube reproduces the model structurally.""" + ossie, _ = convert_cube_to_ossie(files) + files2, _ = convert_ossie_to_cube(ossie) + assert _parse_files(files2) == _parse_files(files), ( + "Cube -> Ossie -> Cube changed the model") + + +def assert_ossie_roundtrip_is_lossless(files): + """Ossie -> Cube -> Ossie reproduces the model too.""" + ossie, _ = convert_cube_to_ossie(files) + files2, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files2) + assert load_yaml(ossie2) == load_yaml(ossie), ( + "Ossie -> Cube -> Ossie changed the model") + + +def check_model(files): + assert_cube_roundtrip_is_lossless(files) + assert_ossie_roundtrip_is_lossless(files) diff --git a/converters/cube/tests/_util.py b/converters/cube/tests/_util.py new file mode 100644 index 00000000..5e18bd28 --- /dev/null +++ b/converters/cube/tests/_util.py @@ -0,0 +1,142 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Shared test helpers: fixture loading and structural lookup.""" + +import copy +import json +import pathlib +import re + +from ossie_cube._common import load_yaml # src is on sys.path via conftest.py + +FIXTURES = pathlib.Path(__file__).resolve().parent / "fixtures" +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] + + +def load_fixture(name): + with open(FIXTURES / name) as fh: + return fh.read() + + +def load_fixture_dir(name): + """Read a fixture Cube model directory as {relative posix path: text}.""" + root = FIXTURES / name + files = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + files[path.relative_to(root).as_posix()] = path.read_text() + return files + + +def parse(yaml_str): + return load_yaml(yaml_str) + + +_ALIAS_DOT_RE = re.compile(r"\$?\{\s*(?:CUBE|TABLE)\s*\}\s*\.") + + +def canon_sql(node): + """Canonicalize the one documented Cube SQL normalization, in place-ish. + + `{CUBE}.column` and a bare `column` are the same thing -- a raw physical column + of the owning cube -- so the converter no longer stashes the original spelling + just to reproduce it. Round-trip assertions therefore compare with the alias + prefix removed. + + Deliberately narrow: `{CUBE.member}` is a *member* reference and means something + else, so it is left alone (and is still stashed, so it round-trips exactly). + """ + if isinstance(node, dict): + return {k: (_ALIAS_DOT_RE.sub("", v) + if k == "sql" and isinstance(v, str) else canon_sql(v)) + for k, v in node.items()} + if isinstance(node, list): + return [canon_sql(v) for v in node] + return node + + +def parse_files(files): + """Parse a Cube model dict into the form round-trip fidelity is asserted on. + + Comments and key order are not part of the data model, so comparison happens on + parsed structures. A non-YAML file (a `.js` model preserved verbatim) is + compared as text. + + Also applies `canon_sql`: the converter no longer stashes a member's exact SQL + spelling just to reproduce `{CUBE}.column` over a bare `column`, since the two + mean the same thing and stashing it put noise into every other converter's view + of the model. That spelling is therefore a documented normalization, not a + difference worth failing on. + """ + out = {} + for name, text in files.items(): + out[name] = (canon_sql(load_yaml(text, name)) + if name.lower().endswith((".yml", ".yaml")) else text) + return out + + +def model_of(ossie_yaml): + """The sole semantic model of an Ossie document.""" + doc = parse(ossie_yaml) + assert len(doc["semantic_model"]) == 1 + return doc["semantic_model"][0] + + +def by_name(items): + """Index a list of named Ossie objects by `name`.""" + return {item["name"]: item for item in items or []} + + +def expr_of(item, dialect="ANSI_SQL"): + """The expression string of an Ossie field or metric in a given dialect.""" + for entry in item["expression"]["dialects"]: + if entry["dialect"] == dialect: + return entry["expression"] + raise AssertionError(f"{item['name']} has no {dialect} expression") + + +def stash_of(item, vendor="CUBE"): + """The parsed vendor stash on an Ossie object, or {} when absent.""" + for ext in item.get("custom_extensions") or []: + if ext["vendor_name"] == vendor: + data = json.loads(ext["data"]) + data.pop("_v", None) + return data + return {} + + +def canon(obj): + """Deep-copy with every `custom_extensions[].data` JSON string parsed into a + dict, so comparisons are insensitive to JSON key order and whitespace.""" + obj = copy.deepcopy(obj) + + def walk(node): + if isinstance(node, dict): + for key, value in node.items(): + if key == "custom_extensions" and isinstance(value, list): + for ext in value: + if isinstance(ext, dict) and isinstance(ext.get("data"), str): + ext["data"] = json.loads(ext["data"]) + else: + walk(value) + elif isinstance(node, list): + for item in node: + walk(item) + + walk(obj) + return obj diff --git a/converters/cube/tests/conftest.py b/converters/cube/tests/conftest.py new file mode 100644 index 00000000..254b3d75 --- /dev/null +++ b/converters/cube/tests/conftest.py @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pathlib +import sys + +# Make the converter modules in ../src, and this directory's own helpers, +# importable from the tests. +_HERE = pathlib.Path(__file__).resolve().parent +sys.path.insert(0, str(_HERE)) +sys.path.insert(0, str(_HERE.parent / "src")) diff --git a/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml new file mode 100644 index 00000000..a28395ab --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/orders.yml @@ -0,0 +1,68 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The fact cube: sits on the many side of the join, so its sum/avg measures are +# not exposed to row multiplication. Exercises a bare `count` (which maps through +# the primary key), a filtered sum, a calculated measure that references two other +# measures, and `meta.ai_context` on both a dimension and a measure. +cubes: + - name: orders + sql_table: public.orders + description: Customer orders + joins: + - name: users + sql: "{CUBE}.user_id = {users}.id" + relationship: many_to_one + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: user_id + sql: user_id + type: number + - name: status + sql: status + type: string + title: Order Status + description: Current order status + meta: + ai_context: Values are pending, shipped, and completed. + - name: created_at + sql: created_at + type: time + - name: is_large + sql: "{CUBE}.amount > 500" + type: boolean + measures: + - name: count + type: count + - name: total_amount + sql: "{CUBE}.amount" + type: sum + description: Total order amount + format: currency + meta: + ai_context: Use this for revenue questions. + - name: completed_amount + sql: "{CUBE}.amount" + type: sum + filters: + - sql: "{CUBE}.status = 'completed'" + - name: avg_order_value + sql: "{total_amount} / {count}" + type: number diff --git a/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml new file mode 100644 index 00000000..d7b59532 --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_cube/model/cubes/users.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The dimension cube: sits on the one side of the join, so it is exposed to row +# multiplication -- both of its measures are deliberately fan-out-safe (a bare +# `count`, which maps to COUNT(DISTINCT ), and a count_distinct). Also +# exercises a `sql`-defined cube, a geo dimension (which splits into two Ossie +# fields), and a segment (which has no Ossie form and rides in the stash). +cubes: + - name: users + sql: SELECT * FROM public.users WHERE deleted_at IS NULL + dimensions: + - name: id + sql: id + type: number + primary_key: true + - name: city + sql: city + type: string + - name: location + type: geo + latitude: + sql: "{CUBE}.lat" + longitude: + sql: "{CUBE}.lon" + measures: + - name: count + type: count + - name: cities + sql: "{CUBE}.city" + type: count_distinct + segments: + - name: active + sql: "{CUBE}.status = 'active'" diff --git a/converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml b/converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml new file mode 100644 index 00000000..b6f0d584 --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_cube/model/views/sales.yml @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# The sole view, so it is the mapped one: its name, description, and +# meta.ai_context become the Ossie model's. The `cubes:` curation has no Ossie +# form and round-trips through the model-level stash. +views: + - name: sales + description: Sales overview + meta: + ai_context: > + Primary view for revenue analysis. Use it for any question about + sales, orders, or customer spend. + cubes: + - join_path: orders + includes: "*" + - join_path: orders.users + includes: + - city diff --git a/converters/cube/tests/fixtures/fixtureA_ossie.yaml b/converters/cube/tests/fixtures/fixtureA_ossie.yaml new file mode 100644 index 00000000..577fd8ed --- /dev/null +++ b/converters/cube/tests/fixtures/fixtureA_ossie.yaml @@ -0,0 +1,181 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# The Ossie form of fixtureA_cube/, asserted as a whole-document snapshot so an +# unintended change anywhere in the output shows up as a readable diff. +# Regenerate with: uv run ossie-cube import -i tests/fixtures/fixtureA_cube + +version: 0.2.0.dev0 +semantic_model: +- name: sales + description: Sales overview + ai_context: + instructions: | + Primary view for revenue analysis. Use it for any question about sales, orders, or customer spend. + relationships: + - name: orders_to_users + from: orders + to: users + from_columns: + - user_id + to_columns: + - id + datasets: + - name: orders + source: public.orders + description: Customer orders + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Decimal + - name: user_id + expression: + dialects: + - dialect: ANSI_SQL + expression: user_id + datatype: Decimal + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: status + datatype: String + label: Order Status + description: Current order status + ai_context: + instructions: Values are pending, shipped, and completed. + - name: created_at + expression: + dialects: + - dialect: ANSI_SQL + expression: created_at + datatype: DateTime + dimension: + is_time: true + - name: is_large + expression: + dialects: + - dialect: ANSI_SQL + expression: amount > 500 + datatype: Boolean + primary_key: + - id + - name: users + source: SELECT * FROM public.users WHERE deleted_at IS NULL + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Decimal + - name: city + expression: + dialects: + - dialect: ANSI_SQL + expression: city + datatype: String + - name: location_latitude + expression: + dialects: + - dialect: ANSI_SQL + expression: lat + datatype: Float + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "geo": {"of": "location", "part": "latitude", "sql": "{CUBE}.lat"}}' + - name: location_longitude + expression: + dialects: + - dialect: ANSI_SQL + expression: lon + datatype: Float + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "geo": {"of": "location", "part": "longitude", "sql": "{CUBE}.lon"}}' + primary_key: + - id + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube_extras": {"segments": [{"name": "active", "sql": "{CUBE}.status + = ''active''"}]}}' + metrics: + - name: orders__count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT orders.id) + datatype: Integer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "name": "count"}' + - name: total_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + description: Total order amount + ai_context: + instructions: Use this for revenue questions. + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "measure": {"name": "total_amount", "sql": + "{CUBE}.amount", "type": "sum", "format": "currency"}}' + - name: completed_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(CASE WHEN (orders.status = 'completed') THEN orders.amount + END) + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "measure": {"name": "completed_amount", "sql": + "{CUBE}.amount", "type": "sum", "filters": [{"sql": "{CUBE}.status = ''completed''"}]}}' + - name: avg_order_value + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) / COUNT(DISTINCT orders.id) + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "orders", "measure": {"name": "avg_order_value", "sql": + "{total_amount} / {count}", "type": "number"}}' + - name: users__count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT users.id) + datatype: Integer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "users", "name": "count"}' + - name: cities + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT users.city) + datatype: Integer + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "users"}' + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "views": {"sales": {"name": "sales", "cubes": [{"join_path": + "orders", "includes": "*"}, {"join_path": "orders.users", "includes": ["city"]}]}}, + "mapped_view": "sales"}' diff --git a/converters/cube/tests/fixtures/hand_authored_ossie.yaml b/converters/cube/tests/fixtures/hand_authored_ossie.yaml new file mode 100644 index 00000000..e62f18f4 --- /dev/null +++ b/converters/cube/tests/fixtures/hand_authored_ossie.yaml @@ -0,0 +1,95 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# An Ossie model authored by hand rather than imported from Cube: no +# custom_extensions[CUBE] stash anywhere. Export therefore has to derive +# everything -- the cube layout, the join orientation, and a generated view -- +# instead of restoring it. Also carries the constructs Cube has no field for +# (unique_keys, a foreign vendor's extensions, structured ai_context), which +# export parks under meta.ossie. + +version: 0.2.0.dev0 +semantic_model: +- name: ecommerce + description: Orders and customers + ai_context: + instructions: Use for sales analysis. + synonyms: + - sales + - purchases + datasets: + - name: orders + source: sales.public.orders + primary_key: + - id + unique_keys: + - - order_number + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Integer + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + datatype: Integer + - name: ordered_at + expression: + dialects: + - dialect: ANSI_SQL + expression: ordered_at + datatype: Date + custom_extensions: + - vendor_name: SNOWFLAKE + data: '{"warehouse": "ANALYTICS_WH"}' + - name: customers + source: sales.public.customers + primary_key: + - id + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + datatype: Integer + - name: email + expression: + dialects: + - dialect: ANSI_SQL + expression: LOWER(email) + datatype: String + relationships: + - name: orders_to_customers + from: orders + to: customers + from_columns: + - customer_id + to_columns: + - id + metrics: + - name: total_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + description: Total revenue + datatype: Decimal diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml new file mode 100644 index 00000000..206ea8d8 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/customer.yml @@ -0,0 +1,83 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: customer + sql_table: tpcds.public.customer + description: Customer dimension with demographic information + meta: + ai_context: 'Also known as: customers, shoppers, buyers.' + ossie: + unique_keys: + - - c_customer_sk + ai_context: + synonyms: + - customers + - shoppers + - buyers + dimensions: + - name: c_customer_sk + sql: c_customer_sk + type: number + primary_key: true + description: Surrogate key for customer + - name: c_customer_id + sql: c_customer_id + type: string + description: Business key for customer + meta: + ai_context: 'Also known as: customer ID, customer number.' + ossie: + ai_context: + synonyms: + - customer ID + - customer number + - name: c_first_name + sql: c_first_name + type: string + description: Customer first name + - name: c_last_name + sql: c_last_name + type: string + description: Customer last name + - name: customer_full_name + sql: c_first_name || ' ' || c_last_name + type: string + description: Customer full name (computed field) + meta: + ai_context: 'Also known as: full name, customer name.' + ossie: + ai_context: + synonyms: + - full name + - customer name + - name: c_email_address + sql: c_email_address + type: string + description: Customer email address + meta: + ai_context: 'Also known as: email, contact.' + ossie: + ai_context: + synonyms: + - email + - contact diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml new file mode 100644 index 00000000..8855e811 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/date_dim.yml @@ -0,0 +1,84 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: date_dim + sql_table: tpcds.public.date_dim + description: Date dimension with calendar attributes + meta: + ai_context: 'Also known as: calendar, dates, time periods.' + ossie: + unique_keys: + - - d_date_sk + ai_context: + synonyms: + - calendar + - dates + - time periods + dimensions: + - name: d_date_sk + sql: d_date_sk + type: number + primary_key: true + description: Surrogate key for date + - name: d_date + sql: d_date + type: time + description: Actual date value + meta: + ai_context: 'Also known as: date, calendar date.' + ossie: + ai_context: + synonyms: + - date + - calendar date + - name: d_year + sql: d_year + type: number + description: Year + meta: + ai_context: 'Also known as: year.' + ossie: + ai_context: + synonyms: + - year + - name: d_quarter_name + sql: d_quarter_name + type: time + description: Quarter name (e.g., 2024Q1) + meta: + ai_context: 'Also known as: quarter, fiscal quarter.' + ossie: + ai_context: + synonyms: + - quarter + - fiscal quarter + - name: d_month_name + sql: d_month_name + type: time + description: Month name + meta: + ai_context: 'Also known as: month.' + ossie: + ai_context: + synonyms: + - month diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml new file mode 100644 index 00000000..79cf11de --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/item.yml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: item + sql_table: tpcds.public.item + description: Item/Product dimension with product attributes + meta: + ai_context: 'Also known as: products, items, merchandise.' + ossie: + unique_keys: + - - i_item_sk + ai_context: + synonyms: + - products + - items + - merchandise + dimensions: + - name: i_item_sk + sql: i_item_sk + type: number + primary_key: true + description: Surrogate key for item + - name: i_item_id + sql: i_item_id + type: string + description: Business key for item + meta: + ai_context: 'Also known as: item ID, product ID, SKU.' + ossie: + ai_context: + synonyms: + - item ID + - product ID + - SKU + - name: i_item_desc + sql: i_item_desc + type: string + description: Item description + meta: + ai_context: 'Also known as: product description, item name.' + ossie: + ai_context: + synonyms: + - product description + - item name + - name: i_brand + sql: i_brand + type: string + description: Brand name + meta: + ai_context: 'Also known as: brand, manufacturer.' + ossie: + ai_context: + synonyms: + - brand + - manufacturer + - name: i_category + sql: i_category + type: string + description: Item category + meta: + ai_context: 'Also known as: product category, department.' + ossie: + ai_context: + synonyms: + - product category + - department + - name: i_current_price + sql: i_current_price + type: number + description: Current price of the item + meta: + ai_context: 'Also known as: price, list price.' + ossie: + ai_context: + synonyms: + - price + - list price diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml new file mode 100644 index 00000000..c3d59832 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store.yml @@ -0,0 +1,97 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: store + sql_table: tpcds.public.store + description: Store dimension with location and store attributes + meta: + ai_context: 'Also known as: stores, retail locations, branches.' + ossie: + unique_keys: + - - s_store_id + ai_context: + synonyms: + - stores + - retail locations + - branches + dimensions: + - name: s_store_sk + sql: s_store_sk + type: number + primary_key: true + description: Surrogate key for store + - name: s_store_id + sql: s_store_id + type: string + description: Business key for store + meta: + ai_context: 'Also known as: store ID, store number.' + ossie: + ai_context: + synonyms: + - store ID + - store number + - name: s_store_name + sql: s_store_name + type: string + description: Store name + meta: + ai_context: 'Also known as: store name, location name.' + ossie: + ai_context: + synonyms: + - store name + - location name + - name: s_city + sql: s_city + type: string + description: City where store is located + meta: + ai_context: 'Also known as: city, location.' + ossie: + ai_context: + synonyms: + - city + - location + - name: s_state + sql: s_state + type: string + description: State where store is located + meta: + ai_context: 'Also known as: state, region.' + ossie: + ai_context: + synonyms: + - state + - region + - name: s_number_employees + sql: s_number_employees + type: number + description: Number of employees at the store + meta: + ai_context: 'Also known as: employee count, staff size.' + ossie: + ai_context: + synonyms: + - employee count + - staff size diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml new file mode 100644 index 00000000..45eb7c44 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/cubes/store_sales.yml @@ -0,0 +1,211 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +cubes: +- name: store_sales + sql_table: tpcds.public.store_sales + description: Fact table containing all store sales transactions + meta: + ai_context: 'Also known as: sales transactions, store purchases, retail sales, + POS data.' + ossie: + unique_keys: + - - ss_item_sk + - ss_ticket_number + ai_context: + synonyms: + - sales transactions + - store purchases + - retail sales + - POS data + joins: + - name: date_dim + sql: '{CUBE}.ss_sold_date_sk = {date_dim}.d_date_sk' + relationship: many_to_one + - name: customer + sql: '{CUBE}.ss_customer_sk = {customer}.c_customer_sk' + relationship: many_to_one + - name: item + sql: '{CUBE}.ss_item_sk = {item}.i_item_sk' + relationship: many_to_one + - name: store + sql: '{CUBE}.ss_store_sk = {store}.s_store_sk' + relationship: many_to_one + dimensions: + - name: ss_sold_date_sk + sql: ss_sold_date_sk + type: number + description: Foreign key to date dimension + meta: + ai_context: 'Also known as: sale date, transaction date.' + ossie: + ai_context: + synonyms: + - sale date + - transaction date + - name: ss_item_sk + sql: ss_item_sk + type: number + primary_key: true + description: Foreign key to item dimension + meta: + ai_context: 'Also known as: product, item.' + ossie: + ai_context: + synonyms: + - product + - item + - name: ss_customer_sk + sql: ss_customer_sk + type: number + description: Foreign key to customer dimension + meta: + ai_context: 'Also known as: customer, buyer.' + ossie: + ai_context: + synonyms: + - customer + - buyer + - name: ss_store_sk + sql: ss_store_sk + type: number + description: Foreign key to store dimension + meta: + ai_context: 'Also known as: store, location.' + ossie: + ai_context: + synonyms: + - store + - location + - name: ss_quantity + sql: ss_quantity + type: number + description: Quantity of items sold + meta: + ai_context: 'Also known as: units sold, quantity.' + ossie: + ai_context: + synonyms: + - units sold + - quantity + - name: ss_sales_price + sql: ss_sales_price + type: number + description: Sales price per unit + meta: + ai_context: 'Also known as: unit price, price.' + ossie: + ai_context: + synonyms: + - unit price + - price + - name: ss_ext_sales_price + sql: ss_ext_sales_price + type: number + description: Extended sales price (quantity * price) + meta: + ai_context: 'Also known as: total price, line total.' + ossie: + ai_context: + synonyms: + - total price + - line total + - name: ss_net_profit + sql: ss_net_profit + type: number + description: Net profit from the sale + meta: + ai_context: 'Also known as: profit, margin.' + ossie: + ai_context: + synonyms: + - profit + - margin + - name: ss_ticket_number + sql: ss_ticket_number + type: string + primary_key: true + public: false + measures: + - name: total_sales + sql: '{CUBE}.ss_ext_sales_price' + type: sum + description: Total sales revenue across all transactions + meta: + ai_context: 'Also known as: total revenue, gross sales, sales amount.' + ossie: + ai_context: + synonyms: + - total revenue + - gross sales + - sales amount + - name: total_profit + sql: '{CUBE}.ss_net_profit' + type: sum + description: Total net profit from store sales + meta: + ai_context: 'Also known as: net profit, total earnings, profit.' + ossie: + ai_context: + synonyms: + - net profit + - total earnings + - profit + - name: customer_lifetime_value + sql: SUM({CUBE}.ss_ext_sales_price) / COUNT(DISTINCT {customer.c_customer_sk}) + type: number + description: Average lifetime sales value per customer + meta: + ai_context: 'Also known as: CLV, LTV, customer value, lifetime revenue.' + ossie: + ai_context: + synonyms: + - CLV + - LTV + - customer value + - lifetime revenue + - name: sales_by_brand + sql: '{CUBE}.ss_ext_sales_price' + type: sum + description: Total sales by brand (requires grouping by item.i_brand) + meta: + ai_context: 'Also known as: brand sales, brand performance, brand revenue.' + ossie: + ai_context: + synonyms: + - brand sales + - brand performance + - brand revenue + - name: store_productivity + sql: SUM({CUBE}.ss_ext_sales_price) / NULLIF(SUM({store.s_number_employees}), + 0) + type: number + description: Sales per employee across stores + meta: + ai_context: 'Also known as: sales per employee, employee productivity, revenue + per employee.' + ossie: + ai_context: + synonyms: + - sales per employee + - employee productivity + - revenue per employee diff --git a/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml b/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml new file mode 100644 index 00000000..b4694965 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_cube/model/views/tpcds_retail_model.yml @@ -0,0 +1,60 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Generated from examples/tpcds_semantic_model.yaml by `ossie-cube export`, then +# kept as a fixture: the converter guide asks every converter to use the TPC-DS +# model as its baseline. Exercises a five-cube star, cross-cube calculated +# measures, a synthesized primary-key dimension, and meta.ossie parking. + +views: +- name: tpcds_retail_model + description: TPC-DS retail semantic model for sales and customer analytics + meta: + ai_context: Use this semantic model for retail analytics. It provides comprehensive + sales, customer, product, and store data from the TPC-DS benchmark. The model + supports time-based analysis, customer segmentation, product performance, and + store operations metrics. + ossie: + custom_extensions: + - vendor_name: SALESFORCE + data: | + { + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "0.1.1" + } + } + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' + cubes: + - join_path: store_sales + includes: '*' + - join_path: store_sales.date_dim + includes: '*' + - join_path: store_sales.customer + includes: '*' + - join_path: store_sales.item + includes: '*' + - join_path: store_sales.store + includes: '*' diff --git a/converters/cube/tests/fixtures/tpcds_ossie.yaml b/converters/cube/tests/fixtures/tpcds_ossie.yaml new file mode 100644 index 00000000..0b323fb9 --- /dev/null +++ b/converters/cube/tests/fixtures/tpcds_ossie.yaml @@ -0,0 +1,558 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# The Ossie form of tpcds_cube/, asserted as a whole-document snapshot. +# Regenerate with: uv run ossie-cube import -i tests/fixtures/tpcds_cube + +version: 0.2.0.dev0 +semantic_model: +- name: tpcds_retail_model + description: TPC-DS retail semantic model for sales and customer analytics + ai_context: + instructions: Use this semantic model for retail analytics. It provides comprehensive + sales, customer, product, and store data from the TPC-DS benchmark. The model + supports time-based analysis, customer segmentation, product performance, and + store operations metrics. + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - d_date_sk + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk + datasets: + - name: store_sales + source: tpcds.public.store_sales + description: Fact table containing all store sales transactions + ai_context: + synonyms: + - sales transactions + - store purchases + - retail sales + - POS data + unique_keys: + - - ss_item_sk + - ss_ticket_number + fields: + - name: ss_sold_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sold_date_sk + datatype: Decimal + description: Foreign key to date dimension + ai_context: + synonyms: + - sale date + - transaction date + - name: ss_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_item_sk + datatype: Decimal + description: Foreign key to item dimension + ai_context: + synonyms: + - product + - item + - name: ss_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_customer_sk + datatype: Decimal + description: Foreign key to customer dimension + ai_context: + synonyms: + - customer + - buyer + - name: ss_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_store_sk + datatype: Decimal + description: Foreign key to store dimension + ai_context: + synonyms: + - store + - location + - name: ss_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_quantity + datatype: Decimal + description: Quantity of items sold + ai_context: + synonyms: + - units sold + - quantity + - name: ss_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sales_price + datatype: Decimal + description: Sales price per unit + ai_context: + synonyms: + - unit price + - price + - name: ss_ext_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ext_sales_price + datatype: Decimal + description: Extended sales price (quantity * price) + ai_context: + synonyms: + - total price + - line total + - name: ss_net_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_net_profit + datatype: Decimal + description: Net profit from the sale + ai_context: + synonyms: + - profit + - margin + - name: ss_ticket_number + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ticket_number + datatype: String + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "public": false}' + primary_key: + - ss_item_sk + - ss_ticket_number + - name: date_dim + source: tpcds.public.date_dim + description: Date dimension with calendar attributes + ai_context: + synonyms: + - calendar + - dates + - time periods + unique_keys: + - - d_date_sk + fields: + - name: d_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date_sk + datatype: Decimal + description: Surrogate key for date + - name: d_date + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date + datatype: DateTime + dimension: + is_time: true + description: Actual date value + ai_context: + synonyms: + - date + - calendar date + - name: d_year + expression: + dialects: + - dialect: ANSI_SQL + expression: d_year + datatype: Decimal + description: Year + ai_context: + synonyms: + - year + - name: d_quarter_name + expression: + dialects: + - dialect: ANSI_SQL + expression: d_quarter_name + datatype: DateTime + dimension: + is_time: true + description: Quarter name (e.g., 2024Q1) + ai_context: + synonyms: + - quarter + - fiscal quarter + - name: d_month_name + expression: + dialects: + - dialect: ANSI_SQL + expression: d_month_name + datatype: DateTime + dimension: + is_time: true + description: Month name + ai_context: + synonyms: + - month + primary_key: + - d_date_sk + - name: customer + source: tpcds.public.customer + description: Customer dimension with demographic information + ai_context: + synonyms: + - customers + - shoppers + - buyers + unique_keys: + - - c_customer_sk + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_sk + datatype: Decimal + description: Surrogate key for customer + - name: c_customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_id + datatype: String + description: Business key for customer + ai_context: + synonyms: + - customer ID + - customer number + - name: c_first_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name + datatype: String + description: Customer first name + - name: c_last_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_last_name + datatype: String + description: Customer last name + - name: customer_full_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name || ' ' || c_last_name + datatype: String + description: Customer full name (computed field) + ai_context: + synonyms: + - full name + - customer name + - name: c_email_address + expression: + dialects: + - dialect: ANSI_SQL + expression: c_email_address + datatype: String + description: Customer email address + ai_context: + synonyms: + - email + - contact + primary_key: + - c_customer_sk + - name: item + source: tpcds.public.item + description: Item/Product dimension with product attributes + ai_context: + synonyms: + - products + - items + - merchandise + unique_keys: + - - i_item_sk + fields: + - name: i_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_sk + datatype: Decimal + description: Surrogate key for item + - name: i_item_id + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_id + datatype: String + description: Business key for item + ai_context: + synonyms: + - item ID + - product ID + - SKU + - name: i_item_desc + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_desc + datatype: String + description: Item description + ai_context: + synonyms: + - product description + - item name + - name: i_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: i_brand + datatype: String + description: Brand name + ai_context: + synonyms: + - brand + - manufacturer + - name: i_category + expression: + dialects: + - dialect: ANSI_SQL + expression: i_category + datatype: String + description: Item category + ai_context: + synonyms: + - product category + - department + - name: i_current_price + expression: + dialects: + - dialect: ANSI_SQL + expression: i_current_price + datatype: Decimal + description: Current price of the item + ai_context: + synonyms: + - price + - list price + primary_key: + - i_item_sk + - name: store + source: tpcds.public.store + description: Store dimension with location and store attributes + ai_context: + synonyms: + - stores + - retail locations + - branches + unique_keys: + - - s_store_id + fields: + - name: s_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_sk + datatype: Decimal + description: Surrogate key for store + - name: s_store_id + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_id + datatype: String + description: Business key for store + ai_context: + synonyms: + - store ID + - store number + - name: s_store_name + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_name + datatype: String + description: Store name + ai_context: + synonyms: + - store name + - location name + - name: s_city + expression: + dialects: + - dialect: ANSI_SQL + expression: s_city + datatype: String + description: City where store is located + ai_context: + synonyms: + - city + - location + - name: s_state + expression: + dialects: + - dialect: ANSI_SQL + expression: s_state + datatype: String + description: State where store is located + ai_context: + synonyms: + - state + - region + - name: s_number_employees + expression: + dialects: + - dialect: ANSI_SQL + expression: s_number_employees + datatype: Decimal + description: Number of employees at the store + ai_context: + synonyms: + - employee count + - staff size + primary_key: + - s_store_sk + metrics: + - name: total_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales revenue across all transactions + ai_context: + synonyms: + - total revenue + - gross sales + - sales amount + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales"}' + - name: total_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_net_profit) + description: Total net profit from store sales + ai_context: + synonyms: + - net profit + - total earnings + - profit + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales"}' + - name: customer_lifetime_value + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) + description: Average lifetime sales value per customer + ai_context: + synonyms: + - CLV + - LTV + - customer value + - lifetime revenue + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "measure": {"name": "customer_lifetime_value", + "sql": "SUM({CUBE}.ss_ext_sales_price) / COUNT(DISTINCT {customer.c_customer_sk})", + "type": "number"}}' + - name: sales_by_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales by brand (requires grouping by item.i_brand) + ai_context: + synonyms: + - brand sales + - brand performance + - brand revenue + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales"}' + - name: store_productivity + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), + 0) + description: Sales per employee across stores + ai_context: + synonyms: + - sales per employee + - employee productivity + - revenue per employee + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "cube": "store_sales", "measure": {"name": "store_productivity", + "sql": "SUM({CUBE}.ss_ext_sales_price) / NULLIF(SUM({store.s_number_employees}), + 0)", "type": "number"}}' + custom_extensions: + - vendor_name: CUBE + data: '{"_v": 1, "views": {"tpcds_retail_model": {"name": "tpcds_retail_model", + "cubes": [{"join_path": "store_sales", "includes": "*"}, {"join_path": "store_sales.date_dim", + "includes": "*"}, {"join_path": "store_sales.customer", "includes": "*"}, {"join_path": + "store_sales.item", "includes": "*"}, {"join_path": "store_sales.store", "includes": + "*"}]}}, "mapped_view": "tpcds_retail_model"}' + - vendor_name: SALESFORCE + data: | + { + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "0.1.1" + } + } + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' diff --git a/converters/cube/tests/test_cli.py b/converters/cube/tests/test_cli.py new file mode 100644 index 00000000..2314280d --- /dev/null +++ b/converters/cube/tests/test_cli.py @@ -0,0 +1,312 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Command-line behavior: what the user actually types, and what they get back. + +Covers the input shapes people reach for first -- a whole model directory, a single +file, and (a common mistake) just the view -- plus the exit codes and where output +goes, since those are the converter's contract with a shell script. +""" + +import pytest +from _util import REPO_ROOT, load_fixture_dir, parse + +from ossie_cube.cli import main + +_ORDERS = ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: count\n" + " type: count\n" +) +_VIEW = ( + "views:\n" + " - name: sales\n" + " description: Sales overview\n" + " cubes:\n" + " - join_path: orders\n" + " includes: '*'\n" +) + + +def _write(root, **files): + for rel, text in files.items(): + path = root / rel.replace("|", "/") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return root + + +# --- input shapes --------------------------------------------------------------- + +def test_a_model_directory_converts(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, "views|sales.yml": _VIEW}) + assert main(["import", "-i", str(model)]) == 0 + doc = parse(capsys.readouterr().out) + assert doc["semantic_model"][0]["name"] == "sales" + + +def test_a_single_file_converts(tmp_path, capsys): + """Pointing at one `.yml` is a natural thing to try and there is nothing + ambiguous about it, so it is accepted rather than refused on a technicality.""" + path = tmp_path / "orders.yml" + path.write_text(_ORDERS) + assert main(["import", "-i", str(path)]) == 0 + doc = parse(capsys.readouterr().out) + assert [d["name"] for d in doc["semantic_model"][0]["datasets"]] == ["orders"] + + +def test_several_paths_merge_into_one_model(tmp_path, capsys): + """Cube has a single model root, but converting part of a model -- or files from + different trees -- should not require assembling a directory first.""" + a = tmp_path / "cubes" / "orders.yml" + b = tmp_path / "views" / "sales.yml" + for path, text in ((a, _ORDERS), (b, _VIEW)): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + assert main(["import", "-i", str(a), str(b)]) == 0 + model = parse(capsys.readouterr().out)["semantic_model"][0] + assert model["name"] == "sales" # the view was picked up + assert [d["name"] for d in model["datasets"]] == ["orders"] + + +def test_several_paths_are_keyed_relative_to_their_common_parent(tmp_path, capsys): + """The keys decide where export writes the files back, so two inputs from + different subtrees have to stay distinguishable.""" + a = tmp_path / "cubes" / "orders.yml" + b = tmp_path / "views" / "sales.yml" + for path, text in ((a, _ORDERS), (b, _VIEW)): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(a), str(b), "-o", str(out)]) == 0 + back = tmp_path / "back" + assert main(["export", "-i", str(out), "-o", str(back)]) == 0 + capsys.readouterr() + assert (back / "cubes" / "orders.yml").is_file() + assert (back / "views" / "sales.yml").is_file() + + +def test_mixing_a_directory_and_a_file_works(tmp_path, capsys): + model = _write(tmp_path / "model", **{"cubes|orders.yml": _ORDERS}) + extra = tmp_path / "extra.yml" + extra.write_text(_VIEW) + assert main(["import", "-i", str(model), str(extra)]) == 0 + assert parse(capsys.readouterr().out)["semantic_model"][0]["name"] == "sales" + + +def test_overlapping_inputs_are_reported(tmp_path, capsys): + """Passing a directory and a file inside it is an easy mistake (an overlapping + glob), and it would otherwise read the same file twice.""" + model = _write(tmp_path / "model", **{"cubes|orders.yml": _ORDERS}) + assert main(["import", "-i", str(model), + str(model / "cubes" / "orders.yml")]) == 1 + err = capsys.readouterr().err + assert "both resolve to 'cubes/orders.yml'" in err + + +def test_the_same_cube_in_two_inputs_is_reported(tmp_path, capsys): + a = tmp_path / "one" / "orders.yml" + b = tmp_path / "two" / "orders.yml" + for path in (a, b): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(_ORDERS) + # Distinct keys ('one/orders.yml', 'two/orders.yml'), but the same cube name. + assert main(["import", "-i", str(a), str(b)]) == 1 + assert "defined twice" in capsys.readouterr().err + + +def test_a_single_path_is_keyed_exactly_as_before(tmp_path, capsys): + """The multi-path anchor must not change the one-directory case, since the keys + are what export writes back.""" + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, "views|sales.yml": _VIEW}) + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(model), "-o", str(out)]) == 0 + back = tmp_path / "back" + assert main(["export", "-i", str(out), "-o", str(back)]) == 0 + capsys.readouterr() + assert (back / "cubes" / "orders.yml").is_file() + assert (back / "views" / "sales.yml").is_file() + + +def test_only_a_view_file_is_refused_with_an_actionable_message(tmp_path, capsys): + """The likeliest mistake for a view-first user: a Cube view looks like the whole + model, but it projects members from cubes and defines none, so the error names + the cubes whose files are missing rather than claiming nothing was recognized.""" + path = tmp_path / "sales.yml" + path.write_text(_VIEW) + assert main(["import", "-i", str(path)]) == 1 + err = capsys.readouterr().err + assert "found only view(s) 'sales' and no cubes" in err + assert "projects members from cubes" in err + assert "'orders'" in err # named from the view's join_path + + +def test_a_view_with_no_cube_references_still_explains_itself(tmp_path, capsys): + path = tmp_path / "bare.yml" + path.write_text("views:\n - name: sales\n description: Sales\n") + assert main(["import", "-i", str(path)]) == 1 + assert "Include the files defining the cubes it draws from" in \ + capsys.readouterr().err + + +def test_a_missing_path_is_reported_not_traced(tmp_path, capsys): + assert main(["import", "-i", str(tmp_path / "nope")]) == 1 + assert "is not a file or directory" in capsys.readouterr().err + + +def test_an_empty_directory_is_reported(tmp_path, capsys): + empty = tmp_path / "empty" + empty.mkdir() + assert main(["import", "-i", str(empty)]) == 1 + assert "holds no files" in capsys.readouterr().err + + +def test_node_modules_and_dotfiles_are_skipped(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, + "node_modules|junk.yml": "cubes:\n - name: junk\n sql_table: t\n", + ".hidden.yml": "cubes:\n - name: hidden\n sql_table: t\n", + }) + assert main(["import", "-i", str(model)]) == 0 + doc = parse(capsys.readouterr().out) + assert [d["name"] for d in doc["semantic_model"][0]["datasets"]] == ["orders"] + + +# --- output and exit codes ------------------------------------------------------ + +def test_output_goes_to_a_file_when_asked(tmp_path, capsys): + model = _write(tmp_path / "model", **{"cubes|orders.yml": _ORDERS}) + out = tmp_path / "model.yaml" + assert main(["import", "-i", str(model), "-o", str(out)]) == 0 + assert capsys.readouterr().out == "" + assert parse(out.read_text())["semantic_model"][0]["datasets"] + + +def test_issues_go_to_stderr_so_stdout_stays_pipeable(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|users.yml": ( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: home\n" + " type: geo\n" + " latitude:\n" + " sql: lat\n" + " longitude:\n" + " sql: lon\n" + )}) + assert main(["import", "-i", str(model)]) == 0 + captured = capsys.readouterr() + assert "GEO_DIMENSION_SPLIT" in captured.err + assert "conversion issue" in captured.err + parse(captured.out) # stdout is still clean YAML + + +def test_fanout_warns_by_default_and_the_flag_exits_nonzero(tmp_path, capsys): + model = _write(tmp_path / "model", **{"cubes|m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: ltv\n" + " sql: \"{CUBE}.ltv\"\n" + " type: sum\n" + )}) + assert main(["import", "-i", str(model)]) == 0 + captured = capsys.readouterr() + assert "FANOUT_UNSAFE_METRIC" in captured.err + assert parse(captured.out)["semantic_model"][0]["metrics"] + + assert main(["import", "-i", str(model), "--strict-fanout"]) == 1 + assert "FANOUT_UNSAFE_METRIC" in capsys.readouterr().err + + +def test_view_and_name_flags_take_effect(tmp_path, capsys): + model = _write(tmp_path / "model", **{ + "cubes|orders.yml": _ORDERS, + "views|a.yml": "views:\n - name: a\n description: A\n", + "views|b.yml": "views:\n - name: b\n description: B\n", + }) + assert main(["import", "-i", str(model), "--view", "b"]) == 0 + assert parse(capsys.readouterr().out)["semantic_model"][0]["description"] == "B" + + assert main(["import", "-i", str(model), "--view", "b", + "--name", "custom"]) == 0 + assert parse(capsys.readouterr().out)["semantic_model"][0]["name"] == "custom" + + assert main(["import", "-i", str(model), "--view", "ghost"]) == 1 + assert "not found" in capsys.readouterr().err + + +# --- export --------------------------------------------------------------------- + +def test_export_writes_the_model_directory(tmp_path, capsys): + out = tmp_path / "out" + assert main(["export", "-i", + str(REPO_ROOT / "examples" / "tpcds_semantic_model.yaml"), + "-o", str(out)]) == 0 + assert (out / "model" / "cubes" / "store_sales.yml").is_file() + assert (out / "model" / "views" / "tpcds_retail_model.yml").is_file() + assert "Wrote 6 file(s)" in capsys.readouterr().err + + +def test_export_of_a_missing_input_is_reported(tmp_path, capsys): + assert main(["export", "-i", str(tmp_path / "nope.yaml"), + "-o", str(tmp_path / "out")]) == 1 + assert "Error:" in capsys.readouterr().err + + +def test_a_cli_round_trip_reproduces_the_fixture(tmp_path, capsys): + fixture = load_fixture_dir("tpcds_cube") + src = _write(tmp_path / "src", **{k.replace("/", "|"): v + for k, v in fixture.items()}) + ossie = tmp_path / "model.yaml" + back = tmp_path / "back" + assert main(["import", "-i", str(src), "-o", str(ossie)]) == 0 + assert main(["export", "-i", str(ossie), "-o", str(back)]) == 0 + capsys.readouterr() + for rel in fixture: + assert (back / rel.replace("/", "/")).is_file(), rel + assert parse((back / rel).read_text()) == parse(fixture[rel]) + + +def test_no_subcommand_is_a_usage_error(): + with pytest.raises(SystemExit) as excinfo: + main([]) + assert excinfo.value.code == 2 diff --git a/converters/cube/tests/test_cube_to_osi.py b/converters/cube/tests/test_cube_to_osi.py new file mode 100644 index 00000000..15f6f26d --- /dev/null +++ b/converters/cube/tests/test_cube_to_osi.py @@ -0,0 +1,547 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Cube data model -> Apache Ossie semantic model.""" + +import pytest +from _util import by_name, expr_of, load_fixture_dir, model_of, stash_of + +from ossie_cube import ConversionError, IssueType, convert_cube_to_ossie +from ossie_cube._common import OSSIE_VERSION, cube_sql_to_ossie + + +@pytest.fixture +def fixture_a(): + return load_fixture_dir("fixtureA_cube") + + +@pytest.fixture +def model_a(fixture_a): + out, issues = convert_cube_to_ossie(fixture_a) + return model_of(out), issues + + +# --- model identity ------------------------------------------------------------- + +def test_version_and_single_model(fixture_a): + out, _ = convert_cube_to_ossie(fixture_a) + from _util import parse + + doc = parse(out) + assert doc["version"] == OSSIE_VERSION + assert len(doc["semantic_model"]) == 1 + + +def test_mapped_view_supplies_model_identity(model_a): + model, _ = model_a + assert model["name"] == "sales" + assert model["description"] == "Sales overview" + assert "revenue analysis" in model["ai_context"]["instructions"] + + +def test_model_name_override(fixture_a): + out, _ = convert_cube_to_ossie(fixture_a, model_name="custom") + assert model_of(out)["name"] == "custom" + + +def test_unknown_view_is_rejected(fixture_a): + with pytest.raises(ConversionError, match="not found"): + convert_cube_to_ossie(fixture_a, view="nope") + + +def test_view_curation_rides_in_the_stash(model_a): + model, _ = model_a + stash = stash_of(model) + assert stash["mapped_view"] == "sales" + # The natively mapped description/ai_context are stripped; the curation stays. + view = stash["views"]["sales"] + assert "description" not in view + assert "meta" not in view + assert view["cubes"][0]["join_path"] == "orders" + + +# --- datasets ------------------------------------------------------------------- + +def test_cubes_become_datasets(model_a): + model, _ = model_a + datasets = by_name(model["datasets"]) + assert set(datasets) == {"orders", "users"} + assert datasets["orders"]["source"] == "public.orders" + assert datasets["orders"]["description"] == "Customer orders" + # A `sql`-defined cube keeps its query as the source. + assert datasets["users"]["source"].startswith("SELECT * FROM public.users") + + +def test_primary_key_from_dimension_flag(model_a): + model, _ = model_a + datasets = by_name(model["datasets"]) + assert datasets["orders"]["primary_key"] == ["id"] + assert datasets["users"]["primary_key"] == ["id"] + + +def test_segments_have_no_ossie_form_and_are_stashed(model_a): + model, _ = model_a + users = by_name(model["datasets"])["users"] + segments = stash_of(users)["cube_extras"]["segments"] + assert segments[0]["name"] == "active" + + +# --- fields --------------------------------------------------------------------- + +def test_dimension_types_map_to_datatypes(model_a): + model, _ = model_a + fields = by_name(by_name(model["datasets"])["orders"]["fields"]) + assert fields["status"]["datatype"] == "String" + assert fields["is_large"]["datatype"] == "Boolean" + assert fields["created_at"]["datatype"] == "DateTime" + assert fields["created_at"]["dimension"]["is_time"] is True + + +def test_number_dimension_maps_to_a_native_datatype(model_a): + """Cube collapses Integer/Decimal/Float into `number`, so no mapping back is + exact. `Decimal` is asserted anyway, because a downstream converter can act on it + -- where omitting it and stashing Cube's `type` in a custom_extension gave every + other spoke a warning and nothing else.""" + model, _ = model_a + fields = by_name(by_name(model["datasets"])["orders"]["fields"]) + assert fields["id"]["datatype"] == "Decimal" + assert stash_of(fields["id"]) == {} + + +def test_dimension_title_becomes_label_and_ai_context_maps(model_a): + model, _ = model_a + status = by_name(by_name(model["datasets"])["orders"]["fields"])["status"] + assert status["label"] == "Order Status" + assert status["description"] == "Current order status" + assert status["ai_context"]["instructions"].startswith("Values are pending") + + +def test_cube_reference_is_stripped_in_a_field_expression(model_a): + """Field expressions are dataset-scoped, so `{CUBE}.amount` reads as `amount`.""" + model, _ = model_a + is_large = by_name(by_name(model["datasets"])["orders"]["fields"])["is_large"] + assert expr_of(is_large) == "amount > 500" + + +def test_geo_dimension_splits_into_two_fields(model_a): + model, issues = model_a + fields = by_name(by_name(model["datasets"])["users"]["fields"]) + assert "location" not in fields + assert expr_of(fields["location_latitude"]) == "lat" + assert expr_of(fields["location_longitude"]) == "lon" + assert fields["location_latitude"]["datatype"] == "Float" + assert stash_of(fields["location_latitude"])["geo"]["of"] == "location" + assert issues.of_type(IssueType.GEO_DIMENSION_SPLIT) + + +# --- relationships -------------------------------------------------------------- + +def test_many_to_one_join_becomes_a_relationship(model_a): + model, _ = model_a + rel = by_name(model["relationships"])["orders_to_users"] + assert rel["from"] == "orders" + assert rel["to"] == "users" + assert rel["from_columns"] == ["user_id"] + assert rel["to_columns"] == ["id"] + # Nothing is stashed: a many_to_one join declared on the many side is exactly + # what `from`(many) -> `to`(one) already says, so recording it again would only + # add a custom_extension for every other converter to warn about and discard. + assert stash_of(rel) == {} + + +def test_one_to_many_join_is_flipped_to_many_side_first(): + """Ossie's `from` is always the many side, so a join declared as one_to_many on + the one side is flipped -- and the declared orientation stashed.""" + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: orders\n" + " sql: \"{CUBE}.id = {orders}.user_id\"\n" + " relationship: one_to_many\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + ) + } + out, _ = convert_cube_to_ossie(files) + rel = model_of(out)["relationships"][0] + assert rel["from"] == "orders" + assert rel["to"] == "users" + assert rel["from_columns"] == ["user_id"] + assert rel["to_columns"] == ["id"] + assert stash_of(rel)["relationship"] == "one_to_many" + assert stash_of(rel)["declared_on"] == "users" + + +def test_non_equi_join_is_preserved_not_guessed_at(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: rates\n" + " sql: \"{CUBE}.day >= {rates}.valid_from\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: day\n" + " sql: day\n" + " type: time\n" + " - name: rates\n" + " sql_table: public.rates\n" + " dimensions:\n" + " - name: valid_from\n" + " sql: valid_from\n" + " type: time\n" + ) + } + out, issues = convert_cube_to_ossie(files) + model = model_of(out) + assert "relationships" not in model + orders = by_name(model["datasets"])["orders"] + assert stash_of(orders)["extra_joins"][0]["join"]["name"] == "rates" + assert issues.of_type(IssueType.PARKED_IN_META) + + +def test_join_to_unknown_cube_is_rejected(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: ghosts\n" + " sql: \"{CUBE}.id = {ghosts}.id\"\n" + " relationship: many_to_one\n" + ) + } + with pytest.raises(ConversionError, match="not a cube in this model"): + convert_cube_to_ossie(files) + + +# --- metrics -------------------------------------------------------------------- + +def test_measures_are_hoisted_and_disambiguated(model_a): + """`count` exists on both cubes, so both are qualified and the original names + stashed; a globally unique measure keeps its own name.""" + model, _ = model_a + metrics = by_name(model["metrics"]) + assert "orders__count" in metrics + assert "users__count" in metrics + assert stash_of(metrics["orders__count"])["name"] == "count" + assert stash_of(metrics["orders__count"])["cube"] == "orders" + assert "total_amount" in metrics + + +def test_bare_count_maps_through_the_primary_key(model_a): + """Cube renders a bare `count` as count(pk), and count(distinct pk) when the + cube is fanned out. COUNT(DISTINCT pk) equals both, so it is the one static + form that stays correct in every join context.""" + model, _ = model_a + metrics = by_name(model["metrics"]) + assert expr_of(metrics["orders__count"]) == "COUNT(DISTINCT orders.id)" + assert expr_of(metrics["users__count"]) == "COUNT(DISTINCT users.id)" + assert metrics["orders__count"]["datatype"] == "Integer" + + +def test_bare_count_without_a_primary_key_is_rejected(): + """The primary key is load-bearing for a correct `count`, so its absence is an + error rather than a silently-different number.""" + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: status\n" + " sql: status\n" + " type: string\n" + " measures:\n" + " - name: count\n" + " type: count\n" + ) + } + with pytest.raises(ConversionError, match="primary key"): + convert_cube_to_ossie(files) + + +def test_aggregate_measures_become_qualified_expressions(model_a): + model, _ = model_a + metrics = by_name(model["metrics"]) + assert expr_of(metrics["total_amount"]) == "SUM(orders.amount)" + assert expr_of(metrics["cities"]) == "COUNT(DISTINCT users.city)" + assert metrics["total_amount"]["description"] == "Total order amount" + assert metrics["total_amount"]["ai_context"]["instructions"].startswith("Use this") + + +def test_measure_filters_fold_into_a_case_expression(model_a): + """Cube's own applyMeasureFilters wraps the operand as + CASE WHEN THEN END inside the aggregate.""" + model, _ = model_a + metric = by_name(model["metrics"])["completed_amount"] + assert expr_of(metric) == ( + "SUM(CASE WHEN (orders.status = 'completed') THEN orders.amount END)") + + +def test_filtered_and_calculated_measures_keep_the_original(model_a): + """Export cannot recover `filters` from the folded CASE, nor un-inline a + calculated measure's references, so both keep the original measure verbatim -- + which is what makes Cube -> Ossie -> Cube lossless.""" + model, _ = model_a + metrics = by_name(model["metrics"]) + assert stash_of(metrics["completed_amount"])["measure"]["filters"] + assert stash_of(metrics["avg_order_value"])["measure"]["sql"] == ( + "{total_amount} / {count}") + # A plain aggregate needs no such copy. + assert "measure" not in stash_of(metrics["orders__count"]) + + +def test_calculated_measure_inlines_its_measure_references(model_a): + """Cube resolves `{total_amount} / {count}` to the referenced measures' own + aggregate SQL; Ossie has no metric-to-metric reference, so it is inlined.""" + model, _ = model_a + metric = by_name(model["metrics"])["avg_order_value"] + # No redundant parentheses: a lone aggregate is already a single term, so an + # inlined reference reads exactly as the expression it stands for. + assert expr_of(metric) == "SUM(orders.amount) / COUNT(DISTINCT orders.id)" + + +def test_measure_reference_cycle_is_rejected(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " measures:\n" + " - name: a\n" + " sql: \"{b} + 1\"\n" + " type: number\n" + " - name: b\n" + " sql: \"{a} + 1\"\n" + " type: number\n" + ) + } + with pytest.raises(ConversionError, match="cycle"): + convert_cube_to_ossie(files) + + +def test_multi_stage_measure_is_dropped_with_an_issue(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " measures:\n" + " - name: rolling\n" + " sql: amount\n" + " type: sum\n" + " multi_stage: true\n" + ) + } + out, issues = convert_cube_to_ossie(files) + assert "metrics" not in model_of(out) + assert issues.of_type(IssueType.MULTI_STAGE_MEASURE_PARKED) + + +# --- fan-out -------------------------------------------------------------------- + +_FANOUT_MODEL = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: lifetime_value\n" + " sql: \"{CUBE}.ltv\"\n" + " type: sum\n" + ) +} + + +def test_fanout_unsafe_metric_is_recorded_by_default(): + """`users` is the one side of a many-to-one join, so summing over it after the + join over-counts. Cube deduplicates on the primary key at query time and a static + Ossie expression cannot -- so the metric converts and the risk is reported, named + down to the relationship responsible. Refusing the whole model over one metric + would leave the spoke on the other side with nothing to convert.""" + out, issues = convert_cube_to_ossie(_FANOUT_MODEL) + metric = by_name(model_of(out)["metrics"])["lifetime_value"] + assert expr_of(metric) == "SUM(users.ltv)" + recorded = issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + assert len(recorded) == 1 + assert recorded[0].element_name == "users.lifetime_value" + assert "over-count" in recorded[0].detail + + +def test_fanout_unsafe_metric_is_refused_under_strict_fanout(): + """Mirrors Cube's own refusal, for a caller who would rather have nothing than a + number that disagrees with Cube.""" + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(_FANOUT_MODEL, strict_fanout=True) + + +def test_idempotent_aggregates_are_never_flagged(fixture_a): + """count / count_distinct / min / max are unaffected by duplicate rows, so a + fanned-out dataset carrying only those raises nothing even under strict mode.""" + _, issues = convert_cube_to_ossie(fixture_a, strict_fanout=True) + assert not issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + + +# --- rejections and preservation ------------------------------------------------ + +def test_jinja_templated_file_is_preserved_not_parsed(): + files = { + "model/cubes/dyn.yml": "cubes:\n - name: o{{ suffix }}\n sql_table: t\n", + "model/cubes/ok.yml": ( + "cubes:\n - name: orders\n sql_table: public.orders\n"), + } + out, issues = convert_cube_to_ossie(files) + model = model_of(out) + assert by_name(model["datasets"]).keys() == {"orders"} + assert "model/cubes/dyn.yml" in stash_of(model)["extra_files"] + assert issues.of_type(IssueType.TEMPLATED_FILE_SKIPPED) + + +def test_join_into_a_skipped_file_explains_itself(): + """A file the converter had to skip whole (Jinja, `.js`) can leave a join + pointing at a cube that is no longer there. The error says so, rather than just + reporting a missing cube.""" + files = { + "model/cubes/dyn.yml": ( + "cubes:\n - name: users\n sql_table: t{{ suffix }}\n"), + "model/cubes/orders.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + ), + } + with pytest.raises(ConversionError, match="model/cubes/dyn.yml"): + convert_cube_to_ossie(files) + + +def test_javascript_model_is_preserved_not_parsed(): + files = { + "model/cubes/orders.js": "cube(`orders`, { sql_table: `public.orders` });", + "model/cubes/ok.yml": ( + "cubes:\n - name: orders_yaml\n sql_table: public.orders\n"), + } + out, issues = convert_cube_to_ossie(files) + assert "model/cubes/orders.js" in stash_of(model_of(out))["extra_files"] + assert issues.of_type(IssueType.TEMPLATED_FILE_SKIPPED) + + +def test_extends_is_refused_rather_than_half_resolved(): + files = { + "model/cubes/m.yml": ( + "cubes:\n" + " - name: base\n" + " sql_table: public.orders\n" + " - name: derived\n" + " extends: base\n" + ) + } + with pytest.raises(ConversionError, match="extends"): + convert_cube_to_ossie(files) + + +def test_cube_without_a_source_is_rejected(): + files = {"model/cubes/m.yml": "cubes:\n - name: orders\n description: x\n"} + with pytest.raises(ConversionError, match="neither 'sql' nor 'sql_table'"): + convert_cube_to_ossie(files) + + +def test_cube_with_both_sources_is_rejected(): + files = { + "model/cubes/m.yml": ( + "cubes:\n - name: orders\n sql: SELECT 1\n sql_table: t\n") + } + with pytest.raises(ConversionError, match="exactly one"): + convert_cube_to_ossie(files) + + +def test_duplicate_cube_name_is_rejected(): + files = { + "model/cubes/a.yml": "cubes:\n - name: orders\n sql_table: a\n", + "model/cubes/b.yml": "cubes:\n - name: orders\n sql_table: b\n", + } + with pytest.raises(ConversionError, match="defined twice"): + convert_cube_to_ossie(files) + + +def test_model_with_no_cubes_is_rejected(): + with pytest.raises(ConversionError, match="no convertible cubes"): + convert_cube_to_ossie({"README.md": "not a model"}) + + +# --- reference translation ------------------------------------------------------ + +@pytest.mark.parametrize("sql,expected", [ + ("{CUBE}.status", "status"), + ("{TABLE}.status", "status"), + ("{CUBE.status}", "status"), + ("{status}", "status"), + ("{orders.status}", "status"), + ("{users.city}", "users.city"), + ("${CUBE}.status", "status"), + ("LOWER({CUBE}.email)", "LOWER(email)"), + (r"'\{literal\}'", "'{literal}'"), +]) +def test_reference_translation_in_a_field_context(sql, expected): + """A field expression is dataset-scoped, so own-cube references reduce to a + bare name. `\\{` stays a literal brace.""" + assert cube_sql_to_ossie(sql, "orders")[0] == expected + + +@pytest.mark.parametrize("sql,expected", [ + ("{CUBE}.amount", "orders.amount"), + ("{CUBE.amount}", "orders.amount"), + ("{amount}", "orders.amount"), + ("{users.city}", "users.city"), +]) +def test_reference_translation_in_a_metric_context(sql, expected): + """A metric expression is model-level, so own-cube references are qualified.""" + assert cube_sql_to_ossie(sql, "orders", self_prefix="orders")[0] == expected diff --git a/converters/cube/tests/test_edge_cases.py b/converters/cube/tests/test_edge_cases.py new file mode 100644 index 00000000..ebcb59fa --- /dev/null +++ b/converters/cube/tests/test_edge_cases.py @@ -0,0 +1,1513 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Coverage-driven tests for paths the fixtures and property tests do not reach. + +The fixture and Hypothesis suites cover the common shapes well, but they generate +inside the round-trippable subset and so never exercise several load-bearing +branches: composite primary keys (central to the fan-out mapping), `count` over an +expression, the export side of the one_to_many flip, off-layout file grouping, the +JavaScript-style mapping form of a collection, and Jinja detection. Each of those +is pinned here, along with the error paths for malformed input. +""" + +import pytest +from _util import by_name, expr_of, model_of, parse, parse_files, stash_of + +from ossie_cube import ( + ConversionError, + IssueType, + convert_cube_to_ossie, + convert_ossie_to_cube, +) +from ossie_cube._common import dump_yaml + + +def _files(**named): + return {f"model/cubes/{n}.yml": t for n, t in named.items()} + + +def _roundtrip(files): + ossie, issues = convert_cube_to_ossie(files) + back, _ = convert_ossie_to_cube(ossie) + return ossie, back, issues + + +# --- composite primary keys ----------------------------------------------------- + +_COMPOSITE = _files(order_lines=( + "cubes:\n" + " - name: order_lines\n" + " sql_table: public.order_lines\n" + " dimensions:\n" + " - name: order_id\n" + " sql: order_id\n" + " type: number\n" + " primary_key: true\n" + " - name: line_no\n" + " sql: line_no\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: count\n" + " type: count\n" +)) + + +def test_composite_primary_key_becomes_a_concatenated_distinct_count(): + """Cube concatenates a composite key with CAST + CONCAT in `primaryKeyCount`; + the Ossie expression mirrors that so the count stays correct under fan-out and + stays portable (both functions are REQUIRED in the expression language).""" + ossie, _ = convert_cube_to_ossie(_COMPOSITE) + model = model_of(ossie) + assert by_name(model["datasets"])["order_lines"]["primary_key"] == [ + "order_id", "line_no"] + assert expr_of(model["metrics"][0]) == ( + "COUNT(DISTINCT CONCAT(CAST(order_lines.order_id AS VARCHAR), " + "CAST(order_lines.line_no AS VARCHAR)))") + + +def test_composite_key_count_converts_back_to_a_bare_count(): + _, back, _ = _roundtrip(_COMPOSITE) + cube = parse(back["model/cubes/order_lines.yml"])["cubes"][0] + assert cube["measures"] == [{"name": "count", "type": "count"}] + assert [d["name"] for d in cube["dimensions"] if d.get("primary_key")] == [ + "order_id", "line_no"] + + +def test_composite_key_roundtrips(): + _, back, _ = _roundtrip(_COMPOSITE) + assert parse_files(back) == parse_files(_COMPOSITE) + + +# --- count over an expression --------------------------------------------------- + +_COUNT_SQL = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: statuses\n" + " sql: \"{CUBE}.status\"\n" + " type: count\n" +)) + + +def test_count_over_an_expression_keeps_its_operand(): + """`type: count` with `sql` is COUNT(x), not COUNT(*) -- Cube only routes + through the primary key when no sql is given.""" + ossie, _ = convert_cube_to_ossie(_COUNT_SQL) + assert expr_of(model_of(ossie)["metrics"][0]) == "COUNT(orders.status)" + + +def test_count_over_an_expression_roundtrips(): + _, back, _ = _roundtrip(_COUNT_SQL) + assert parse_files(back) == parse_files(_COUNT_SQL) + + +def test_count_over_an_expression_is_fanout_unsafe(): + """Unlike a bare count, COUNT(x) over a fanned-out dataset over-counts.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: emails\n" + " sql: \"{CUBE}.email\"\n" + " type: count\n" + )) + _, issues = convert_cube_to_ossie(files) + assert issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files, strict_fanout=True) + + +_MULTI_STAGE = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: revenue\n" + " sql: amount\n" + " type: sum\n" + " - name: rolling\n" + " sql: amount\n" + " type: sum\n" + " multi_stage: true\n" + " rolling_window:\n" + " trailing: 3 month\n" + " - name: cnt\n" + " type: count\n" +)) + + +def test_a_multi_stage_measure_is_not_an_ossie_metric(): + """It renders as a window function over another grain, which an Ossie expression + has no form for -- so it gets no `metrics` entry, and that is reported.""" + ossie, issues = convert_cube_to_ossie(_MULTI_STAGE) + assert [m["name"] for m in model_of(ossie)["metrics"]] == ["revenue", "cnt"] + parked = issues.of_type(IssueType.MULTI_STAGE_MEASURE_PARKED) + assert [i.element_name for i in parked] == ["orders.rolling"] + + +def test_a_multi_stage_measure_survives_the_round_trip_in_place(): + """It used to be lost outright: no metric, and `measures` is a natively-mapped key + so `cube_extras` did not carry it either -- while the issue claimed it had been + preserved. Now it rides on the dataset's stash with its position, like an + unconvertible join, and comes back interleaved with the rebuilt measures.""" + ossie, _ = convert_cube_to_ossie(_MULTI_STAGE) + stashed = stash_of(by_name(model_of(ossie)["datasets"])["orders"]) + assert stashed["extra_measures"] == [ + {"index": 1, "measure": { + "name": "rolling", "sql": "amount", "type": "sum", + "multi_stage": True, "rolling_window": {"trailing": "3 month"}}}] + + back, _ = convert_ossie_to_cube(ossie) + assert parse_files(back) == parse_files(_MULTI_STAGE) + # Order matters: it goes back between the two ordinary measures. + names = [m["name"] for m in parse( + back["model/cubes/orders.yml"])["cubes"][0]["measures"]] + assert names == ["revenue", "rolling", "cnt"] + + +def test_count_star_is_not_emitted_as_a_bare_cube_count(): + """A bare Cube `type: count` is this converter's form for + `COUNT(DISTINCT )`. Emitting one for `COUNT(*)` round-tripped back as a + different expression, and on a dataset with no primary key produced a measure + the importer refuses -- export generating what its own import rejects.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " metrics:\n" + " - name: n\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: COUNT(*)\n" + ) + files, _ = convert_ossie_to_cube(ossie) + measure = parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"][0] + assert measure == {"name": "n", "sql": "COUNT(*)", "type": "number"} + + # And it survives the trip back, without a primary key anywhere in sight. + ossie2, _ = convert_cube_to_ossie(files) + assert expr_of(model_of(ossie2)["metrics"][0]) == "COUNT(*)" + + +def test_field_and_metric_foreign_extensions_survive_the_round_trip(): + """Foreign-vendor extensions are parked under `meta.ossie` at every level, but + only datasets were reading them back -- so field- and metric-level ones were + parked and then silently dropped on re-import.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " fields:\n" + " - name: status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status\n" + " datatype: String\n" + " custom_extensions:\n" + " - vendor_name: SNOWFLAKE\n" + " data: '{\"collation\": \"en\"}'\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" + " metrics:\n" + " - name: total\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: SUM(orders.amount)\n" + " custom_extensions:\n" + " - vendor_name: DBT\n" + " data: '{\"model\": \"fct_orders\"}'\n" + ) + files, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files) + model = model_of(ossie2) + + field = by_name(by_name(model["datasets"])["orders"]["fields"])["status"] + exts = {e["vendor_name"]: e["data"] for e in field["custom_extensions"]} + assert exts["SNOWFLAKE"] == '{"collation": "en"}' + # A plain scalar field needs no CUBE stash at all any more, so the foreign + # extension is the only entry -- which is the point of the reduction. + assert list(exts) == ["SNOWFLAKE"] + + metric = by_name(model["metrics"])["total"] + mexts = {e["vendor_name"]: e["data"] for e in metric["custom_extensions"]} + assert mexts["DBT"] == '{"model": "fct_orders"}' + assert metric["custom_extensions"][0]["vendor_name"] == "CUBE" + + +@pytest.mark.parametrize("sql_table,parts,warns", [ + ("orders", 1, True), + ("public.orders", 2, True), + ("tpcds.public.orders", 3, False), + ('"My.Catalog".public.orders', 3, False), # dots inside quotes are not parts + ("a.b.c.d", 4, False), +]) +def test_a_source_that_other_converters_reject_is_reported(sql_table, parts, warns): + """Cube is happy with a one- or two-part `sql_table`, but the Databricks, + Snowflake and NVIDIA GSF converters all reject a source shorter than + `catalog.schema.table` -- so a model that converts cleanly here still cannot + travel. Reported at the point the Ossie document is produced, rather than being + discovered three hops later.""" + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + # Single-quoted so a value containing double quotes stays one YAML scalar. + f" sql_table: '{sql_table}'\n")) + _, issues = convert_cube_to_ossie(files) + reported = issues.of_type(IssueType.SOURCE_NOT_FULLY_QUALIFIED) + assert bool(reported) is warns + if warns: + assert f"{parts} part(s)" in reported[0].detail + + +def test_a_sql_defined_cube_is_not_reported_as_unqualified(): + """A `sql:` cube is a query, not a table path, and every converter accepts one.""" + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql: SELECT * FROM public.orders\n")) + _, issues = convert_cube_to_ossie(files) + assert not issues.of_type(IssueType.SOURCE_NOT_FULLY_QUALIFIED) + + +# --- join orientation, both ways ------------------------------------------------ + +_ONE_TO_MANY = _files(m=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: orders\n" + " sql: \"{CUBE}.id = {orders}.user_id\"\n" + " relationship: one_to_many\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" +)) + + +def test_one_to_many_is_flipped_back_onto_its_original_cube(): + """Ossie's `from` is always the many side, so import flips a one_to_many. Export + has to flip it back -- onto `users`, not `orders`.""" + _, back, _ = _roundtrip(_ONE_TO_MANY) + cubes = by_name(parse(back["model/cubes/m.yml"])["cubes"]) + assert cubes["users"]["joins"] == [{ + "name": "orders", "sql": "{CUBE}.id = {orders}.user_id", + "relationship": "one_to_many"}] + assert "joins" not in cubes["orders"] + + +@pytest.mark.parametrize("declared", ["one_to_one", "hasOne", "has_one"]) +def test_a_one_to_one_join_does_not_make_its_target_fanned_out(declared): + """A one-to-one join multiplies neither side, so a `sum` across it is safe. It was + being treated like any other relationship, whose `to` side *is* fanned out, and a + valid measure was refused under strict mode.""" + files = _files(m=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: profiles\n" + " sql: \"{CUBE}.id = {profiles}.user_id\"\n" + f" relationship: {declared}\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " - name: profiles\n" + " sql_table: public.profiles\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: score_total\n" + " sql: \"{CUBE}.score\"\n" + " type: sum\n" + )) + # Strict mode is the default; this must simply convert. + ossie, issues = convert_cube_to_ossie(files) + assert not issues.of_type(IssueType.FANOUT_UNSAFE_METRIC) + assert expr_of(by_name(model_of(ossie)["metrics"])["score_total"]) == ( + "SUM(profiles.score)") + + +def test_a_many_to_one_join_still_makes_its_target_fanned_out(): + """The counterpart: excluding one-to-one must not weaken the ordinary case.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " dimensions:\n" + " - name: user_id\n" + " sql: user_id\n" + " type: number\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: ltv\n" + " sql: \"{CUBE}.ltv\"\n" + " type: sum\n" + )) + with pytest.raises(ConversionError, match="FANOUT_UNSAFE_METRIC"): + convert_cube_to_ossie(files, strict_fanout=True) + + +def test_one_to_one_keeps_its_declared_orientation(): + files = _files(m=_ONE_TO_MANY["model/cubes/m.yml"].replace( + "one_to_many", "one_to_one")) + ossie, issues = convert_cube_to_ossie(files) + rel = model_of(ossie)["relationships"][0] + assert (rel["from"], rel["to"]) == ("users", "orders") + assert any("one_to_one" in i.detail for i in issues.of_type( + IssueType.PARKED_IN_META)) + _, back, _ = _roundtrip(files) + assert parse_files(back) == parse_files(files) + + +@pytest.mark.parametrize("alias,emitted", [ + ("belongsTo", "belongs_to"), + ("belongs_to", "belongs_to"), + ("hasMany", "has_many"), + ("hasOne", "has_one"), +]) +def test_legacy_relationship_spellings_are_accepted_and_kept_semantically( + alias, emitted): + """Cube still accepts belongsTo/hasMany/hasOne. The *kind* of relationship is + preserved rather than modernized to many_to_one, but the spelling is normalized + to snake_case along with every other key -- the documented normalization.""" + files = _files(m=_ONE_TO_MANY["model/cubes/m.yml"].replace( + "one_to_many", alias)) + _, back, _ = _roundtrip(files) + joins = [c.get("joins") for c in parse(back["model/cubes/m.yml"])["cubes"] + if c.get("joins")] + assert joins[0][0]["relationship"] == emitted + + +def test_two_joins_between_one_pair_get_distinct_relationship_names(): + """Ossie relationship names are unique per model, so a second join between the + same two cubes is suffixed rather than colliding.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.buyer_id = {users}.id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " joins:\n" + " - name: orders\n" + " sql: \"{CUBE}.id = {orders}.seller_id\"\n" + " relationship: one_to_many\n" + )) + ossie, _ = convert_cube_to_ossie(files) + names = [r["name"] for r in model_of(ossie)["relationships"]] + assert names == ["orders_to_users", "orders_to_users_2"] + + +def test_unconvertible_join_is_restored_at_its_original_position(): + """A non-equi join has no Ossie form, so it rides in the stash -- and export has + to put it back among the converted joins, in order.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: rates\n" + " sql: \"{CUBE}.day >= {rates}.valid_from\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {users}.id\"\n" + " relationship: many_to_one\n" + " - name: rates\n" + " sql_table: public.rates\n" + " - name: users\n" + " sql_table: public.users\n" + )) + _, back, issues = _roundtrip(files) + assert parse_files(back) == parse_files(files) + orders = by_name(parse(back["model/cubes/m.yml"])["cubes"])["orders"] + assert [j["name"] for j in orders["joins"]] == ["rates", "users"] + assert issues.of_type(IssueType.PARKED_IN_META) + + +def test_join_clause_written_target_side_first_still_decomposes(): + """Either side of the equality may name either cube.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{users.id} = {CUBE}.user_id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + )) + ossie, back, _ = _roundtrip(files) + rel = model_of(ossie)["relationships"][0] + assert (rel["from_columns"], rel["to_columns"]) == (["user_id"], ["id"]) + assert parse_files(back) == parse_files(files) + + +def test_join_clause_not_spanning_both_cubes_is_preserved(): + """A clause has to relate the two joined cubes. One comparing a cube to itself + (or reaching a third cube) is a valid Cube join with no Ossie relationship form, + so it is preserved verbatim instead of guessed at.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.a = {CUBE}.b\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + )) + ossie, back, issues = _roundtrip(files) + assert "relationships" not in model_of(ossie) + assert any("references cubes other than" in i.detail for i in issues) + assert parse_files(back) == parse_files(files) + + +def test_join_clause_reaching_an_unrelated_cube_is_preserved(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.user_id = {regions}.id\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + " - name: regions\n" + " sql_table: public.regions\n" + )) + ossie, back, issues = _roundtrip(files) + assert "relationships" not in model_of(ossie) + assert any("not between two member references" in i.detail for i in issues) + assert parse_files(back) == parse_files(files) + + +def test_join_clause_that_is_not_a_single_equality_is_preserved(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " joins:\n" + " - name: users\n" + " sql: \"{CUBE}.a = {users}.b = 1\"\n" + " relationship: many_to_one\n" + " - name: users\n" + " sql_table: public.users\n" + )) + ossie, back, issues = _roundtrip(files) + assert "relationships" not in model_of(ossie) + assert any("not a single equality" in i.detail for i in issues) + assert parse_files(back) == parse_files(files) + + +def test_metric_without_a_usable_dialect_is_dropped_with_an_issue(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " metrics:\n" + " - name: m\n" + " expression:\n" + " dialects:\n" + " - dialect: MAQL\n" + " expression: SELECT SUM(x)\n" + ) + files, issues = convert_ossie_to_cube(ossie) + assert "measures" not in parse(files["model/cubes/orders.yml"])["cubes"][0] + assert issues.of_type(IssueType.NO_USABLE_DIALECT) + + +# --- file layout ---------------------------------------------------------------- + +def test_off_layout_files_are_restored_with_their_grouping(): + """Import accepts any layout. Several cubes in one oddly-named file have to go + back into that same file, not be split into the canonical per-cube layout.""" + files = { + "schema/warehouse/everything.yaml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " - name: users\n" + " sql_table: public.users\n" + "views:\n" + " - name: main\n" + " description: All of it\n" + ), + } + ossie, back, _ = _roundtrip(files) + assert set(back) == {"schema/warehouse/everything.yaml"} + assert parse_files(back) == parse_files(files) + stash = stash_of(model_of(ossie)) + assert stash["cube_files"]["orders"] == "schema/warehouse/everything.yaml" + assert stash["view_files"]["main"] == "schema/warehouse/everything.yaml" + + +_MIXED_VIEW_FILE = ( + "views:\n" + " - name: sales\n" + " description: Sales overview\n" + " meta:\n" + " ai_context: Use for revenue questions.\n" + " cubes:\n" + " - join_path: orders\n" + " includes: '*'\n" + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " measures:\n" + " - name: revenue\n" + " sql: \"{CUBE}.amount\"\n" + " type: sum\n" +) + + +def test_a_view_file_may_also_define_cubes(): + """`cubes:` and `views:` are independent top-level keys, so one file can hold + both -- a self-contained model. Note the view's own nested `cubes:` (its + include list) is a different key at a different level and is not confused with + cube definitions.""" + files = {"model/views/sales.yml": _MIXED_VIEW_FILE} + ossie, _ = convert_cube_to_ossie(files) + model = model_of(ossie) + # The view supplied the model identity... + assert model["name"] == "sales" + assert model["description"] == "Sales overview" + assert model["ai_context"]["instructions"] == "Use for revenue questions." + # ...and the cube in the same file became the dataset. + assert [d["name"] for d in model["datasets"]] == ["orders"] + assert expr_of(model["metrics"][0]) == "SUM(orders.amount)" + # The view's include list round-trips as curation, not as a dataset. + assert stash_of(model)["views"]["sales"]["cubes"] == [ + {"join_path": "orders", "includes": "*"}] + + +def test_a_mixed_file_is_rebuilt_as_one_file(): + """Both halves have to go back into the single file they came from, rather than + being split into the canonical per-cube and per-view layout.""" + files = {"model/views/sales.yml": _MIXED_VIEW_FILE} + _, back, _ = _roundtrip(files) + assert set(back) == {"model/views/sales.yml"} + assert parse_files(back) == parse_files(files) + rebuilt = parse(back["model/views/sales.yml"]) + assert [c["name"] for c in rebuilt["cubes"]] == ["orders"] + assert [v["name"] for v in rebuilt["views"]] == ["sales"] + + +def test_a_cube_file_may_also_define_views(): + """The mirror image: the canonical cube path holding the view. The view's path is + the off-layout one here, so it is the one that gets stashed.""" + files = {"model/cubes/orders.yml": _MIXED_VIEW_FILE} + ossie, back, _ = _roundtrip(files) + assert stash_of(model_of(ossie))["view_files"]["sales"] == ( + "model/cubes/orders.yml") + assert "cube_files" not in stash_of(model_of(ossie)) + assert set(back) == {"model/cubes/orders.yml"} + assert parse_files(back) == parse_files(files) + + +def test_a_single_monolithic_file_round_trips(): + """Neither path is canonical, so both are stashed and both return to the one + file -- the shape you get from `-i model.yml`.""" + files = {"model.yml": _MIXED_VIEW_FILE} + ossie, back, _ = _roundtrip(files) + stash = stash_of(model_of(ossie)) + assert stash["cube_files"]["orders"] == "model.yml" + assert stash["view_files"]["sales"] == "model.yml" + assert set(back) == {"model.yml"} + assert parse_files(back) == parse_files(files) + + +def test_non_model_yaml_is_preserved_verbatim(): + files = { + "model/cubes/orders.yml": ( + "cubes:\n - name: orders\n sql_table: public.orders\n"), + "model/notes.yaml": "just: some data\n", + } + ossie, back, issues = _roundtrip(files) + assert back["model/notes.yaml"] == "just: some data\n" + assert issues.of_type(IssueType.PARKED_IN_META) + + +# --- the JavaScript-style mapping form ------------------------------------------ + +def test_collections_may_be_mappings_keyed_by_name(): + """Cube's post-transpile schema keys dimensions/measures/joins by name, and a + model converted from JavaScript can carry that shape. Both forms are accepted; + export always emits the list form YAML models use.""" + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " id:\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + " status:\n" + " sql: status\n" + " type: string\n" + " measures:\n" + " count:\n" + " type: count\n" + )) + ossie, _ = convert_cube_to_ossie(files) + model = model_of(ossie) + fields = by_name(by_name(model["datasets"])["orders"]["fields"]) + assert set(fields) == {"id", "status"} + assert expr_of(model["metrics"][0]) == "COUNT(DISTINCT orders.id)" + + back, _ = convert_ossie_to_cube(ossie) + cube = parse(back["model/cubes/m.yml"])["cubes"][0] + assert isinstance(cube["dimensions"], list) + assert isinstance(cube["measures"], list) + + +def test_a_collection_of_the_wrong_shape_is_rejected(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions: not-a-collection\n" + )) + with pytest.raises(ConversionError, match="expected a list or mapping"): + convert_cube_to_ossie(files) + + +# --- Jinja ---------------------------------------------------------------------- + +def test_jinja_anywhere_disqualifies_the_whole_file(): + """Jinja is detected per *file*, not per member -- Cube's own CubeSchemaConverter + uses the same file-level rule. So templating inside a single dimension's `sql` + still costs the whole file, which is preserved verbatim rather than + half-converted. There is deliberately no member-level Jinja path.""" + templated = ( + "cubes:\n" + " - name: templated\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: dyn\n" + " sql: \"{{ 'x' }}\"\n" + " type: string\n" + ) + files = { + "model/cubes/templated.yml": templated, + "model/cubes/plain.yml": ( + "cubes:\n - name: plain\n sql_table: public.plain\n"), + } + ossie, issues = convert_cube_to_ossie(files) + model = model_of(ossie) + assert [d["name"] for d in model["datasets"]] == ["plain"] + assert stash_of(model)["extra_files"]["model/cubes/templated.yml"] == templated + assert issues.of_type(IssueType.TEMPLATED_FILE_SKIPPED) + + # And it comes back byte-for-byte, since it was never parsed. + back, _ = convert_ossie_to_cube(ossie) + assert back["model/cubes/templated.yml"] == templated + + +# --- metadata corners ----------------------------------------------------------- + +def test_measure_title_survives_the_round_trip(): + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " measures:\n" + " - name: revenue\n" + " sql: \"{CUBE}.amount\"\n" + " type: sum\n" + " title: Total Revenue\n" + )) + ossie, back, _ = _roundtrip(files) + assert stash_of(model_of(ossie)["metrics"][0])["title"] == "Total Revenue" + assert parse_files(back) == parse_files(files) + + +def test_geo_dimension_extras_survive_the_split_and_merge(): + files = _files(users=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: home\n" + " type: geo\n" + " title: Home Location\n" + " description: Where they live\n" + " latitude:\n" + " sql: \"{CUBE}.lat\"\n" + " longitude:\n" + " sql: \"{CUBE}.lon\"\n" + )) + _, back, issues = _roundtrip(files) + assert parse_files(back) == parse_files(files) + assert issues.of_type(IssueType.GEO_DIMENSION_SPLIT) + + +_GEO_MODEL = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: users\n" + " source: public.users\n" + " fields:\n" + " - name: home_latitude\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: lat\n" + " datatype: Float\n" + " custom_extensions:\n" + " - vendor_name: CUBE\n" + " data: '{\"_v\": 1, \"geo\": {\"of\": \"home\", \"part\": \"latitude\"," + " \"sql\": \"{CUBE}.lat\"}}'\n" + " - name: home_longitude\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: lon\n" + " datatype: Float\n" + " custom_extensions:\n" + " - vendor_name: CUBE\n" + " data: '{\"_v\": 1, \"geo\": {\"of\": \"home\", \"part\": \"longitude\"," + " \"sql\": \"{CUBE}.lon\"}}'\n" + " metrics:\n" + " - name: avg_lat\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: AVG(users.home_latitude)\n" +) + + +def test_a_metric_referencing_a_geo_half_inlines_its_sql(): + """A split geo half's name exists only in Ossie: Cube has neither a column nor a + member called `home_latitude`, since the halves merge into the `home` dimension. + So a reference to one is replaced by the half's own SQL, which is valid Cube.""" + files, _ = convert_ossie_to_cube(_GEO_MODEL) + cube = parse(files["model/cubes/users.yml"])["cubes"][0] + assert cube["measures"] == [ + {"name": "avg_lat", "sql": "{CUBE}.lat", "type": "avg"}] + # And the dimension itself still merges back to a single geo member. + assert cube["dimensions"] == [{ + "name": "home", "type": "geo", + "latitude": {"sql": "{CUBE}.lat"}, + "longitude": {"sql": "{CUBE}.lon"}}] + + +def _two_cube_geo_model(expression): + """`_GEO_MODEL` plus an `orders.amount` field, and `expression` as the metric.""" + return _GEO_MODEL.replace( + " - name: users\n", " - name: orders\n source: public.orders\n" + " fields:\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" + " - name: users\n", 1 + ).replace(" expression: AVG(users.home_latitude)\n", + f" expression: {expression}\n") + + +def test_a_geo_half_reference_is_requalified_when_it_crosses_cubes(): + """`{CUBE}` means "the cube this is declared on", so inlining a snippet into + another cube's SQL has to name the original cube explicitly. + + One aggregate reading two datasets cannot be decomposed, so it lands on the base + cube and the `users` half travels there with it. + """ + model = _two_cube_geo_model("AVG(users.home_latitude - orders.amount)") + files, _ = convert_ossie_to_cube(model, base_cube="orders") + cube = parse(files["model/cubes/orders.yml"])["cubes"][0] + assert cube["measures"] == [ + {"name": "avg_lat", "sql": "{users}.lat - {CUBE}.amount", "type": "avg"}] + + +def test_a_decomposed_part_lands_on_the_cube_its_operand_reads(): + """Two aggregates over two datasets: each part is declared on the cube it reads, + which is what lets Cube correct row multiplication for each independently. So the + geo half needs no requalification -- its part lives on `users` already.""" + model = _two_cube_geo_model( + "AVG(users.home_latitude) - MIN(orders.amount)") + files, _ = convert_ossie_to_cube(model, base_cube="orders") + on_users = by_name(parse(files["model/cubes/users.yml"])["cubes"][0]["measures"]) + on_orders = by_name(parse(files["model/cubes/orders.yml"])["cubes"][0]["measures"]) + assert on_users["avg_lat_part_1"]["sql"] == "{CUBE}.lat" + assert on_users["avg_lat_part_1"]["public"] is False + assert on_orders["avg_lat_part_2"]["sql"] == "{CUBE}.amount" + # The public measure stays on the base cube, naming the foreign part by its cube + # and its own with `{CUBE.x}`. + assert on_orders["avg_lat"]["sql"] == ( + "{users.avg_lat_part_1} - {CUBE.avg_lat_part_2}") + assert "public" not in on_orders["avg_lat"] + + +def test_geo_half_references_normalize_to_the_underlying_column(): + """Documented normalization: after a round trip the metric names the column the + geo half actually reads rather than the Ossie-only field name. Semantically the + same reference, and it is what Cube can express.""" + files, _ = convert_ossie_to_cube(_GEO_MODEL) + ossie2, _ = convert_cube_to_ossie(files) + metric = model_of(ossie2)["metrics"][0] + assert expr_of(metric) == "AVG(users.lat)" + + +def _geo_stash(part, of="home"): + return ('{"_v": 1, "geo": {"of": "' + of + '", "part": "' + part + + '", "sql": "{CUBE}.' + part[:3] + '"}}') + + +def _ossie_fields(*specs): + """Build an Ossie model from (field name, expression, geo part or None) specs.""" + out = ("version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: users\n" + " source: public.users\n" + " fields:\n") + for fname, expr, part in specs: + out += (f" - name: {fname}\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + f" expression: {expr}\n" + " datatype: String\n") + if part: + out += (" custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{_geo_stash(part)}'\n") + return out + + +def _ossie_pk(primary_key, *specs): + """An Ossie model with a primary_key and (name, expression, geo part) fields.""" + out = ("version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " primary_key:\n") + for col in primary_key: + out += f" - {col}\n" + out += " fields:\n" + for fname, expr, part in specs: + out += (f" - name: {fname}\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + f" expression: {expr}\n" + " datatype: String\n") + if part: + out += (" custom_extensions:\n" + " - vendor_name: CUBE\n" + f" data: '{_geo_stash(part, of=fname.rsplit('_', 1)[0])}'\n") + return out + + +def _dims(files): + return parse(files["model/cubes/orders.yml"])["cubes"][0]["dimensions"] + + +def test_a_computed_dimension_does_not_cover_a_primary_key(): + """`primary_key: true` in Cube declares that dimension's own sql to be the key. + Marking a computed dimension would declare `LOWER(email)` as the key when Ossie + named the `id` column -- so a name match alone must not count as coverage.""" + files, issues = convert_ossie_to_cube( + _ossie_pk(["id"], ("id", "LOWER(email)", None))) + dims = by_name(_dims(files)) + assert "primary_key" not in dims["id"] + assert dims["id"]["sql"] == "LOWER(email)" + # A private scalar dimension carries the key instead, under a free name. + assert dims["id_pk"] == {"name": "id_pk", "sql": "id", "type": "string", + "primary_key": True, "public": False} + assert issues.of_type(IssueType.APPROXIMATED) + + +def test_a_merged_geo_dimension_does_not_cover_a_primary_key(): + """A geo dimension has two sql expressions and no single one, so it cannot be + the key even though its name matches.""" + files, _ = convert_ossie_to_cube(_ossie_pk( + ["location"], + ("location_latitude", "lat", "latitude"), + ("location_longitude", "lon", "longitude"))) + dims = by_name(_dims(files)) + assert dims["location"]["type"] == "geo" + assert "primary_key" not in dims["location"] + assert dims["location_pk"] == { + "name": "location_pk", "sql": "location", "type": "string", + "primary_key": True, "public": False} + + +def test_a_scalar_dimension_backed_by_the_key_column_covers_it(): + """The legitimate case: a differently-named dimension whose sql *is* the key + column. It stays the key, and nothing is synthesized alongside it.""" + files, issues = convert_ossie_to_cube( + _ossie_pk(["id"], ("order_id", "id", None))) + dims = _dims(files) + assert len(dims) == 1 + assert dims[0]["name"] == "order_id" + assert dims[0]["primary_key"] is True + assert not issues.of_type(IssueType.APPROXIMATED) + + +def test_a_scalar_dimension_named_as_the_key_covers_it(): + """Import records the *dimension name* in `primary_key`, not the column, so a + scalar dimension matching by name has to keep covering it -- otherwise + `Cube -> Ossie -> Cube` would synthesize a bogus duplicate key.""" + src = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + " dimensions:\n" + " - name: order_id\n" + " sql: id\n" + " type: number\n" + " primary_key: true\n" + )) + ossie, _ = convert_cube_to_ossie(src) + assert by_name(model_of(ossie)["datasets"])["orders"]["primary_key"] == [ + "order_id"] + back, _ = convert_ossie_to_cube(ossie) + assert parse_files(back) == parse_files(src) + + +def test_a_synthesized_key_name_avoids_every_existing_member(): + """Suffixing has to keep going while names are taken, and the result must still + be a single non-public scalar dimension.""" + files, _ = convert_ossie_to_cube(_ossie_pk( + ["id"], + ("id", "LOWER(email)", None), + ("id_pk", "UPPER(email)", None), + ("id_pk_2", "TRIM(email)", None))) + dims = by_name(_dims(files)) + keys = [n for n, d in dims.items() if d.get("primary_key")] + assert keys == ["id_pk_3"] + assert dims["id_pk_3"] == {"name": "id_pk_3", "sql": "id", "type": "string", + "primary_key": True, "public": False} + # Nothing was overwritten. + assert dims["id"]["sql"] == "LOWER(email)" + assert dims["id_pk"]["sql"] == "UPPER(email)" + assert dims["id_pk_2"]["sql"] == "TRIM(email)" + + +def test_geo_halves_may_appear_in_any_order_without_clobbering_a_dimension(): + """The geo dimension is assembled from two fields that need not be adjacent and + may come in either order. Holding its place with a list index computed mid-loop + overwrote whatever real dimension already sat at that index -- here `city` + vanished entirely.""" + model = _ossie_fields( + ("home_longitude", "lon", "longitude"), + ("city", "city", None), + ("home_latitude", "lat", "latitude"), + ) + files, _ = convert_ossie_to_cube(model) + dims = parse(files["model/cubes/users.yml"])["cubes"][0]["dimensions"] + assert [d["name"] for d in dims] == ["home", "city"] + assert by_name(dims)["home"] == { + "name": "home", "type": "geo", + "latitude": {"sql": "{CUBE}.lat"}, + "longitude": {"sql": "{CUBE}.lon"}} + assert by_name(dims)["city"]["sql"] == "city" + + +def test_a_geo_base_colliding_with_a_field_is_rejected_in_either_order(): + """The base is the merged dimension's name, so it cannot also be an ordinary + dimension -- that would emit two members of the same name. Whether the ordinary + field comes first must not decide whether this is caught.""" + for specs in ( + (("home", "home", None), ("home_latitude", "lat", "latitude"), + ("home_longitude", "lon", "longitude")), + (("home_latitude", "lat", "latitude"), + ("home_longitude", "lon", "longitude"), ("home", "home", None)), + ): + with pytest.raises(ConversionError, match="collides"): + convert_ossie_to_cube(_ossie_fields(*specs)) + + +def test_two_fields_claiming_the_same_geo_half_are_rejected(): + model = _ossie_fields( + ("a_lat", "lat", "latitude"), + ("b_lat", "lat2", "latitude"), + ("home_longitude", "lon", "longitude"), + ) + with pytest.raises(ConversionError, match="both claim the latitude"): + convert_ossie_to_cube(model) + + +def test_a_geo_dimension_missing_a_half_is_rejected_on_export(): + model = _ossie_fields(("home_latitude", "lat", "latitude")) + with pytest.raises(ConversionError, match="missing its longitude half"): + convert_ossie_to_cube(model) + + +def test_an_unknown_geo_part_is_rejected(): + model = _ossie_fields(("home_altitude", "alt", "altitude")) + with pytest.raises(ConversionError, match="geo part 'altitude'"): + convert_ossie_to_cube(model) + + +def test_geo_dimension_missing_a_half_is_rejected(): + files = _files(users=( + "cubes:\n" + " - name: users\n" + " sql_table: public.users\n" + " dimensions:\n" + " - name: home\n" + " type: geo\n" + " latitude:\n" + " sql: lat\n" + )) + with pytest.raises(ConversionError, match="missing 'longitude.sql'"): + convert_cube_to_ossie(files) + + +def test_ai_context_examples_reach_cube_as_prose_and_park_structurally(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " ai_context:\n" + " instructions: Sales model.\n" + " examples:\n" + " - What were sales last month?\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + ) + files, _ = convert_ossie_to_cube(ossie) + meta = parse(files["model/views/shop.yml"])["views"][0]["meta"] + assert meta["ai_context"] == ( + "Sales model.\nExample questions: What were sales last month?") + assert meta["ossie"]["ai_context"]["examples"] == [ + "What were sales last month?"] + # And the structured form is what comes back, not the flattened prose. + ossie2, _ = convert_cube_to_ossie(files) + assert model_of(ossie2)["ai_context"]["examples"] == [ + "What were sales last month?"] + + +def test_a_plain_string_ai_context_survives_as_a_string(): + """Ossie allows `ai_context` to be a bare string. Import reads Cube's prose back + as {'instructions': ...}, so the original scalar has to be parked to survive.""" + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: public.orders\n" + " ai_context: orders, purchases, sales\n" + ) + files, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files) + ds = by_name(model_of(ossie2)["datasets"])["orders"] + assert ds["ai_context"] == "orders, purchases, sales" + + +# --- multiple views ------------------------------------------------------------- + +_TWO_VIEWS = { + "model/cubes/orders.yml": ( + "cubes:\n - name: orders\n sql_table: public.orders\n"), + "model/views/a.yml": "views:\n - name: a\n description: View A\n", + "model/views/b.yml": "views:\n - name: b\n description: View B\n", +} + + +def test_several_views_need_an_explicit_choice(): + _, issues = convert_cube_to_ossie(_TWO_VIEWS) + assert any("none chosen with --view" in i.detail + for i in issues.of_type(IssueType.PARKED_IN_META)) + + +def test_choosing_a_view_maps_its_metadata_onto_the_model(): + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") + model = model_of(ossie) + assert model["name"] == "b" + assert model["description"] == "View B" + # The unchosen view is still preserved whole. + assert set(stash_of(model)["views"]) == {"a", "b"} + + +def test_foreign_extensions_with_no_mapped_view_are_refused_not_dropped(): + """Model-level foreign-vendor extensions ride on the view that represents the + model. With several views and none mapped there is no such view, and picking one + arbitrarily would not survive a re-import -- only the mapped view's parked + extensions are read back. So this is refused rather than silently losing them.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS) + doc = parse(ossie) + doc["semantic_model"][0].setdefault("custom_extensions", []).append( + {"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "ANALYTICS_WH"}'}) + with pytest.raises(ConversionError, match="SNOWFLAKE"): + convert_ossie_to_cube(dump_yaml(doc)) + + +def test_foreign_extensions_survive_once_a_view_is_mapped(): + """The fix the error message points at: choose the view the model maps to, and + the extensions have a home again.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") + doc = parse(ossie) + doc["semantic_model"][0].setdefault("custom_extensions", []).append( + {"vendor_name": "SNOWFLAKE", "data": '{"warehouse": "ANALYTICS_WH"}'}) + files, _ = convert_ossie_to_cube(dump_yaml(doc)) + parked = parse(files["model/views/b.yml"])["views"][0]["meta"]["ossie"] + assert parked["custom_extensions"][0]["vendor_name"] == "SNOWFLAKE" + # And they come back as Ossie extensions, not just stashed text. + ossie2, _ = convert_cube_to_ossie(files, view="b") + vendors = {e["vendor_name"] for e in model_of(ossie2)["custom_extensions"]} + assert "SNOWFLAKE" in vendors + + +_TWO_VIEWS_ONE_FILE = { + "model/all.yml": ( + "cubes:\n" + " - name: orders\n" + " sql_table: public.orders\n" + "views:\n" + " - name: alpha\n" + " description: A\n" + " cubes:\n" + " - join_path: orders\n" + " includes: '*'\n" + " - name: beta\n" + " description: B\n" + ), +} + + +def test_several_views_in_one_file_all_survive(): + """Views were keyed one-per-path on export, so two sharing a file meant the second + overwrote the first. The lost one here is `alpha` -- the *mapped* view, which is + where the model's own description and AI context live.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS_ONE_FILE, view="alpha") + back, _ = convert_ossie_to_cube(ossie) + assert set(back) == {"model/all.yml"} + rebuilt = parse(back["model/all.yml"]) + # Declaration order preserved, both present. + assert [v["name"] for v in rebuilt["views"]] == ["alpha", "beta"] + assert [c["name"] for c in rebuilt["cubes"]] == ["orders"] + assert parse_files(back) == parse_files(_TWO_VIEWS_ONE_FILE) + + +def test_the_mapped_view_in_a_shared_file_still_carries_model_metadata(): + """The mapped view is the model's home for description and AI context, so it has + to be the one updated -- not whichever view happens to be written last.""" + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS_ONE_FILE, view="alpha") + model = model_of(ossie) + assert model["name"] == "alpha" + assert model["description"] == "A" + + model["description"] = "edited" + files, _ = convert_ossie_to_cube(dump_yaml({ + "version": "0.2.0.dev0", "semantic_model": [model]})) + views = by_name(parse(files["model/all.yml"])["views"]) + assert views["alpha"]["description"] == "edited" + assert views["beta"]["description"] == "B" + + +def test_both_views_are_restored_on_export(): + ossie, _ = convert_cube_to_ossie(_TWO_VIEWS, view="b") + back, _ = convert_ossie_to_cube(ossie) + assert parse_files(back) == parse_files(_TWO_VIEWS) + + +# --- malformed input ------------------------------------------------------------ + +def test_malformed_yaml_is_reported_cleanly(): + with pytest.raises(ConversionError, match="Invalid YAML"): + convert_cube_to_ossie({"model/cubes/m.yml": "cubes: [oops\n"}) + + +def test_empty_input_is_rejected(): + with pytest.raises(ConversionError, match="non-empty mapping"): + convert_cube_to_ossie({}) + + +def test_a_non_string_name_is_rejected_cleanly(): + files = _files(m="cubes:\n - name: 42\n sql_table: t\n") + with pytest.raises(ConversionError, match="must be a string"): + convert_cube_to_ossie(files) + + +def test_ossie_root_must_be_a_mapping(): + with pytest.raises(ConversionError, match="expected a mapping at the root"): + convert_ossie_to_cube("- just\n- a\n- list\n") + + +def test_measure_without_a_type_is_rejected(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: t\n" + " measures:\n" + " - name: m\n" + " sql: amount\n" + )) + with pytest.raises(ConversionError, match="missing required 'type'"): + convert_cube_to_ossie(files) + + +def test_unknown_dimension_type_is_rejected(): + files = _files(m=( + "cubes:\n" + " - name: orders\n" + " sql_table: t\n" + " dimensions:\n" + " - name: d\n" + " sql: d\n" + " type: quaternion\n" + )) + with pytest.raises(ConversionError, match="unknown type 'quaternion'"): + convert_cube_to_ossie(files) + + +def test_unknown_ossie_datatype_is_rejected(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: f\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: f\n" + " datatype: Quaternion\n" + ) + with pytest.raises(ConversionError, match="unknown datatype"): + convert_ossie_to_cube(ossie) + + +def test_dataset_without_a_source_is_rejected_on_export(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: shop\n" + " datasets:\n" + " - name: orders\n" + ) + with pytest.raises(ConversionError, match="missing/empty 'source'"): + convert_ossie_to_cube(ossie) + + +def test_several_semantic_models_convert_the_first_with_an_issue(): + ossie = ( + "version: 0.2.0.dev0\n" + "semantic_model:\n" + "- name: first\n" + " datasets:\n" + " - name: orders\n" + " source: t\n" + "- name: second\n" + " datasets:\n" + " - name: users\n" + " source: t\n" + ) + files, issues = convert_ossie_to_cube(ossie) + assert set(files) == {"model/cubes/orders.yml", "model/views/first.yml"} + # The other models are not preserved anywhere, so this is a drop. + dropped = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT) + assert any("only the first is converted" in i.detail for i in dropped) + + +# --- string literals ------------------------------------------------------------ +# +# The two directions are deliberately asymmetric, so both are pinned here. A Cube +# YAML `sql` is compiled as a Python f-string (`f""` in YamlCompiler), which +# interpolates `{...}` anywhere in the value -- SQL's own quotes mean nothing to it. +# So on import a reference inside a literal is a real reference, while on export a +# rewrite must stop at the quotes or it would destroy the literal's text. + +@pytest.mark.parametrize("sql,expected", [ + ("a = 'x'", [("a = ", False), ("'x'", True)]), + ("'x' = a", [("'x'", True), (" = a", False)]), + ("'it''s'", [("'it'", True), ("'s'", True)]), + ('"col" = `c`', [('"col"', True), (" = ", False), ("`c`", True)]), + ("a = 'unterminated", [("a = ", False), ("'unterminated", True)]), + ("plain", [("plain", False)]), +]) +def test_quoted_runs_splits_sql_into_code_and_quoted_text(sql, expected): + from ossie_cube._common import quoted_runs + assert quoted_runs(sql) == expected + + +@pytest.mark.parametrize("expr,expected", [ + ("SUM(orders.amount)", {"orders"}), + ("SUM(orders.amount) / COUNT(users.id)", {"orders", "users"}), + ("SUM(orders.amount) || ' per users.id unit'", {"orders"}), + ("'orders.amount'", set()), + ("SUM(ghost.amount)", set()), +]) +def test_referenced_datasets_ignores_quoted_text(expr, expected): + from ossie_cube._common import referenced_datasets + assert referenced_datasets(expr, {"orders", "users"}) == expected + + +def test_a_reference_inside_a_literal_is_still_translated_on_import(): + """Not an oversight: Cube would have interpolated it, so dropping it would lose a + reference the model really does resolve.""" + files = _files(orders=( + "cubes:\n" + " - name: orders\n" + " sql_table: a.b.orders\n" + " dimensions:\n" + " - name: note\n" + " sql: \"CONCAT({CUBE}.status, ' {CUBE}.status ')\"\n" + " type: string\n" + )) + ossie, _ = convert_cube_to_ossie(files) + field = by_name(by_name(model_of(ossie)["datasets"])["orders"]["fields"])["note"] + assert expr_of(field) == "CONCAT(status, ' status ')" + + +# --- aggregate span scanning ----------------------------------------------------- +# +# The scanner decides whether a metric is decomposed into one measure per aggregate, +# so its rejection paths matter as much as its matches: a false positive splices a +# measure reference into text that was never a call. + +@pytest.mark.parametrize("expr,expected", [ + # Two aggregates -- the case decomposition exists for. + ("SUM(a.x) / COUNT(b.y)", ["SUM(a.x)", "COUNT(b.y)"]), + # Only the outermost of a nested pair. + ("SUM(a.x) / NULLIF(SUM(b.y), 0)", ["SUM(a.x)", "SUM(b.y)"]), + # A closing paren inside a literal does not end the call. + ("SUM(a.x || ')') / COUNT(b.y)", ["SUM(a.x || ')')", "COUNT(b.y)"]), + # Part of a longer identifier, not a call. + ("MY_SUM(a.x) / 2", []), + ("SUMMARY(a.x) - MIN(b.y)", ["MIN(b.y)"]), + # A name with no argument list at all. + ("a.count / b.total", []), + # Whitespace between the name and its parens is still a call. + ("SUM (a.x) - MIN (b.y)", ["SUM (a.x)", "MIN (b.y)"]), + # Unbalanced parens: not a span, and not a crash. + ("SUM(a.x / MIN(b.y)", []), + # A single aggregate needs no decomposition. + ("SUM(a.x)", []), + # Unparseable input falls back to one opaque measure. + ("SUM(a.x) /// COUNT(", []), +]) +def test_aggregate_spans_only_matches_real_calls(expr, expected): + from ossie_cube.expressions import aggregate_spans + assert [expr[s:e] for s, e in aggregate_spans(expr)] == expected + + +@pytest.mark.parametrize("expr,expected", [ + ("SUM(a.x)", False), + # One self-contained term: the space is inside the parens, so inlining it into a + # larger expression needs no parentheses. + ("COUNT(DISTINCT a.x)", False), + ("SUM(a.x) / 2", True), + ("CASE WHEN a.x THEN 1 END", True), # a top-level space is structure + ("'a + b'", False), # operators inside a literal are text + ("'a b'", False), + ('"a b"', False), +]) +def test_has_top_level_operator_ignores_quoted_text(expr, expected): + from ossie_cube.expressions import has_top_level_operator + assert has_top_level_operator(expr) is expected diff --git a/converters/cube/tests/test_osi_to_cube.py b/converters/cube/tests/test_osi_to_cube.py new file mode 100644 index 00000000..f685b029 --- /dev/null +++ b/converters/cube/tests/test_osi_to_cube.py @@ -0,0 +1,546 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Apache Ossie semantic model -> Cube data model.""" + +import pytest +from _util import by_name, expr_of, model_of, parse + +from ossie_cube import ( + ConversionError, + IssueType, + convert_cube_to_ossie, + convert_ossie_to_cube, +) +from ossie_cube._common import OSSIE_VERSION + + +def _ossie(datasets, relationships="", metrics="", model_extra=""): + return (f"version: {OSSIE_VERSION}\n" + "semantic_model:\n" + "- name: shop\n" + f"{model_extra}" + " datasets:\n" + f"{datasets}" + f"{relationships}" + f"{metrics}") + + +_ORDERS = ( + " - name: orders\n" + " source: sales.public.orders\n" + " primary_key:\n" + " - id\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" +) + + +def _cubes(files, path="model/cubes/orders.yml"): + return by_name(parse(files[path])["cubes"]) + + +# --- layout --------------------------------------------------------------------- + +def test_emits_one_file_per_cube_plus_a_view(): + files, _ = convert_ossie_to_cube(_ossie(_ORDERS)) + assert set(files) == {"model/cubes/orders.yml", "model/views/shop.yml"} + + +def test_version_is_enforced(): + with pytest.raises(ConversionError, match="Unsupported Ossie version"): + convert_ossie_to_cube("version: 9.9.9\nsemantic_model: []\n") + + +def test_model_without_datasets_is_rejected(): + with pytest.raises(ConversionError, match="no datasets"): + convert_ossie_to_cube( + f"version: {OSSIE_VERSION}\nsemantic_model:\n- name: shop\n datasets: []\n") + + +def test_relationship_to_unknown_dataset_is_rejected(): + rel = (" relationships:\n" + " - name: r\n from: orders\n to: ghosts\n" + " from_columns: [x]\n to_columns: [y]\n") + with pytest.raises(ConversionError, match="unknown dataset"): + convert_ossie_to_cube(_ossie(_ORDERS, rel)) + + +def test_mismatched_relationship_columns_are_rejected(): + rel = (" relationships:\n" + " - name: r\n from: orders\n to: orders\n" + " from_columns: [a, b]\n to_columns: [c]\n") + with pytest.raises(ConversionError, match="same length"): + convert_ossie_to_cube(_ossie(_ORDERS, rel)) + + +# --- datasets and fields -------------------------------------------------------- + +def test_source_becomes_sql_table_or_sql(): + files, _ = convert_ossie_to_cube(_ossie(_ORDERS)) + assert _cubes(files)["orders"]["sql_table"] == "sales.public.orders" + + query = _ORDERS.replace("source: sales.public.orders", + "source: SELECT * FROM raw.orders") + files, _ = convert_ossie_to_cube(_ossie(query)) + cube = _cubes(files)["orders"] + assert cube["sql"] == "SELECT * FROM raw.orders" + assert "sql_table" not in cube + + +def test_every_dimension_declares_a_type(): + """Cube's schema requires `type` on every dimension, so the converter always + emits one -- falling back to `string` with an issue when Ossie carries none.""" + no_type = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: note\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: note\n" + ) + files, issues = convert_ossie_to_cube(_ossie(no_type)) + assert _cubes(files)["orders"]["dimensions"][0]["type"] == "string" + # A guess, not a loss and not a park: Cube demands a type Ossie never gave. + assert issues.of_type(IssueType.APPROXIMATED) + + +@pytest.mark.parametrize("datatype,expected", [ + ("String", "string"), + ("Integer", "number"), + ("Decimal", "number"), + ("Float", "number"), + ("Boolean", "boolean"), + ("Date", "time"), + ("DateTime", "time"), + ("DateTimeTz", "time"), + ("Opaque", "string"), +]) +def test_datatype_maps_to_cube_type(datatype, expected): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: f\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: f\n" + f" datatype: {datatype}\n" + ) + files, _ = convert_ossie_to_cube(_ossie(ds)) + assert _cubes(files)["orders"]["dimensions"][0]["type"] == expected + + +def test_is_time_on_a_non_temporal_datatype_is_reported(): + """Cube marks time dimensions by `type`, so an Integer year grain cannot carry + the temporal role -- that is a real loss and it is reported, not hidden.""" + ds = ( + " - name: date_dim\n" + " source: t\n" + " fields:\n" + " - name: d_year\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: d_year\n" + " datatype: Integer\n" + " dimension:\n" + " is_time: true\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds)) + dim = parse(files["model/cubes/date_dim.yml"])["cubes"][0]["dimensions"][0] + assert dim["type"] == "number" + # The temporal role is gone from the output, so this is a drop. + detail = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT)[0].detail + assert "temporal role is not carried" in detail + + +def test_primary_key_column_without_a_field_is_synthesized(): + ds = ( + " - name: orders\n" + " source: t\n" + " primary_key:\n" + " - ticket_no\n" + " fields:\n" + " - name: amount\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: amount\n" + " datatype: Decimal\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds)) + dims = by_name(_cubes(files)["orders"]["dimensions"]) + assert dims["ticket_no"] == { + "name": "ticket_no", "sql": "ticket_no", "type": "string", + "primary_key": True, "public": False} + # `type: string` is chosen by the converter, not carried by Ossie. + assert issues.of_type(IssueType.APPROXIMATED) + + +def test_field_name_is_sanitized_and_collisions_are_rejected(): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: Order Status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status\n" + " - name: order status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status2\n" + ) + with pytest.raises(ConversionError, match="collides"): + convert_ossie_to_cube(_ossie(ds)) + + +def test_field_collision_is_rejected_before_any_metric_is_placed(): + """Dimension names are resolved once, up front. Resolving them per stage let a + collision go undetected while measures were being placed -- so the member set + that decides `{CUBE.member}` vs `{CUBE}.column` could be silently short a name, + and the error surfaced later and less clearly.""" + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: Order Status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status\n" + " - name: order status\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: status2\n" + ) + metrics = _metric("m", "SUM(orders.amount)") + with pytest.raises(ConversionError, match="collides"): + convert_ossie_to_cube(_ossie(ds, metrics=metrics)) + + +def test_missing_dialect_drops_the_field_with_an_issue(): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: f\n" + " expression:\n" + " dialects:\n" + " - dialect: MDX\n" + " expression: '[f]'\n" + ) + files, issues = convert_ossie_to_cube(_ossie(ds)) + assert "dimensions" not in _cubes(files)["orders"] + assert issues.of_type(IssueType.NO_USABLE_DIALECT) + + +def test_preferred_dialect_wins_over_ansi(): + ds = ( + " - name: orders\n" + " source: t\n" + " fields:\n" + " - name: email\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: LOWER(email)\n" + " - dialect: SNOWFLAKE\n" + " expression: LOWER(email)::VARCHAR\n" + " datatype: String\n" + ) + files, _ = convert_ossie_to_cube(_ossie(ds), dialect="SNOWFLAKE") + assert _cubes(files)["orders"]["dimensions"][0]["sql"] == "LOWER(email)::VARCHAR" + + +# --- joins ---------------------------------------------------------------------- + +_TWO_DATASETS = _ORDERS + ( + " - name: users\n" + " source: sales.public.users\n" + " primary_key:\n" + " - id\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" +) +_REL = (" relationships:\n" + " - name: orders_to_users\n" + " from: orders\n" + " to: users\n" + " from_columns: [user_id]\n" + " to_columns: [id]\n") + + +def test_relationship_lands_on_the_many_side_as_many_to_one(): + files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL)) + join = _cubes(files)["orders"]["joins"][0] + # Alias-dot on both sides: Ossie's from_columns/to_columns name columns, so the + # far side is a raw column reference too, not a member reference. + assert join == {"name": "users", "sql": "{CUBE}.user_id = {users}.id", + "relationship": "many_to_one"} + # The one side declares nothing; Cube needs the join on one side only. + assert "joins" not in _cubes(files, "model/cubes/users.yml")["users"] + + +def test_composite_relationship_becomes_an_and_chain(): + rel = (" relationships:\n" + " - name: r\n from: orders\n to: users\n" + " from_columns: [user_id, region]\n" + " to_columns: [id, region]\n") + files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) + assert _cubes(files)["orders"]["joins"][0]["sql"] == ( + "{CUBE}.user_id = {users}.id AND {CUBE}.region = {users}.region") + + +def test_relationship_ai_context_is_reported_as_dropped_not_parked(): + """A Cube join entry takes only name/sql/relationship -- no `meta` -- so this is + one of the few things that genuinely cannot be preserved. It is reported under + DROPPED_NO_CUBE_EQUIVALENT rather than PARKED_IN_META, so a caller gating on + issue types can tell real loss from "preserved but invisible to Cube".""" + rel = _REL + " ai_context:\n instructions: Join carefully.\n" + _, issues = convert_ossie_to_cube(_ossie(_TWO_DATASETS, rel)) + dropped = issues.of_type(IssueType.DROPPED_NO_CUBE_EQUIVALENT) + assert [i.element_name for i in dropped] == ["relationship 'orders_to_users'"] + assert "ai_context" in dropped[0].detail + assert not issues.of_type(IssueType.PARKED_IN_META) + + +# --- metrics -------------------------------------------------------------------- + +def _metric(name, expr): + return (" metrics:\n" + f" - name: {name}\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + f" expression: {expr}\n") + + +@pytest.mark.parametrize("expr,expected", [ + ("SUM(orders.amount)", {"type": "sum", "sql": "{CUBE}.amount"}), + ("AVG(orders.amount)", {"type": "avg", "sql": "{CUBE}.amount"}), + ("MIN(orders.amount)", {"type": "min", "sql": "{CUBE}.amount"}), + ("MAX(orders.amount)", {"type": "max", "sql": "{CUBE}.amount"}), + ("COUNT(DISTINCT orders.amount)", + {"type": "count_distinct", "sql": "{CUBE}.amount"}), + ("APPROX_COUNT_DISTINCT(orders.amount)", + {"type": "count_distinct_approx", "sql": "{CUBE}.amount"}), +]) +def test_aggregate_expressions_become_structured_measures(expr, expected): + files, _ = convert_ossie_to_cube(_ossie(_ORDERS, metrics=_metric("m", expr))) + measure = _cubes(files)["orders"]["measures"][0] + assert {k: v for k, v in measure.items() if k != "name"} == expected + + +def test_count_distinct_over_the_primary_key_becomes_a_bare_count(): + """The inverse of the import rule: COUNT(DISTINCT ) is exactly Cube's + fan-out-safe `type: count`, so it round-trips back to the idiomatic form.""" + files, _ = convert_ossie_to_cube( + _ossie(_ORDERS, metrics=_metric("m", "COUNT(DISTINCT orders.id)"))) + measure = _cubes(files)["orders"]["measures"][0] + assert measure == {"name": "m", "type": "count"} + + +def test_declared_member_gets_a_member_reference_and_a_raw_column_does_not(): + """`{CUBE.member}` reuses a declared member's SQL and is compile-time checked; + `{CUBE}.column` passes a raw column through. The choice follows from whether + the dataset declares a field of that name.""" + files, _ = convert_ossie_to_cube( + _ossie(_ORDERS, metrics=_metric("m", "SUM(orders.shipping_fee)"))) + # `shipping_fee` is not a declared field, so it stays a raw column. + assert _cubes(files)["orders"]["measures"][0]["sql"] == "{CUBE}.shipping_fee" + + +def test_a_ratio_is_split_into_one_measure_per_aggregate(): + """Each aggregate becomes its own `public: false` measure on the cube its operand + comes from, and the public measure references them. Cube corrects for row + multiplication per measure, so splitting is what lets each aggregate be corrected + on its own cube instead of the whole ratio being one opaque expression.""" + files, _ = convert_ossie_to_cube(_ossie( + _TWO_DATASETS, _REL, + _metric("aov", "SUM(orders.amount) / COUNT(DISTINCT users.id)"))) + orders = by_name(_cubes(files)["orders"]["measures"]) + users = by_name(parse(files["model/cubes/users.yml"])["cubes"][0]["measures"]) + + assert orders["aov_part_1"] == { + "name": "aov_part_1", "sql": "{CUBE}.amount", "type": "sum", + "meta": {"ossie": {"part_of": "aov"}}, "public": False} + # `users.id` is that cube's primary key, so its aggregate is a bare Cube count -- + # the form Cube corrects for fan-out. + assert users["aov_part_2"] == { + "name": "aov_part_2", "type": "count", + "meta": {"ossie": {"part_of": "aov"}}, "public": False} + # `{CUBE.aov_part_1}` rather than `{orders.aov_part_1}`: an own-cube reference + # stays correct when the cube is extended. + assert orders["aov"] == { + "name": "aov", "type": "number", + "sql": "{CUBE.aov_part_1} / {users.aov_part_2}"} + + +def test_a_dotted_token_inside_a_string_literal_is_left_alone(): + """Cube compiles a YAML `sql` as a Python f-string, so a `{...}` written into a + string literal is still interpolated -- it would replace the literal's own text + with a column reference. So the rewrite has to stop at the quotes.""" + files, _ = convert_ossie_to_cube(_ossie(_ORDERS, metrics=_metric( + "m", "CONCAT(CAST(SUM(orders.amount) AS VARCHAR), ' orders.amount ')"))) + assert _cubes(files)["orders"]["measures"][0]["sql"] == ( + "CONCAT(CAST(SUM({CUBE}.amount) AS VARCHAR), ' orders.amount ')") + + +def test_an_aggregate_name_inside_a_string_literal_is_not_an_aggregate(): + """Otherwise the literal is treated as a second aggregate and gets a measure + reference spliced into the middle of it.""" + files, issues = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL, _metric( + "label", "SUM(orders.amount) || ' per COUNT(users.id) unit'"))) + measures = _cubes(files)["orders"]["measures"] + # One measure, not a decomposed pair, and the literal survives verbatim. + assert [m["name"] for m in measures] == ["label"] + assert measures[0]["sql"] == ( + "SUM({CUBE}.amount) || ' per COUNT(users.id) unit'") + assert "measures" not in _cubes(files, "model/cubes/users.yml")["users"] + # `users` is named only inside the literal, so this is not a cross-cube metric. + assert not issues.of_type(IssueType.APPROXIMATED) + + +@pytest.mark.parametrize("shape,expr", [ + ("decomposed", "SUM(orders.amount) / COUNT(DISTINCT users.id)"), + ("single aggregate", "SUM(orders.amount - users.id)"), + ("calculated", "SUM(orders.amount) + users.id"), +]) +def test_a_cross_dataset_metric_is_reported_whatever_shape_it_takes(shape, expr): + """Cube reaches another cube's members through an implicit join, so the model + needs a join path this converter cannot verify. The report used to come only from + the calculated-measure fallback, which meant the decomposed shape -- the one with + the *most* cross-cube references -- reported nothing.""" + _, issues = convert_ossie_to_cube( + _ossie(_TWO_DATASETS, _REL, _metric("m", expr))) + reported = issues.of_type(IssueType.APPROXIMATED) + assert len(reported) == 1, shape + assert "orders, users" in reported[0].detail + assert "join path" in reported[0].detail + + +def test_a_single_dataset_metric_is_not_reported(): + _, issues = convert_ossie_to_cube(_ossie( + _TWO_DATASETS, _REL, _metric("m", "SUM(orders.amount)"))) + assert not issues.of_type(IssueType.APPROXIMATED) + + +def test_a_split_ratio_comes_back_as_the_metric_it_was_split_from(): + """The split is an implementation detail of the Cube side: the parts are marked + generated, so import skips them and inlines their SQL back through the public + measure's references, recovering the original expression verbatim.""" + expression = "SUM(orders.amount) / COUNT(DISTINCT users.id)" + files, _ = convert_ossie_to_cube( + _ossie(_TWO_DATASETS, _REL, _metric("aov", expression))) + ossie, _ = convert_cube_to_ossie(files) + metrics = model_of(ossie)["metrics"] + assert [m["name"] for m in metrics] == ["aov"] + assert expr_of(metrics[0]) == expression + + +def test_metric_lands_on_the_dataset_its_expression_references(): + files, _ = convert_ossie_to_cube(_ossie( + _TWO_DATASETS, _REL, _metric("users_seen", "COUNT(DISTINCT users.id)"))) + assert "measures" not in _cubes(files)["orders"] + assert _cubes(files, "model/cubes/users.yml")["users"]["measures"][0]["name"] == ( + "users_seen") + + +def test_two_metrics_colliding_on_one_cube_are_rejected(): + metrics = (" metrics:\n" + " - name: Total Amount\n" + " expression:\n dialects:\n - dialect: ANSI_SQL\n" + " expression: SUM(orders.amount)\n" + " - name: total amount\n" + " expression:\n dialects:\n - dialect: ANSI_SQL\n" + " expression: SUM(orders.id)\n") + with pytest.raises(ConversionError, match="two metrics map to measure"): + convert_ossie_to_cube(_ossie(_ORDERS, metrics=metrics)) + + +# --- views ---------------------------------------------------------------------- + +def test_generated_view_is_rooted_at_the_fk_sink(): + files, _ = convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL)) + view = parse(files["model/views/shop.yml"])["views"][0] + assert view["cubes"] == [ + {"join_path": "orders", "includes": "*"}, + {"join_path": "orders.users", "includes": "*"}, + ] + + +def test_ambiguous_base_cube_is_rejected_and_the_hint_resolves_it(): + two_facts = _TWO_DATASETS # no relationships at all + with pytest.raises(ConversionError, match="no relationships"): + convert_ossie_to_cube(_ossie(two_facts)) + files, _ = convert_ossie_to_cube(_ossie(two_facts), base_cube="orders") + assert parse(files["model/views/shop.yml"])["views"][0]["cubes"][0][ + "join_path"] == "orders" + + +def test_unknown_base_cube_is_rejected(): + with pytest.raises(ConversionError, match="not a dataset"): + convert_ossie_to_cube(_ossie(_TWO_DATASETS, _REL), base_cube="nope") + + +def test_synonyms_reach_cube_as_prose_and_are_parked_structurally(): + """Cube has no synonyms field; its docs express them as ai_context prose. The + structured list is parked so the Ossie round trip stays exact.""" + ds = ( + " - name: orders\n" + " source: t\n" + " ai_context:\n" + " instructions: Order facts.\n" + " synonyms:\n" + " - purchases\n" + " - sales\n" + " fields:\n" + " - name: id\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" + " expression: id\n" + " datatype: Integer\n" + ) + files, _ = convert_ossie_to_cube(_ossie(ds)) + meta = _cubes(files)["orders"]["meta"] + assert meta["ai_context"] == "Order facts.\nAlso known as: purchases, sales." + assert meta["ossie"]["ai_context"]["synonyms"] == ["purchases", "sales"] diff --git a/converters/cube/tests/test_roundtrip.py b/converters/cube/tests/test_roundtrip.py new file mode 100644 index 00000000..8416088c --- /dev/null +++ b/converters/cube/tests/test_roundtrip.py @@ -0,0 +1,146 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Fixture-based round-trip tests. + +- Cube -> Ossie -> Cube must be lossless (the stash carries everything). +- Ossie -> Cube -> Ossie must be identical up to the documented normalizations. +- Every Ossie document the importer emits must validate against the core-spec + JSON schema (skipped when jsonschema is not installed). +""" + +import json + +import pytest +from _util import (REPO_ROOT, canon, load_fixture, load_fixture_dir, parse, + parse_files) + +from ossie_cube import convert_cube_to_ossie, convert_ossie_to_cube + +FIXTURES = ["fixtureA_cube", "tpcds_cube"] + + +@pytest.mark.parametrize("fixture", FIXTURES) +def test_cube_roundtrip_is_lossless(fixture): + """Cube -> Ossie -> Cube reproduces the original model, structurally. + + Compared parsed rather than byte-for-byte: YAML comments (including the + license headers on the fixtures) are not part of the data model, and key order + within a mapping is not semantic. + """ + files = load_fixture_dir(fixture) + ossie, _ = convert_cube_to_ossie(files) + files2, _ = convert_ossie_to_cube(ossie) + assert parse_files(files2) == parse_files(files) + + +@pytest.mark.parametrize("cube_dir,ossie_file", [ + ("fixtureA_cube", "fixtureA_ossie.yaml"), + ("tpcds_cube", "tpcds_ossie.yaml"), +]) +def test_import_matches_the_committed_ossie_fixture(cube_dir, ossie_file): + """Whole-document snapshot, so an unintended change anywhere in the output shows + up as a readable diff rather than slipping past field-level assertions. + + Regenerate with `ossie-cube import -i tests/fixtures/` when a change + to the output is intended. + """ + ossie, _ = convert_cube_to_ossie(load_fixture_dir(cube_dir)) + assert canon(parse(ossie)) == canon(parse(load_fixture(ossie_file))) + + +@pytest.mark.parametrize("cube_dir,ossie_file", [ + ("fixtureA_cube", "fixtureA_ossie.yaml"), + ("tpcds_cube", "tpcds_ossie.yaml"), +]) +def test_export_of_the_ossie_fixture_matches_the_cube_fixture(cube_dir, ossie_file): + """The same snapshot in the other direction: the committed Ossie fixture has to + export back to the committed Cube fixture.""" + files, _ = convert_ossie_to_cube(load_fixture(ossie_file)) + assert parse_files(files) == parse_files(load_fixture_dir(cube_dir)) + + +@pytest.mark.parametrize("fixture", FIXTURES) +def test_imported_ossie_validates_against_core_spec_schema(fixture): + jsonschema = pytest.importorskip("jsonschema") + with open(REPO_ROOT / "core-spec" / "osi-schema.json") as fh: + schema = json.load(fh) + ossie, _ = convert_cube_to_ossie(load_fixture_dir(fixture)) + jsonschema.validate(parse(ossie), schema) + + +@pytest.mark.parametrize("fixture", FIXTURES) +def test_ossie_roundtrip_is_lossless(fixture): + """Ossie -> Cube -> Ossie reproduces the model too. + + Cube has a `meta` field at every level, so the export direction parks what + Cube has no slot for under `meta.ossie` instead of dropping it -- which makes + this direction lossless as well, unlike converters whose target format has + nowhere to put the leftovers. + """ + ossie, _ = convert_cube_to_ossie(load_fixture_dir(fixture)) + files, _ = convert_ossie_to_cube(ossie) + ossie2, _ = convert_cube_to_ossie(files) + assert parse(ossie2) == parse(ossie) + + +def test_hand_authored_ossie_gets_a_generated_view(): + """A model with no stashed views is not from Cube, so export has to invent the + view -- the model boundary Cube users work with.""" + ossie = load_fixture("hand_authored_ossie.yaml") + files, _ = convert_ossie_to_cube(ossie) + assert set(files) == { + "model/cubes/orders.yml", "model/cubes/customers.yml", + "model/views/ecommerce.yml", + } + view = parse(files["model/views/ecommerce.yml"])["views"][0] + assert view["name"] == "ecommerce" + assert view["description"] == "Orders and customers" + # Rooted at the FK sink, with the joined cube addressed by its join path. + assert view["cubes"] == [ + {"join_path": "orders", "includes": "*"}, + {"join_path": "orders.customers", "includes": "*"}, + ] + + +def test_hand_authored_ossie_survives_the_round_trip(): + files, _ = convert_ossie_to_cube(load_fixture("hand_authored_ossie.yaml")) + ossie2, _ = convert_cube_to_ossie(files) + model = parse(ossie2)["semantic_model"][0] + assert model["name"] == "ecommerce" + assert model["description"] == "Orders and customers" + assert [d["name"] for d in model["datasets"]] == ["orders", "customers"] + assert model["relationships"][0]["from_columns"] == ["customer_id"] + metrics = {m["name"]: m for m in model["metrics"]} + assert metrics["total_revenue"]["expression"]["dialects"][0]["expression"] == ( + "SUM(orders.amount)") + + +def test_ossie_only_constructs_are_parked_not_dropped(): + """`unique_keys` and a foreign vendor's extensions have no Cube field, so they + ride under `meta.ossie` and come back intact.""" + files, _ = convert_ossie_to_cube(load_fixture("hand_authored_ossie.yaml")) + orders = parse(files["model/cubes/orders.yml"])["cubes"][0] + parked = orders["meta"]["ossie"] + assert parked["unique_keys"] == [["order_number"]] + assert parked["custom_extensions"][0]["vendor_name"] == "SNOWFLAKE" + + ossie2, _ = convert_cube_to_ossie(files) + ds = {d["name"]: d for d in parse(ossie2)["semantic_model"][0]["datasets"]} + assert ds["orders"]["unique_keys"] == [["order_number"]] + vendors = {e["vendor_name"] for e in ds["orders"]["custom_extensions"]} + assert "SNOWFLAKE" in vendors diff --git a/converters/cube/tests/test_roundtrip_properties.py b/converters/cube/tests/test_roundtrip_properties.py new file mode 100644 index 00000000..3137245f --- /dev/null +++ b/converters/cube/tests/test_roundtrip_properties.py @@ -0,0 +1,90 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Property-based round-trip tests over generated Cube models. + +The generators live in `_roundtrip_helpers` and depend only on a tiny +chance/count/pick/text interface, so the same model space is explored whether +Hypothesis is installed or not. Without it, a seeded sweep runs instead -- the +properties are still checked in CI on a Python where hypothesis fails to build. +""" + +import pytest +from _roundtrip_helpers import RandomRnd, build_cube_model, check_model + +try: + from hypothesis import HealthCheck, given, settings + from hypothesis import strategies as st + + HAVE_HYPOTHESIS = True +except ImportError: # pragma: no cover - exercised only without hypothesis + HAVE_HYPOTHESIS = False + + +SEEDS = list(range(60)) + + +@pytest.mark.parametrize("seed", SEEDS) +def test_seeded_models_roundtrip(seed): + """A deterministic sweep, so a failure names a reproducible seed.""" + check_model(build_cube_model(RandomRnd(seed))) + + +if HAVE_HYPOTHESIS: + class _HypothesisRnd: + """The `Rnd` interface backed by a Hypothesis data strategy.""" + + def __init__(self, data): + self.data = data + + def chance(self, p=0.5): + # `st.booleans()` is unweighted, so it would ignore `p` and explore a + # different distribution than RandomRnd -- defeating the point of the + # two drivers sharing one generator. Drawn so the minimal value (0) + # means False, which shrinks toward the smallest model rather than the + # largest. + return self.data.draw( + st.integers(min_value=0, max_value=99)) >= 100 - round(p * 100) + + def count(self, lo, hi): + return self.data.draw(st.integers(min_value=lo, max_value=hi)) + + def pick(self, seq): + return self.data.draw(st.sampled_from(list(seq))) + + def text(self): + # Printable, no leading/trailing whitespace and no newlines, so the + # value survives a YAML dump/load cycle verbatim. Round-tripping + # arbitrary Unicode is a PyYAML property, not a converter one. + # + # Jinja delimiters are excluded because they are out of the + # round-trippable subset by design: the converter treats a file + # containing them as templated and preserves it whole, exactly as + # Cube's own CubeSchemaConverter does. That behavior has its own + # targeted test. + return self.data.draw(st.text( + alphabet=st.characters(min_codepoint=32, max_codepoint=126), + min_size=1, max_size=24, + ).map(str.strip).filter( + lambda s: s and not s.startswith("#") + and not any(t in s for t in ("{{", "}}", "{%", "%}")))) + + @settings(max_examples=150, deadline=None, + suppress_health_check=[HealthCheck.too_slow]) + @given(st.data()) + def test_generated_models_roundtrip(data): + check_model(build_cube_model(_HypothesisRnd(data))) diff --git a/converters/cube/tools/interop_matrix.py b/converters/cube/tools/interop_matrix.py new file mode 100644 index 00000000..89939abd --- /dev/null +++ b/converters/cube/tools/interop_matrix.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +# +# /// script +# requires-python = ">=3.11" +# /// + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Does a converted Cube model actually reach the other spokes? + +Ossie is a hub: `Cube -> Ossie` is only half the point, and a converter can pass its +own round-trip tests while emitting something the next converter chokes on. This +script runs `Cube -> Ossie -> every other spoke` and prints what each one made of it, +so a change can be judged on interop instead of on self-consistency. + + uv run tools/interop_matrix.py # the committed tpcds fixture + uv run tools/interop_matrix.py path/to/cube/model # any Cube model directory + uv run tools/interop_matrix.py --keep # leave the outputs to read + +Columns: + + result OK / EMPTY (exit 0, nothing written) / FAIL / SKIP (deps not installed) + warns lines the spoke wrote to stderr that read as warnings + foreign those warnings that name a `custom_extensions` vendor -- the cost this + converter imposes on every other spoke by stashing, and the number to + watch when deciding whether something belongs in a stash at all + +Each spoke runs in its own `uv` environment, so the first run for a given spoke +resolves its dependencies (`uv sync` there first to keep this fast) -- which leaves a +`uv.lock` and a `.venv` in that converter's directory. Those belong to the converter, +not to this run: check `git status` before committing. The Java converters (polaris, +salesforce) are listed as unsupported rather than skipped silently; they need Maven, +not uv. + +Stdlib only, so it needs no environment of its own. +""" + +import argparse +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +# (directory under converters/, argv to convert Ossie -> spoke, output is a directory) +# +# The invocations differ per spoke because the CLIs do: some take an `export` +# subcommand, some a named direction, snowflake takes none, gooddata ships no CLI at +# all and is driven through its Python API. +SPOKES = [ + ("databricks", ["ossie-databricks", "export"], False), + ("dbt", ["ossie-dbt", "osi-to-msi"], False), + ("gooddata", None, False), # API-only; see _run_gooddata + ("gsf", ["ossie-gsf", "export"], False), + ("honeydew", ["honeydew-osi", "osi-to-honeydew"], True), + ("omni", ["osi-omni", "export"], True), + ("orionbelt", ["ossie-orionbelt", "osi-to-obml"], False), + ("snowflake", ["ossie-snowflake"], False), + ("wisdom", ["ossie-wisdom", "osi-to-wisdom"], False), +] + +# Converters written in Java: a different toolchain, not a missing dependency. +UNSUPPORTED = ["polaris", "salesforce"] + +_WARN_RE = re.compile(r"warn", re.I) +# Python's `warnings.warn` prints the message and then echoes the calling source +# line, which would otherwise count the same warning twice. +_ECHO_RE = re.compile(r"^\s*warnings\.warn\b") +_FOREIGN_RE = re.compile(r"custom_extension|vendor|foreign", re.I) + + +def repo_root(): + for parent in Path(__file__).resolve().parents: + if (parent / "converters").is_dir() and (parent / "core-spec").is_dir(): + return parent + sys.exit("cannot locate the repository root from this script's path") + + +# Resolving a converter's dependencies on a cold cache is the slow part; a spoke that +# has not finished by then is hung rather than working. Without a timeout one such +# spoke takes the whole run down with it and prints nothing. +_TIMEOUT_S = 600 + + +def run(cwd, argv): + """Run `argv` in `cwd`, or return a synthetic failure rather than raising.""" + try: + return subprocess.run(argv, cwd=cwd, capture_output=True, text=True, + timeout=_TIMEOUT_S) + except subprocess.TimeoutExpired: + return subprocess.CompletedProcess( + argv, 1, "", f"timed out after {_TIMEOUT_S}s") + except FileNotFoundError as e: + return subprocess.CompletedProcess(argv, 1, "", f"{argv[0]}: {e.strerror}") + + +def count_warnings(stderr): + """(warnings, of which are about a foreign vendor extension). + + A line count, so a warning whose message wraps counts more than once. It is a + relative measure -- run it before and after a change -- not an exact tally. + """ + warns = [ln for ln in stderr.splitlines() + if _WARN_RE.search(ln) and not _ECHO_RE.match(ln)] + return len(warns), len([ln for ln in warns if _FOREIGN_RE.search(ln)]) + + +def import_issues(stderr): + """The issue types `ossie-cube import` reported, as {type: count}. + + Its own issues do not read as warnings -- they are `[TYPE] element: detail` lines + -- so they are counted from their structure rather than by keyword. + """ + found = {} + for ln in stderr.splitlines(): + m = re.match(r"\s+\[([A-Z_]+)\]", ln) + if m: + found[m.group(1)] = found.get(m.group(1), 0) + 1 + return found + + +# uv's wording for "this converter's environment could not be built", which is not +# the converter rejecting the model. Matching on message text is unavoidable (uv exits +# 1 either way) and will drift, so a message that stops matching shows up as a FAIL +# with the reason in the note column rather than as a silent mislabel. +_ENV_FAILURE_MARKERS = ( + "No solution found", + "no such command", + "Failed to spawn", + "does not exist", +) + + +def _is_environment_failure(stderr): + return any(marker in stderr for marker in _ENV_FAILURE_MARKERS) + + +def produced_output(dest, is_dir): + if not dest.exists(): + return False + return any(dest.rglob("*")) if is_dir else dest.stat().st_size > 0 + + +def _run_gooddata(root, ossie, dest): + """gooddata ships no console script, so drive its API the way its README does.""" + script = ( + "import json, sys, yaml\n" + "from ossie_gooddata import osi_to_gooddata\n" + "from ossie_gooddata.models import gd_model_to_dict\n" + "model = yaml.safe_load(open(sys.argv[1]).read())\n" + "out = gd_model_to_dict(osi_to_gooddata(model))\n" + "open(sys.argv[2], 'w').write(json.dumps(out, indent=2, default=str))\n" + ) + return run(root / "converters/gooddata", + ["uv", "run", "--quiet", "python", "-c", script, + str(ossie), str(dest)]) + + +def cube_to_ossie(root, model_dir, dest): + r = run(root / "converters/cube", + ["uv", "run", "--quiet", "ossie-cube", "import", + "-i", str(model_dir), "-o", str(dest)]) + return r + + +def validate_ossie(root, ossie): + """Run the repo's own validator on the intermediate model. + + A spoke rejecting the model is only interesting once the model is known good, so + this is checked before the matrix rather than left to be inferred from it. + """ + return run(root, ["uv", "run", "--quiet", "validation/validate.py", str(ossie)]) + + +def main(): + root = repo_root() + ap = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument( + "model", nargs="?", + default=str(root / "converters/cube/tests/fixtures/tpcds_cube"), + help="Cube model directory (default: the committed tpcds fixture)") + ap.add_argument("--spokes", help="comma-separated subset to run") + ap.add_argument("--keep", action="store_true", + help="keep the converted outputs and print where they are") + args = ap.parse_args() + + model_dir = Path(args.model).expanduser().resolve() + if not model_dir.exists(): + sys.exit(f"no such Cube model: {model_dir}") + + wanted = None + if args.spokes: + wanted = {s.strip() for s in args.spokes.split(",") if s.strip()} + unknown = wanted - {name for name, _, _ in SPOKES} + if unknown: + sys.exit(f"unknown spoke(s): {', '.join(sorted(unknown))}") + + out = Path(tempfile.mkdtemp(prefix="ossie-interop-")) + try: + ossie = out / "from_cube.yaml" + r = cube_to_ossie(root, model_dir, ossie) + if r.returncode != 0: + print(f"Cube -> Ossie FAILED\n{r.stderr}", file=sys.stderr) + return 1 + + text = ossie.read_text() + reported = import_issues(r.stderr) + print(f"model: {model_dir}") + print(f"Ossie: {len(text.splitlines())} lines, " + f"{text.count('vendor_name: CUBE')} CUBE stash entries") + if reported: + print("issues: " + ", ".join( + f"{n}x {kind}" for kind, n in sorted(reported.items()))) + + v = validate_ossie(root, ossie) + print(f"spec: {'valid' if v.returncode == 0 else 'INVALID'} " + f"(validation/validate.py)") + if v.returncode != 0: + print(v.stdout.strip() or v.stderr.strip()) + + print() + print(f"{'spoke':<12} {'result':<7} {'warns':>5} {'foreign':>8} note") + print("-" * 76) + + failures = 0 + for name, argv, is_dir in SPOKES: + if wanted and name not in wanted: + continue + dest = out / (name if is_dir else f"{name}.out") + if argv is None: + r = _run_gooddata(root, ossie, dest) + else: + r = run(root / "converters" / name, + ["uv", "run", "--quiet", *argv, + "-i", str(ossie), "-o", str(dest)]) + + warns, foreign = count_warnings(r.stderr) + note = "" + if r.returncode != 0: + tail = (r.stderr.strip().splitlines() or [""])[-1] + result = "SKIP" if _is_environment_failure(r.stderr) else "FAIL" + note = tail[:40] + failures += result == "FAIL" + else: + result = "OK" if produced_output(dest, is_dir) else "EMPTY" + print(f"{name:<12} {result:<7} {warns:>5} {foreign:>8} {note}") + + if not wanted: + for name in UNSUPPORTED: + print(f"{name:<12} {'--':<7} {'':>5} {'':>8} " + "Java converter, needs Maven") + + if args.keep: + print(f"\noutputs: {out}") + return 1 if failures else 0 + finally: + if not args.keep: + shutil.rmtree(out, ignore_errors=True) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/converters/cube/uv.lock b/converters/cube/uv.lock new file mode 100644 index 00000000..05578dff --- /dev/null +++ b/converters/cube/uv.lock @@ -0,0 +1,413 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "apache-ossie-cube" +version = "0.2.0.dev0" +source = { editable = "." } +dependencies = [ + { name = "pyyaml" }, + { name = "sqlglot" }, +] + +[package.dev-dependencies] +dev = [ + { name = "hypothesis" }, + { name = "jsonschema" }, + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "pyyaml", specifier = ">=6.0" }, + { name = "sqlglot", specifier = ">=20.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "hypothesis", specifier = ">=6.0" }, + { name = "jsonschema", specifier = ">=4.0" }, + { name = "pytest", specifier = ">=8.0" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "hypothesis" +version = "6.163.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/61/08/4cbfa0327e9df00f57fc67f91847add2f0dd6c23408935273b095ee9a9f1/hypothesis-6.163.0.tar.gz", hash = "sha256:520480d4bd3a17557616c25923640953e360332c89d012fffcebd69857e674a9", size = 490145, upload-time = "2026-07-28T07:16:46.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/b1/3eea7a422de342bd0095a9672c3e03fe0466e4561a55499a1e01b4a9f098/hypothesis-6.163.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:331906cb029b6b360b8ebac3ec00c3cfa720037fe2efb294a503a1979c9a9a8f", size = 769898, upload-time = "2026-07-28T07:15:19.323Z" }, + { url = "https://files.pythonhosted.org/packages/bc/24/45b5c948c76c16ecf1e4ad1ca4a4a3fce55b3317ee172bc8230ff2956ae1/hypothesis-6.163.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:b4ad2134405d5345434c22dea96bbc12c85abcfc3c253a8063dbc9ff01164555", size = 765438, upload-time = "2026-07-28T07:16:04.69Z" }, + { url = "https://files.pythonhosted.org/packages/0f/72/7725039a75b3679dc445a169026b11860a0e67a600c4ffed49a42040a000/hypothesis-6.163.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50073f8e63c1e7d3403899755657a990d8bba7b5b5bff66b1c56796d4969bb28", size = 1094699, upload-time = "2026-07-28T07:15:56.668Z" }, + { url = "https://files.pythonhosted.org/packages/dd/fa/bcfa3879f303a302ec6e5f4d35b583924867a0821b42fa275f440f3b8dab/hypothesis-6.163.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8c5d1e6bad47edf6fb1d7406cf6d67314ac08325c63a49550d782a4596ea302b", size = 1123315, upload-time = "2026-07-28T07:16:40.381Z" }, + { url = "https://files.pythonhosted.org/packages/5f/7f/e7fe2f0658db5182bb1d4b266d17f8f81cf3db82eeba27677ddfea13ac09/hypothesis-6.163.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c0ec3b709508ccd835d8ded1db025b7800618f2289a22a6bfd4927da5f4eb33c", size = 1144220, upload-time = "2026-07-28T07:15:20.617Z" }, + { url = "https://files.pythonhosted.org/packages/90/14/c26f93a4693bfd83d7ba14b7043d986458026f6635d1b157bc2f1c56a59e/hypothesis-6.163.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:7cb3d927360fe73f9a06d646e6082237142ee39c24679c7133d22bf06dd03b45", size = 1099527, upload-time = "2026-07-28T07:16:17.955Z" }, + { url = "https://files.pythonhosted.org/packages/ea/f5/4c1dde28d2e18e169dd6c471ef4bcf12abf3c6a3f4a1744bdabdbdf411ba/hypothesis-6.163.0-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3f3cceb4720a39127622fbf3bcebe1775b894372c53b5edddfdef10bbdeef9ec", size = 1136272, upload-time = "2026-07-28T07:16:38.182Z" }, + { url = "https://files.pythonhosted.org/packages/3a/5b/54153509ed42cc17e1b65efe34a0c781f42fb04c902dc5f25486c537ec88/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:59f5fdb8addb44c17520a60d50542d9db6ceba577bbf54efefa9c10ee20be140", size = 1268518, upload-time = "2026-07-28T07:15:42.991Z" }, + { url = "https://files.pythonhosted.org/packages/81/86/f86f0d15d91b9cbff3546c1b8891950b2b08c0527e7494133fb2180e1b8b/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:f1fe222f50a1898e87a1e7323ab35f9e956278efabe4dd55a1342808206d05ad", size = 1396357, upload-time = "2026-07-28T07:15:26.696Z" }, + { url = "https://files.pythonhosted.org/packages/46/3a/c096b6b272f15e17e8e65c6ccfb2e1d167456ff5bbd9db33dca3185199f6/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:56ed585baab75cb98462c57ca88bbdc6a9d935a14118dd572fb476c3ecec2a06", size = 1269128, upload-time = "2026-07-28T07:16:11.902Z" }, + { url = "https://files.pythonhosted.org/packages/bb/9f/0807445874b3083a22c9a14a0ffc31bd0060ef3b408ce4cb31c20779cdf8/hypothesis-6.163.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:2849c23b2e0fe2eef4c1ec336b01eac7ad7397c49fca43c264f59ec1e6046eac", size = 1311189, upload-time = "2026-07-28T07:16:08.545Z" }, + { url = "https://files.pythonhosted.org/packages/d6/de/3319aa8fcffd1641defc1c5eea1ad646a33d1b62528ef487a89752bc8079/hypothesis-6.163.0-cp310-abi3-win32.whl", hash = "sha256:b2ddcdaf6691101e06dc4a5add7b8c8fdf1e68daba599255a281f3f3550d3331", size = 655743, upload-time = "2026-07-28T07:16:30.966Z" }, + { url = "https://files.pythonhosted.org/packages/8e/48/36bc72910451e6e88b75e59a6ddbf0db34ff61a3b11c9439801dfd5fec20/hypothesis-6.163.0-cp310-abi3-win_amd64.whl", hash = "sha256:4ab0dadc09c537d4ac57e564039dfe7daf09c98375306d54bfc0fd6c218efcca", size = 661902, upload-time = "2026-07-28T07:16:34.407Z" }, + { url = "https://files.pythonhosted.org/packages/63/80/796ac61dddb3ede550ea127e28e027a57a4ea481581c0bb27701fefd655a/hypothesis-6.163.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:213527755f0fc2b1f3721e73fd60023e2752a48f914e3e2df8d35111956ae5c8", size = 770366, upload-time = "2026-07-28T07:15:22.003Z" }, + { url = "https://files.pythonhosted.org/packages/eb/87/53924f322922bcfc05e40c79d432978333827b335fa9efa5fd1e8cbf3c90/hypothesis-6.163.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ca1b48bde68c528a79dec2a2859e05035802e5b1c9c3579f388c9de6ed6d0148", size = 766150, upload-time = "2026-07-28T07:15:16.151Z" }, + { url = "https://files.pythonhosted.org/packages/f3/fb/b62480e6510052139d7b2a0219d4ae8bd52ccad0ed74db15dd61330a6962/hypothesis-6.163.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a3db868a943c814cc557104712d43bf609adfe5ea9f708f38377d366b4855f8", size = 1095046, upload-time = "2026-07-28T07:15:45.796Z" }, + { url = "https://files.pythonhosted.org/packages/4d/d7/fa8aac7b47f41d0fe30fa270c5026f37a0c0c60d7b9a1a93dcc8fcfc50ce/hypothesis-6.163.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4159a1c2560e10de51b1c14956e277eb1b37526c9abef9e87c1e531760486448", size = 1144516, upload-time = "2026-07-28T07:16:32.662Z" }, + { url = "https://files.pythonhosted.org/packages/4f/26/c543c76d8a8b8f58f2d7adf0cb42e4928be3464e95f4fa9d7221b42ea9ce/hypothesis-6.163.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ffdda3006a383a48f71a23b4f2b3fae3fe1b09af67925d885985f7ec34d66bcb", size = 1268886, upload-time = "2026-07-28T07:16:36.102Z" }, + { url = "https://files.pythonhosted.org/packages/d9/54/3613ef980cfa60f5c6bfbc533989d6c67b6b5f4e9e638fc6d010a5dd852f/hypothesis-6.163.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3b6cee2afe6c67b31a4a64b63a876e0b020befdc61daabea80f7a0e14f19203a", size = 1311472, upload-time = "2026-07-28T07:16:13.707Z" }, + { url = "https://files.pythonhosted.org/packages/af/0c/cbaebc49fd5807a4b4286dea6e60d5330951e43115d002371bdfc99e70f8/hypothesis-6.163.0-cp311-cp311-win_amd64.whl", hash = "sha256:0a933aca9ebf9daf951d07cf01200c94c321b6ee0b42cc7b67675c9686d914c2", size = 661580, upload-time = "2026-07-28T07:15:32.939Z" }, + { url = "https://files.pythonhosted.org/packages/8f/2c/74e989557efc429b28282cbe754c17fc74495467a72a1c94b5eb734fe374/hypothesis-6.163.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a57352efa938889ea9992667a5014c0fc870d03945de71918574d1cf28276378", size = 771445, upload-time = "2026-07-28T07:16:44.2Z" }, + { url = "https://files.pythonhosted.org/packages/fd/08/3ed2089d8cbeae125ea92879e82b99ea3a8b676c837710019249ccab379f/hypothesis-6.163.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0a0c396244c13805edcb73ff467c4c8178ccefc41c4ef5ed00a68e612fd773e9", size = 763070, upload-time = "2026-07-28T07:15:34.318Z" }, + { url = "https://files.pythonhosted.org/packages/e5/1e/4e85a11f15e8ebd730f3b3e4a5d83653f7da482a4e81fa36958951d89b4a/hypothesis-6.163.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28a6cc1c25a6cc9b6ec079eaabd32ac769994831ecddd57123ce43c9056dcf34", size = 1093500, upload-time = "2026-07-28T07:15:49.539Z" }, + { url = "https://files.pythonhosted.org/packages/1f/29/c4790d2a5e6f48e6be5f868124a0d3c7e1d0102e2ca50503a0718e51289c/hypothesis-6.163.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bd312b15044b1c1a0920a5827a830559b2d1fa380851cedf509f8b835309c5b9", size = 1143541, upload-time = "2026-07-28T07:15:44.393Z" }, + { url = "https://files.pythonhosted.org/packages/da/a8/01b72694e758449e9b530b72ecaf5f3625c80a3968de0d12ee6e784de22e/hypothesis-6.163.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b839dfd1342bb50570cb0c66b80322307cdb468abf14faf5df4dab022bc1b9ce", size = 1266326, upload-time = "2026-07-28T07:15:12.604Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a3/b3997add991e2fc6123b3c8dde3631f9f633db67ba23f5b2567736b50b9e/hypothesis-6.163.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c4f5be1482189c7b0a1dcac269fffe97a7d18cc04ac9a9a4d6613212dd87f38b", size = 1310536, upload-time = "2026-07-28T07:15:17.798Z" }, + { url = "https://files.pythonhosted.org/packages/5d/fe/36f576185d63ee0b4d94ed9564415de08baed4937e785b3695c8dd665c9c/hypothesis-6.163.0-cp312-cp312-win_amd64.whl", hash = "sha256:7ca7b20bf38d51e15f7808b0239791c4792b1709ce0c63093acaff56a09c31e6", size = 659021, upload-time = "2026-07-28T07:15:53.143Z" }, + { url = "https://files.pythonhosted.org/packages/58/69/c474a3fa1c33d9a6e059e820221275d9c60eb10085f6164212c291856157/hypothesis-6.163.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:a16ebce774755a7a652bd44c62101dc914372ed1a98935969624848c9627b4a4", size = 771335, upload-time = "2026-07-28T07:15:14.988Z" }, + { url = "https://files.pythonhosted.org/packages/54/27/8951688de58314780ba0af5bf2675709009dcc1fb568d244f2a03c4de8e9/hypothesis-6.163.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:40dfab6fe6a02a80abef81aebf88e53cd529e3f2f6ba3486b674a67b1f4a3512", size = 763020, upload-time = "2026-07-28T07:15:25.218Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d7/9eb7451400507f4b5c972c81c2bc57d0009597ea37295519a9645963a50e/hypothesis-6.163.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f28ad27193c1fbcfb52ef2ee63d2b721563525089e80962b4268b306dac45507", size = 1093415, upload-time = "2026-07-28T07:16:27.408Z" }, + { url = "https://files.pythonhosted.org/packages/24/cc/c12c780676c7a4a4051d05e7b278a94bf1bb496ef31882881160f16866c9/hypothesis-6.163.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8aac96db8a6c7ee43aba2ee0d3c43893da1fb7c38ed54790c1be2b6d8fd87b96", size = 1143356, upload-time = "2026-07-28T07:16:01.571Z" }, + { url = "https://files.pythonhosted.org/packages/4b/c1/61c5ebdb77a3f259803e60a12f56de35f8b6d641a6219f798dcfa69dece3/hypothesis-6.163.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9d23f0f3a14bb6e6f99c793d340196dba4af95ba25bfcab624d1794f540f5e27", size = 1266375, upload-time = "2026-07-28T07:15:11.353Z" }, + { url = "https://files.pythonhosted.org/packages/a2/53/f3a89b4d21d89098d1dec749632aa0fece04f5d17a9e7e91c7f10596d55a/hypothesis-6.163.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b123b4995a7612f1130e2b2362c9a5d0568df887bf7e7bdb45c23af8cd5423c9", size = 1310257, upload-time = "2026-07-28T07:16:42.213Z" }, + { url = "https://files.pythonhosted.org/packages/06/e7/88e71c4cec0df68aa7a2fb251083c544f30697bd90fb4b1ed19de493ab81/hypothesis-6.163.0-cp313-cp313-win_amd64.whl", hash = "sha256:b268211e625cd550e361fc387bf1db5deb1e9cae0ce4041116f0a0aafeef7c06", size = 658983, upload-time = "2026-07-28T07:16:10.289Z" }, + { url = "https://files.pythonhosted.org/packages/71/df/e3b2f0419cebcc86bba96a357d7ef37790538ed6b09f3183ae01f1fc3d23/hypothesis-6.163.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9105c66ea8dbc108adc42058bb7b65bd953f53ee178bf63bf9ebb0cded6c8c96", size = 771564, upload-time = "2026-07-28T07:15:28.017Z" }, + { url = "https://files.pythonhosted.org/packages/f1/f8/a6f75e61ecd983029f8463bf007498155c4ed33114e6931e7aa3fbaf651e/hypothesis-6.163.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e165f6cc2075059b7c95dac1612bfb25494f72d90f56880e84c288b089f8a896", size = 763164, upload-time = "2026-07-28T07:15:29.973Z" }, + { url = "https://files.pythonhosted.org/packages/5e/07/5193812567f6ca46c1f0cd02dabfd47c85d7c9e3287b8a870fc8150ee462/hypothesis-6.163.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0cba5202f74e7e4cdb676d86f26e8cc1b4fdc88f7f58ba73c8ac45b6b22f3070", size = 1093916, upload-time = "2026-07-28T07:16:25.626Z" }, + { url = "https://files.pythonhosted.org/packages/e4/0c/9d61f0e306499d734dc63ebe374b01c5f50be7072fd5e76eed25cc0b86ac/hypothesis-6.163.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7f706df6839dcc53f20833f2933cbcd126fd2fdee7c312e053de49df4b64e44", size = 1143547, upload-time = "2026-07-28T07:15:07.311Z" }, + { url = "https://files.pythonhosted.org/packages/60/b4/2a9eb04c9847ddf7e6b30bba8bc27b7efe5511d368335b6a41657b3f02dd/hypothesis-6.163.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:487ab8ec2f01a225d6a1e2ceadc5290cde2c691952bd2e7f76199cf82e06fb25", size = 1266755, upload-time = "2026-07-28T07:15:41.457Z" }, + { url = "https://files.pythonhosted.org/packages/d2/f3/8d1903fbfcbf48b90bb5590826821afc9b4fc494391ab1fdfab9df23e928/hypothesis-6.163.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6ae63dec6d1d467b7f4737455f81a7a82f14a41c14510937fcfbc726a085b5f8", size = 1310560, upload-time = "2026-07-28T07:15:58.419Z" }, + { url = "https://files.pythonhosted.org/packages/47/2b/1a8c0457b44775d0aad369f21cbae8026b772a71aa917d1b956b7349b2a4/hypothesis-6.163.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:31dc46c48aa53c3ec92d03120978ca7f19b9cf96d195ed3fc93503f1433c94a6", size = 603067, upload-time = "2026-07-28T07:16:06.697Z" }, + { url = "https://files.pythonhosted.org/packages/10/56/a9cd947064043035457dec0124ac2437ca72acf358b30cf203a229ad831b/hypothesis-6.163.0-cp314-cp314-win_amd64.whl", hash = "sha256:320b076bf6436f971f1c73ee651e60001226d1b4e341f2c4a1ca87248261ca03", size = 658931, upload-time = "2026-07-28T07:15:37.174Z" }, + { url = "https://files.pythonhosted.org/packages/77/37/2d16317fda0ecd915cb094be9a9e8911106e693ed03a4d680de314711854/hypothesis-6.163.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ab34c61d9249f1a8129cb4276062c04e3e47b5be8de6446e7c7fe11362d6fe43", size = 770142, upload-time = "2026-07-28T07:16:21.827Z" }, + { url = "https://files.pythonhosted.org/packages/3f/65/d80a9bfb7868f2c6c072a684993548391c56e0afa1972f79ee1fe168d91b/hypothesis-6.163.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f2f1b67a48da86d3e41c9445367b49a49f7efdb60fc8b5e3593f05e6afb2efbe", size = 761694, upload-time = "2026-07-28T07:15:31.36Z" }, + { url = "https://files.pythonhosted.org/packages/a7/01/c1f2515c638d2637300bb2fd6af129319eae75cdf2ed7804644535f64fb2/hypothesis-6.163.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c084749c115ea7918cf7efa144682783da17eec70d1276689182b871126e715", size = 1092511, upload-time = "2026-07-28T07:15:51.503Z" }, + { url = "https://files.pythonhosted.org/packages/cd/f8/b3cd946308b5a09e52b075792610da310c0b7972bb1e8aa673400b86540c/hypothesis-6.163.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e72e8d5818e5ef8cd6a2191c386e3fd1a6d9e3739cf97289b4d9b5dbc8e38d", size = 1142425, upload-time = "2026-07-28T07:16:15.626Z" }, + { url = "https://files.pythonhosted.org/packages/85/96/b122859e6f7335b54aff759f573a813158d2249488450a75e76462ddaf61/hypothesis-6.163.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5a3ac6c62d49f7fe518dfe7fa924fa03aac839993702207802b0e45f9e1b0dab", size = 1264946, upload-time = "2026-07-28T07:15:23.848Z" }, + { url = "https://files.pythonhosted.org/packages/ba/2e/e0226e8c904b8b4788eb52dacd303c91acbd260904abf64e8f9bad03c88a/hypothesis-6.163.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e568a3d766b7ba8df00e0c33efc4c6530cde14fbc72daabe4824eed211ed7596", size = 1309320, upload-time = "2026-07-28T07:16:29.218Z" }, + { url = "https://files.pythonhosted.org/packages/8c/48/e8bd29fed17c9608524b6a39db4a27b6eec7ce85bda78bba3ee0deebd80e/hypothesis-6.163.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8f22fb8218ba6a452bf9000fc656e1ed57625d17cc8a3871a0fcea3b1b69ebf", size = 659064, upload-time = "2026-07-28T07:16:00.075Z" }, + { url = "https://files.pythonhosted.org/packages/39/db/d4e877b8639bbebedeff4a0511f5fc459c06a31d79a79bf75584eddda8da/hypothesis-6.163.0-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:002a9709345892279fb0e81b5a05b72d08cfe81f937339827be0d588607ca9b0", size = 771252, upload-time = "2026-07-28T07:15:08.694Z" }, + { url = "https://files.pythonhosted.org/packages/ea/28/3492490e997e5c8c9244a56728a623ca817577af3462fca5d1def060a066/hypothesis-6.163.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:52f16840add2eb02c2416f3b83cec4f527b6c19699f2d31eff4859233c715526", size = 767161, upload-time = "2026-07-28T07:15:54.972Z" }, + { url = "https://files.pythonhosted.org/packages/51/73/37a4d4a6f3f0789fb2fddc851da957075fbe223706d1148ccc4dda825171/hypothesis-6.163.0-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:34fc895691a2420595506eb17f3a104f2fa9039f013c0770a6cc2743ccaf6fed", size = 1096016, upload-time = "2026-07-28T07:15:09.821Z" }, + { url = "https://files.pythonhosted.org/packages/48/46/20bc7801f8b539334dc0439c28753632a078c96d6732eccf1bd4880644d2/hypothesis-6.163.0-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ef8954e37c80e0c46e6161eef1c72c71059b95250e620a77bd646f6c7a52a2d", size = 1145797, upload-time = "2026-07-28T07:15:39.864Z" }, + { url = "https://files.pythonhosted.org/packages/94/c2/4d5c8feb78ed86e698d4e0665d3179eb657ae66d3606ffe096d8055aaa82/hypothesis-6.163.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:d0838a28e9943d5b834ebae59b02adda76e2cd1e65caa808104c72102052057d", size = 662696, upload-time = "2026-07-28T07:15:38.519Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "packaging" +version = "26.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d7/f1/e7a6dd94a8d4a5626c03e4e99c87f241ba9e350cd9e6d75123f992427270/packaging-26.2.tar.gz", hash = "sha256:ff452ff5a3e828ce110190feff1178bb1f2ea2281fa2075aadb987c2fb221661", size = 228134, upload-time = "2026-04-24T20:15:23.917Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/b2/87e62e8c3e2f4b32e5fe99e0b86d576da1312593b39f47d8ceef365e95ed/packaging-26.2-py3-none-any.whl", hash = "sha256:5fc45236b9446107ff2415ce77c807cee2862cb6fac22b8a73826d0693b0980e", size = 100195, upload-time = "2026-04-24T20:15:22.081Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.20.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/2a/9618a122aeb2a169a28b03889a2995fe297588964333d4a7d67bdf46e147/rpds_py-2026.6.3.tar.gz", hash = "sha256:1cebd1337c242e4ec2293e541f712b2da849b29f48f0c293684b71c0632625d4", size = 64051, upload-time = "2026-06-30T07:17:53.009Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/1f/a2dca5ffdbf1d475ffc4e80e4d5d720ff3a00f691795910116960ee12511/rpds_py-2026.6.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:7b689145a1485c335569bd056464f3243a29af7ed3871c7be31ad624ba239bc7", size = 342174, upload-time = "2026-06-30T07:14:54.821Z" }, + { url = "https://files.pythonhosted.org/packages/4d/dc/323d08583c0832911768663d1944f0107fcd4088704858d84b5e06d105a0/rpds_py-2026.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:db08f45aecde626498fb3df07bcf6d2ec040af42e859a4f5040d79c200342911", size = 345513, upload-time = "2026-06-30T07:14:56.515Z" }, + { url = "https://files.pythonhosted.org/packages/0b/2a/e31989834d18d2f26ec1d2774c5b1eb3331df4ea8ada525175294c94b48a/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:acc992ab27b15f852c76755eb2ab7dce86585ddadba6fa5946e58556088845b4", size = 373783, upload-time = "2026-06-30T07:14:57.736Z" }, + { url = "https://files.pythonhosted.org/packages/87/fe/e80107ee3639585c9941c17d6a42cd65325022f656c023191fce78c324c8/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f88d653e7b3b779d71ae7454e20dcc9b6bae903f33c269db9f2be41bda3f261", size = 378316, upload-time = "2026-06-30T07:14:59.077Z" }, + { url = "https://files.pythonhosted.org/packages/22/6f/81e3adf81acfb6fa694de2a6e4e7d8863121e3e0799e0a7725e6cf5679c4/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e52655eaf81e32593abedaa4bfe33170c8cfedf3365ed9be6e11e07f148f0278", size = 499423, upload-time = "2026-06-30T07:15:00.488Z" }, + { url = "https://files.pythonhosted.org/packages/2d/9a/41263969df0ce3d9af2a96d5005a288200af1989aed3354bfceb5fc0b21f/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dfcc8b909769d19db55c7cc9541eb64b9b774b1057ffffb4f1048070475bb9f9", size = 386077, upload-time = "2026-06-30T07:15:01.911Z" }, + { url = "https://files.pythonhosted.org/packages/5e/19/7e98f468bd50346faff5b10e5297374b443bfdddacc8e9fbc65984539597/rpds_py-2026.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c1255b302953c86a486b81d330d5ee1d5bd937691ce271b6be0ef0e299eaab7", size = 371315, upload-time = "2026-06-30T07:15:03.317Z" }, + { url = "https://files.pythonhosted.org/packages/99/3c/2b973b4d371906a134b03decfea7f5d9835a2c6d263454392e15b64b5b18/rpds_py-2026.6.3-cp311-cp311-manylinux_2_31_riscv64.whl", hash = "sha256:8d2294a31386bfa251d8c8a39472beee17db67d4f1a6eabea665d35c9a4461c3", size = 383502, upload-time = "2026-06-30T07:15:04.627Z" }, + { url = "https://files.pythonhosted.org/packages/98/2a/12e2799500af0a307bca76b63361c51f9fe479223561489c29eea1f2ee41/rpds_py-2026.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:f8f23ead891a3b762f35ab3b04623da7056545b48aa60d59957e6789914545da", size = 402673, upload-time = "2026-06-30T07:15:05.856Z" }, + { url = "https://files.pythonhosted.org/packages/2d/e3/21e5872d165fe08be4f229e3d5ee9d90019c0bf0e5538de60dbd54009450/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:421aba32367055614287a4292b6a17f1939c9452299f7a0209c117e990b646d4", size = 549964, upload-time = "2026-06-30T07:15:07.159Z" }, + { url = "https://files.pythonhosted.org/packages/1a/d0/5ee0fe36844297de8123bee27bc12078c1a7416ad9f1b8a8ca18d6b0c0ac/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:1e5822dfc2f0d4ab7e745eaa6d85945069329beeccef965af3f3bb26058fcab6", size = 615446, upload-time = "2026-06-30T07:15:08.531Z" }, + { url = "https://files.pythonhosted.org/packages/b1/80/1ea5873cb683f2fbe5f21b23ea1f6d179ead19f3c5b249b7eb5dca568ef2/rpds_py-2026.6.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83e35b57523816c8613fd0776b40cd8bb9f596b37ddd2692eb4a6bb5ab2f8c93", size = 576975, upload-time = "2026-06-30T07:15:09.97Z" }, + { url = "https://files.pythonhosted.org/packages/c9/e1/90ef639217a5ddb15b7f4f61b1c33911fd044ad03c311bafdd2bcab85582/rpds_py-2026.6.3-cp311-cp311-win32.whl", hash = "sha256:de3eceba0b683bcbb1ab93da016d0270df1f9ae7be716b40214c5dafac6ea45a", size = 204453, upload-time = "2026-06-30T07:15:11.324Z" }, + { url = "https://files.pythonhosted.org/packages/f2/b7/b7a1695d7af36f521fb11e80d6d3adbd744f73b921859bd3c2a2c0dc706f/rpds_py-2026.6.3-cp311-cp311-win_amd64.whl", hash = "sha256:2c54a076ca4d370980ab57bc0e31df57bbe8d41340436a90ef8b1219a3cbb127", size = 223219, upload-time = "2026-06-30T07:15:12.476Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a2/145afacf796e4506062825941176ad9445c2dcf2b3b6a1f13d3030a15e19/rpds_py-2026.6.3-cp311-cp311-win_arm64.whl", hash = "sha256:168c733a7112e071bb7a66460e667edfcff06c017a3c523f7a8a8e08d0140804", size = 219137, upload-time = "2026-06-30T07:15:13.631Z" }, + { url = "https://files.pythonhosted.org/packages/5c/be/2e8974163072e7bab7df1a5acd54c4498e75e35d6d18b864d3a9d5dadc92/rpds_py-2026.6.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:a0811d33247c3d6128a3001d763f2aa056bb3425204335400ac54f89eec3a0d0", size = 343691, upload-time = "2026-06-30T07:15:14.96Z" }, + { url = "https://files.pythonhosted.org/packages/a4/73/319dfa745dd668efe89309141ded489126461fcecd2b8f3a3cda185129b6/rpds_py-2026.6.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:538949e262e46caa31ac01bdb3c1e8f642622922cacbabbae6a8445d9dc33eaf", size = 338542, upload-time = "2026-06-30T07:15:16.267Z" }, + { url = "https://files.pythonhosted.org/packages/21/63/4239893be1c4d09b709b1a8f6be4188f0870084ff547f46606b8a75f1b03/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:55927d532399c2c646100ff7feb48eaa940ad70f42cd68e1328f3ded9f81ca24", size = 368180, upload-time = "2026-06-30T07:15:17.62Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ca/9c5de382225234ceb37b1844ebdb140db12b2a278bb9efe2fcd19f6c82ce/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f56f1695bc5c0871cbc33dc0130fcf503aab0c57dcc5a6700a4f49eba4f2652e", size = 375067, upload-time = "2026-06-30T07:15:18.952Z" }, + { url = "https://files.pythonhosted.org/packages/87/dc/863f69d1bf04ade34b7fe0d59b9fdf6f0135fe2d7cbca74f1d665589559d/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:270b293dae9058fc9fcedab50f13cebf46fb8ed1d1d54e0521a9da5d6b211975", size = 490509, upload-time = "2026-06-30T07:15:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ef/eac16a12048b45ec7c7fa94f2be3438a5f26bf9cc8580b18a1cfd609b7f6/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:127565fead0a10943b282957bd5447804ff3160ad79f2ad2635e6d249e380680", size = 382754, upload-time = "2026-06-30T07:15:21.831Z" }, + { url = "https://files.pythonhosted.org/packages/04/8f/d2f3f532616be4d06c316ef119683e832bd3d41e112bf3a88f4151c95b17/rpds_py-2026.6.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ecabd69db66de867690f9797f2f8fa27ba501bbc24540cbdbdc649cd15888ba6", size = 366189, upload-time = "2026-06-30T07:15:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/e3/29/41a7b0e98a4b44cd676ab7598419623373eb43b20be68c084935c1a8cf88/rpds_py-2026.6.3-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:58eadac9cd119677b60e1cf8ac4052f35949d71b8a9e5556efccbe82533cf22a", size = 377750, upload-time = "2026-06-30T07:15:24.659Z" }, + { url = "https://files.pythonhosted.org/packages/2e/05/ecda0bec46f9a1565090bcdc941d023f6a25aff85fda28f89f8d19878152/rpds_py-2026.6.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7491ee23305ac3eb59e492b6945881f5cd77a6f731061a3f25b77fd40f9e99a4", size = 395576, upload-time = "2026-06-30T07:15:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/68/a8/6ed52f03ee6cb854ce78785cc9a9a672eb880e83fd7224d471f667d151f1/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:2c99f7e8ccb3dd6e3e4bfeac657a7b208c9bac8075f4b078c02d7404c34107fa", size = 543807, upload-time = "2026-06-30T07:15:27.356Z" }, + { url = "https://files.pythonhosted.org/packages/8f/d6/156c0d3eea27ba09b92562ba2364ba124c0a061b199e17eac637cd25a5e2/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:62698275682bf121181861295c9181e789030a2d516071f5b8f3c23c170cd0fc", size = 611187, upload-time = "2026-06-30T07:15:28.931Z" }, + { url = "https://files.pythonhosted.org/packages/f1/31/774212ed989c62f7f310220089f9b0a3fb8f40f5443d1727abd5d9f52bc9/rpds_py-2026.6.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a214c993455f99a89aaeadc9b21241900037adc9d97203e374d75513c5911822", size = 573030, upload-time = "2026-06-30T07:15:30.553Z" }, + { url = "https://files.pythonhosted.org/packages/c9/50/22f73127a41f1ce4f87fe39aadfb9a126345801c274aa93ae88456249327/rpds_py-2026.6.3-cp312-cp312-win32.whl", hash = "sha256:501f9f04a588d6a09179368c57071301445191767c64e4b52a6aa9871f1ef5ed", size = 202185, upload-time = "2026-06-30T07:15:32.027Z" }, + { url = "https://files.pythonhosted.org/packages/04/3a/f0ee4d4dde9d3b69dedf1b5f74e7a40017046d55052d173e418c6a94f960/rpds_py-2026.6.3-cp312-cp312-win_amd64.whl", hash = "sha256:2c958bf94822e9290a40aaf2a822d4bc5c88099093e3948ad6c571eca9272e5f", size = 220394, upload-time = "2026-06-30T07:15:33.359Z" }, + { url = "https://files.pythonhosted.org/packages/f3/83/3382fe37f809b59f02aac04dbc4e765b480b46ee0227ed516e3bdc4d3dfc/rpds_py-2026.6.3-cp312-cp312-win_arm64.whl", hash = "sha256:22bffe6042b9bcb0822bcd1955ec00e245daf17b4344e4ed8e9551b976b63e96", size = 215753, upload-time = "2026-06-30T07:15:34.778Z" }, + { url = "https://files.pythonhosted.org/packages/a4/9e/b818ee580026ec578138e961027a68820c40afeb1ec8f6819b54fb99e196/rpds_py-2026.6.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:3cfe765c1da0072636ca06628261e0ea05688e160d5c8a03e0217c3854037223", size = 343012, upload-time = "2026-06-30T07:15:36.005Z" }, + { url = "https://files.pythonhosted.org/packages/f3/6b/686d9dc4359a8f163cfbbf89ee0b4e586431de22fe8248edb63a8cf50d49/rpds_py-2026.6.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f4d78253f6996be4901669ad25319f842f740eccf4d58e3c7f3dd39e6dde1d8f", size = 338203, upload-time = "2026-06-30T07:15:37.462Z" }, + { url = "https://files.pythonhosted.org/packages/9e/9b/069aa329940f8207615e091f5eedbbd40e1e15eac68a0790fd05ccdf796c/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:54f45a148e28767bf343d33a684693c70e451c6f4c0e9904709a723fafbdfc1f", size = 367984, upload-time = "2026-06-30T07:15:39.008Z" }, + { url = "https://files.pythonhosted.org/packages/14/db/34c203e4becff3703e4d3bc121842c00b8689197f398161203a880052f4e/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:842e7b070435622248c7a2c44ae53fa1440e073cc3023bc919fed570884097a7", size = 374815, upload-time = "2026-06-30T07:15:40.253Z" }, + { url = "https://files.pythonhosted.org/packages/ee/7d/8071067d2cc453d916ad836e828c943f575e8a44612537759002a1e07381/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8020133a74bd81b4572dd8e4be028a6b1ebcd70e6726edc3918008c08bee6ee6", size = 490545, upload-time = "2026-06-30T07:15:41.729Z" }, + { url = "https://files.pythonhosted.org/packages/a3/42/da06c5aa8f0484ff07f270787434204d9f4535e2f8c3b51ed402267e63c3/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cdc7e35386f3847df728fbcb5e887e2d79c19e2fa1eba9e51b6621d23e3243af", size = 382828, upload-time = "2026-06-30T07:15:43.327Z" }, + { url = "https://files.pythonhosted.org/packages/57/d7/fe978efc2ae50abe48eb7464668ea99f53c010c60aeebb7b35ad27f23661/rpds_py-2026.6.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acac386b453c2516111b50985d60ce46e7fadb5ea71ae7b25f4c946935bf27cf", size = 365678, upload-time = "2026-06-30T07:15:44.992Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/1d8922e1990b2a6eb532b6ff53d3e73d2b3bbffc84116c75826bee73dfc6/rpds_py-2026.6.3-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:425560c6fa0415f27261727bb20bd097568485e5eb0c121f1949417d1c516885", size = 377811, upload-time = "2026-06-30T07:15:46.523Z" }, + { url = "https://files.pythonhosted.org/packages/b1/3d/198dceafb4fb034a6a47347e1b0735d34e0bd4a50be4e898d408ee66cb14/rpds_py-2026.6.3-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a550fb4950a06dde3beb4721f5ad4b25bf4513784665b0a8522c792e2bd822a4", size = 395382, upload-time = "2026-06-30T07:15:47.955Z" }, + { url = "https://files.pythonhosted.org/packages/1f/f1/13968e49655d40b6b19d8b9140296bbc6f1d86b3f0f6c346cf9f1adddf4b/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4f4bca01b63096f606e095734dd56e74e175f94cfbf24ff3d63281cec61f7bb7", size = 543832, upload-time = "2026-06-30T07:15:49.33Z" }, + { url = "https://files.pythonhosted.org/packages/ac/ab/289bcb1b90bd3e40a2900c561fa0e2087345ecbb094f0b870f2345142b7c/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ccffae9a092a00deb7efd545fe5e2c33c33b88e7c054337e9a74c179347d0b7d", size = 611011, upload-time = "2026-06-30T07:15:50.847Z" }, + { url = "https://files.pythonhosted.org/packages/1e/16/5043105e679436ccfbc8e5e0dd2d663ed18a8b8113515fd06a5e5d77c83e/rpds_py-2026.6.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1cf01971c4f2c5553b772a542e4aaf191789cd331bc2cd4ff0e6e65ba49e1e97", size = 572431, upload-time = "2026-06-30T07:15:52.394Z" }, + { url = "https://files.pythonhosted.org/packages/85/ed/adab103321c0a6565d5ae1c2998349bc3ee175b82ccc5ae8fc04cc413075/rpds_py-2026.6.3-cp313-cp313-win32.whl", hash = "sha256:8c3d1e9c15b9d51ca0391e13da1a25a0a4df3c58a37c9dc368e0736cf7f69df0", size = 201710, upload-time = "2026-06-30T07:15:53.894Z" }, + { url = "https://files.pythonhosted.org/packages/7b/ed/a03b09668e74e5dabbf2e211f6468e1820c0552f7b0500082da31841bf7b/rpds_py-2026.6.3-cp313-cp313-win_amd64.whl", hash = "sha256:9250a9a0a6fd4648b3f868da8d91a4c52b5811a62df58e753d50ae4454a36f80", size = 219454, upload-time = "2026-06-30T07:15:55.25Z" }, + { url = "https://files.pythonhosted.org/packages/27/17/b8642c12930b71bc2b25831f6708ccf0f75abcd11883932ec9ce54ba3a78/rpds_py-2026.6.3-cp313-cp313-win_arm64.whl", hash = "sha256:900a67df3fd1660b035a4761c4ce73c382ea6b35f90f9863c36c6fd8bf8b09bb", size = 215063, upload-time = "2026-06-30T07:15:56.573Z" }, + { url = "https://files.pythonhosted.org/packages/b6/36/7fbe9dcdaf857fb3f63c2a2284b62492d95f5e8334e947e5fb6e7f68c9be/rpds_py-2026.6.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:931908d9fc855d8f74783377822be318edb6dcb19e47169dc038f9a1bf60b06e", size = 344510, upload-time = "2026-06-30T07:15:57.921Z" }, + { url = "https://files.pythonhosted.org/packages/ba/54/f785cc3d3f60839ca57a5af4927a9f347b07b2799c373fc20f7949f87c7e/rpds_py-2026.6.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:d7469697dce35be237db177d42e2a2ee26e6dcc5fc052078a6fefabd288c6edd", size = 339495, upload-time = "2026-06-30T07:15:59.238Z" }, + { url = "https://files.pythonhosted.org/packages/63/ef/d4cdaf309e6b095b43597103cf8c0b951d6cca2acce68c474f75ec12e0c7/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bcfbcf66006befb9fd2aeaa9e01feaf881b4dc330a02ba07d2322b1c11be7b5d", size = 369454, upload-time = "2026-06-30T07:16:01.021Z" }, + { url = "https://files.pythonhosted.org/packages/96/4a/9559a68b7ee15db09d7981212e8c2e219d2a1d6d4faa0391d813c3496a36/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:847927daf4cffbd4e90e42bc890069897101edd015f956cb8721b3473372edda", size = 374583, upload-time = "2026-06-30T07:16:02.287Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/8964aa7d2c6e8ac43eba8eb6e6b0fdda1f46d39f2fc3e6aa9f2cb17f485d/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:aca6c1ef08a82bfe327cc156da694660f599923e2e6665b6d81c9c2d0ac9ffc8", size = 492919, upload-time = "2026-06-30T07:16:03.723Z" }, + { url = "https://files.pythonhosted.org/packages/8f/97/6908094ac804115e65aedfd90f1b5fee4eebebd3f6c4cfc5419939267565/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ae50181a047c871561212bb97f7932a2d45fb53e947bd9b57ebad85b529cbc53", size = 383725, upload-time = "2026-06-30T07:16:05.305Z" }, + { url = "https://files.pythonhosted.org/packages/d1/9c/0d1fdc2e7aba23e290d603bc494e97bd205bae262ce33c6b32a69768ed5e/rpds_py-2026.6.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc319e5a1de4b6913aac94bf6a2f9e847371e0a140a43dd4991db1a09bc2d504", size = 367255, upload-time = "2026-06-30T07:16:07.086Z" }, + { url = "https://files.pythonhosted.org/packages/c4/fe/f0209ca4a9ed074bc8acb44dfd0e81c3122e94c9689f5645b7973a866719/rpds_py-2026.6.3-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:e4316bf32babbed84e691e352faf967ce2f0f024174a8643c37c94a1080374fc", size = 379060, upload-time = "2026-06-30T07:16:08.525Z" }, + { url = "https://files.pythonhosted.org/packages/c6/8d/f1cc54c616b9d8897de8738aac148d20afca93f68187475fe194d09a71b9/rpds_py-2026.6.3-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:8c6e5a2f750cc71c3e3b11d71661f21d6f9bc6cebc6564b1466417a1ec03ec77", size = 395960, upload-time = "2026-06-30T07:16:09.989Z" }, + { url = "https://files.pythonhosted.org/packages/fb/04/aafff00f73aeca2945f734f1d483c64ab8f472d0864ab02377fd8e89c3b2/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:4470ce197d4090875cf6affbf1f853338387428df97c4fb7b7106317b8214698", size = 545356, upload-time = "2026-06-30T07:16:11.816Z" }, + { url = "https://files.pythonhosted.org/packages/fd/cc/e229663b9e4ddac5a4acbe9085dd80a71af2a5d356b8b39d6bff233f24b0/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:ea964164cc9afa72d4d9b23cc28dafae93693c0a53e0b42acbff15b22c3f9ddd", size = 612319, upload-time = "2026-06-30T07:16:13.586Z" }, + { url = "https://files.pythonhosted.org/packages/e3/7a/8a0e6d3e6cd066af108b71b43122c3fe158dd9eb86acac626593a2582eb1/rpds_py-2026.6.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:639c8929aa0afe81be836b04de888460d6bed38b9c54cfc18da8f6bfabf5af5d", size = 573508, upload-time = "2026-06-30T07:16:15.23Z" }, + { url = "https://files.pythonhosted.org/packages/87/03/2a69ab618a789cf6cf85c86bb844c62d090e700ab1a2aa676b3741b6c516/rpds_py-2026.6.3-cp314-cp314-win32.whl", hash = "sha256:882076c00c0a608b131187055ddc5ae29f2e7eaf870d6168980420d58528a5c8", size = 202504, upload-time = "2026-06-30T07:16:16.893Z" }, + { url = "https://files.pythonhosted.org/packages/85/62/a3892ba945f4e24c78f352e5de3c7620d8479f73f211406a97263d13c7d2/rpds_py-2026.6.3-cp314-cp314-win_amd64.whl", hash = "sha256:0be972be84cfcaf46c8c6edf690ca0f154ac17babf1f6a955a51579b34ad2dc5", size = 220380, upload-time = "2026-06-30T07:16:18.108Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e7/c2bd44dc831931815ad11ebb5f430b5a0a4d3caa9de837107876c30c3432/rpds_py-2026.6.3-cp314-cp314-win_arm64.whl", hash = "sha256:2a9c6f195058cb45335e8cc3802745c603d716eb96bc9625950c1aac71c0c703", size = 215976, upload-time = "2026-06-30T07:16:19.654Z" }, + { url = "https://files.pythonhosted.org/packages/79/9c/fff7b74bce9a091ec9a012a03f9ff5f69364eaf9451060dfc4486da2ffdd/rpds_py-2026.6.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:f90938e92afda60266da758ee7d363447f7f0138c9559f9e1811629580582d90", size = 346840, upload-time = "2026-06-30T07:16:21.268Z" }, + { url = "https://files.pythonhosted.org/packages/e9/44/77bcb1168b33704908295533d27f10eb811e9e3e193e8993dc99572211d3/rpds_py-2026.6.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ec829541c45bca16e61c7ae50c20501f213605beb75d1aba91a6ee37fbbb56a4", size = 340282, upload-time = "2026-06-30T07:16:22.875Z" }, + { url = "https://files.pythonhosted.org/packages/87/3c/7a9081c7c9e645b39efe19e4ffbeccd80add246327cd9b888aecffd72317/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afd70d95892096cdb26f15a00c45907b17817577aa8d1c76b2dcc2788391f9e9", size = 370403, upload-time = "2026-06-30T07:16:24.415Z" }, + { url = "https://files.pythonhosted.org/packages/f7/69/af47021eb7dad6ff3396cb001c08f0f3c4d06c20253f75be6421a59fe6b7/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:29dfa0533a5d4c94d4dfa1b694fcb56c9c63aad8330ffdd816fd225d0a7a162f", size = 376055, upload-time = "2026-06-30T07:16:26.111Z" }, + { url = "https://files.pythonhosted.org/packages/81/fc/a3bcf517084396a6dd258c592567a3c011ba4557f2fde23dceaf26e74f2e/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:af05d726809bff6b141be124d4c7ce998f9c9c7f30edb1f46c07aa103d540b41", size = 494419, upload-time = "2026-06-30T07:16:27.596Z" }, + { url = "https://files.pythonhosted.org/packages/c9/eb/13d529d1788135425c7bf207f8463458ca5d92e43f3f701365b83e9dffc1/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9826217f048f620d9a712672818bf231442c1b35d96b227a07eabd11b4bb6945", size = 384848, upload-time = "2026-06-30T07:16:29.183Z" }, + { url = "https://files.pythonhosted.org/packages/8e/f4/b7ac49f30013aba8f7b9566b1dd07e81de95e708c1374b7bacc5b9bc5c9c/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:536bceea4fa4acf7e1c61da2b5786304367c816c8895be71b8f537c480b0ea1f", size = 371369, upload-time = "2026-06-30T07:16:30.912Z" }, + { url = "https://files.pythonhosted.org/packages/31/86/6260bafa622f788b07ddec0e52d810305c8b9b0b8c27f58a2ab04bf62b4f/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:bc0011654b91cc4fb2ae701bec0a0ba1e552c0714247fa7af6c59e0ccfa3a4e1", size = 379673, upload-time = "2026-06-30T07:16:32.486Z" }, + { url = "https://files.pythonhosted.org/packages/19/c3/03f1ee79a047b48daeca157c89a18509cde22b6b951d642b9b0af1be660a/rpds_py-2026.6.3-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:539d75de9e0d536c84ff18dfeb805398e58227001ce09231a26a08b9aed1ee0e", size = 397500, upload-time = "2026-06-30T07:16:34.471Z" }, + { url = "https://files.pythonhosted.org/packages/f0/95/8ed0cd8c377dca12aea498f119fe639fc474d1461545c39d2b5872eb1c0f/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:166cf54d9f44fc6ceb53c7860258dde44a81406646de79f8ed3234fca3b6e538", size = 545978, upload-time = "2026-06-30T07:16:36.45Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f2/0eb57f0eaa83f8fc152a7e03de968ab77e1f00732bebc892b190c6eebde7/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:d34c20167764fbcf927194d532dd7e0c56772f0a5f943fa5ef9e9afbba8fb9db", size = 613350, upload-time = "2026-06-30T07:16:38.213Z" }, + { url = "https://files.pythonhosted.org/packages/5b/de/e0674bdbc3ef7634989b3f854c3f34bc1f587d36e5bfdc5c378d57034619/rpds_py-2026.6.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:ea7bb13b7c9a29791f87a0387ba7d3ad3a6d783d827e4d3f27b40a0ff44495e2", size = 576486, upload-time = "2026-06-30T07:16:39.797Z" }, + { url = "https://files.pythonhosted.org/packages/f2/f6/21101359743cd136ada781e8210a85769578422ba460672eea0e29739200/rpds_py-2026.6.3-cp314-cp314t-win32.whl", hash = "sha256:6de4744d05bd1aa1be4ed7ea1189e3979196808008113bbbf899a460966b925e", size = 201068, upload-time = "2026-06-30T07:16:41.316Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b2/9574d4d44f7760c2aa32d92a0a4f41698e33f5b204a0bf5c9758f52c79d5/rpds_py-2026.6.3-cp314-cp314t-win_amd64.whl", hash = "sha256:c7b9a2f8f4d8e90af72571d3d495deebdd7e3c75451f5b41719aee166e940fc2", size = 220600, upload-time = "2026-06-30T07:16:43.091Z" }, + { url = "https://files.pythonhosted.org/packages/08/ae/f23a2697e6ee6340a578b0f136be6483657bef0c6f9497b752bb5c0964bb/rpds_py-2026.6.3-cp315-cp315-macosx_10_12_x86_64.whl", hash = "sha256:e059c5dde6452b44424bd1834557556c226b57781dee1227af23518459722b13", size = 344726, upload-time = "2026-06-30T07:16:44.5Z" }, + { url = "https://files.pythonhosted.org/packages/c3/63/e7b3a1a5358dd32c930a1062d8e15b67fd6e8922e81df9e91706d66ee5c8/rpds_py-2026.6.3-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2f7c26fbc5acd2522b95d4177fe4710ffd8e9b20529e703ffbf8db4d93903f05", size = 339587, upload-time = "2026-06-30T07:16:46.255Z" }, + { url = "https://files.pythonhosted.org/packages/ec/64/10a85681916ca55fffb91b0a211f84e34297c109243484dd6394660a8a7c/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a3086b538543802f84c843911242db20447de00d8752dd0efc936dbcf02218ba", size = 369585, upload-time = "2026-06-30T07:16:48.101Z" }, + { url = "https://files.pythonhosted.org/packages/76/c2/baf95c7c38823e12ba34407c5f5767a89e5cf2233895e56f608167ae9493/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8f2e5c5ee828d42cb11760761c0af6507927bec42d0ad5458f97c9203b054617", size = 375479, upload-time = "2026-06-30T07:16:49.93Z" }, + { url = "https://files.pythonhosted.org/packages/6a/94/0aad06c72d65101e11d33528d438cda99a39ce0da99466e156158f2541d3/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ed0c1e5d10cdc7135537988c74a0188da68e2f3c30813ba3744ab1e42e0480f9", size = 492418, upload-time = "2026-06-30T07:16:51.641Z" }, + { url = "https://files.pythonhosted.org/packages/b5/17/de3f5a479a1f056535d7489819639d8cd591ea6281d700390b43b1abd745/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8c2642a7603ec0b16ed77da4555db3b4b472341904873788327c0b0d7b95f1bb", size = 384123, upload-time = "2026-06-30T07:16:53.622Z" }, + { url = "https://files.pythonhosted.org/packages/46/7d/bf09bd1b145bb2671c03e1e6d1ab8651858d90d8c7dfeadd85a37a934fd8/rpds_py-2026.6.3-cp315-cp315-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8e4320744c1ffdd95a603def63344bfab2d33edeab301c5007e7de9f9f5b3885", size = 367351, upload-time = "2026-06-30T07:16:55.241Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ea/1bb734f314b8be319149ddee80b18bd41372bdcfbdf88d28131c0cd37719/rpds_py-2026.6.3-cp315-cp315-manylinux_2_31_riscv64.whl", hash = "sha256:a9f4645593036b81bbdb36b9c8e0ea0d1c3fee968c4d59db0344c14087ef143a", size = 378827, upload-time = "2026-06-30T07:16:56.841Z" }, + { url = "https://files.pythonhosted.org/packages/4b/93/d9611e5b25e26df9a3649813ed66193ace9347a7c7fc4ab7cf70e94851c0/rpds_py-2026.6.3-cp315-cp315-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:e55d236be29255554da47abe5c577637db7c24a02b8b46f0ca9524c855801868", size = 395966, upload-time = "2026-06-30T07:16:58.557Z" }, + { url = "https://files.pythonhosted.org/packages/c3/cb/99d77e16e5534ae1d90629bbe419ba6ee170833a6a85e3aa1cc41726fbbc/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:24e9c5386e16669b674a69c156c8eeefcb578f3b3397b713b08e6d60f3c7b187", size = 545680, upload-time = "2026-06-30T07:17:00.164Z" }, + { url = "https://files.pythonhosted.org/packages/59/15/11a29755f790cef7a2f755e8e14f4f0c33f39489e1893a632a2eee59672b/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:c60924535c75f1566b6eb75b5c31a48a43fef04fa2d0d201acbad8a9969c6107", size = 611853, upload-time = "2026-06-30T07:17:01.962Z" }, + { url = "https://files.pythonhosted.org/packages/68/86/0c27547e21644da938fb530f7e1a8148dd24d02db07e7a5f2567a17ce710/rpds_py-2026.6.3-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:38a2fea2787428f811719ceb9114cb78964a3138838320c29ac39526c79c16ba", size = 573715, upload-time = "2026-06-30T07:17:03.693Z" }, + { url = "https://files.pythonhosted.org/packages/29/71/4d8fcf700931815594bce892255bbd973b94efaf0fc1932b0590df18d886/rpds_py-2026.6.3-cp315-cp315-win32.whl", hash = "sha256:d483fe17f01ad64b7bf7cc38fcefff1ca9fb83f8c2b2542b68f97ffe0611b369", size = 202864, upload-time = "2026-06-30T07:17:05.746Z" }, + { url = "https://files.pythonhosted.org/packages/eb/62/b577562de0edbb55b2be85ce5fd09c33e386b9b13eee09833af4240fd5c4/rpds_py-2026.6.3-cp315-cp315-win_amd64.whl", hash = "sha256:67e3a721ffc5d8d2210d3671872298c4a84e4b8035cfe42ffd7cde35d772b146", size = 220430, upload-time = "2026-06-30T07:17:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c8/95/d6d0b2509825141eef60669a5739eec88dbc6a48053d6c92993a5704defe/rpds_py-2026.6.3-cp315-cp315-win_arm64.whl", hash = "sha256:6e84adbcf4bf841aed8116a8264b9f50b4cb3e7bd89b516122e616ac56ca269e", size = 215877, upload-time = "2026-06-30T07:17:09.008Z" }, + { url = "https://files.pythonhosted.org/packages/b7/bf/f3ea278f0afd615c1d0f19cb69043a41526e2bb600c2b536eb192218eb27/rpds_py-2026.6.3-cp315-cp315t-macosx_10_12_x86_64.whl", hash = "sha256:ae6dd8f10bd17aad820876d24caec9efdafd80a318d16c0a48edb5e136902c6b", size = 346933, upload-time = "2026-06-30T07:17:10.762Z" }, + { url = "https://files.pythonhosted.org/packages/9d/29/9907bdf1c5346763cf10b7f6852aad86652168c259def904cbe0082c5864/rpds_py-2026.6.3-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:bdbd97738551fca3917c1bd7188bec1920bb520104f28e7e1007f9ceb17b7690", size = 340274, upload-time = "2026-06-30T07:17:12.266Z" }, + { url = "https://files.pythonhosted.org/packages/6f/2c/8e03767b5778ef25cebf74a7a91a2c3806f8eced4c92cb7406bbe060756d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b95977e7211527ab0ba576e286d023389fbeeb32a6b7b771665d333c60e5342", size = 370763, upload-time = "2026-06-30T07:17:14.107Z" }, + { url = "https://files.pythonhosted.org/packages/2e/e1/df2a7e1ba2efd796af26194250b8d42c821b46592311595162af9ef0528d/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d15fde0e6fb0d88a60d221204873743e5d9f0b7d29165e62cd86d0413ad74ba6", size = 376467, upload-time = "2026-06-30T07:17:15.76Z" }, + { url = "https://files.pythonhosted.org/packages/6b/de/8a0814d1946af29cb068fb259aa8622f856df1d0bab58429448726b537f5/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a136d453475ac0fcbda502ef1e6504bd28d6d904700915d278deeab0d00fe140", size = 496689, upload-time = "2026-06-30T07:17:17.308Z" }, + { url = "https://files.pythonhosted.org/packages/df/f3/f19e0c852ba13694f5a79f3b719331051573cb5693feacf8a88ffffc3a71/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f826877d462181e5eb1c26a0026b8d0cab05d99844ecb6d8bf3627a2ca0c0442", size = 385340, upload-time = "2026-06-30T07:17:18.928Z" }, + { url = "https://files.pythonhosted.org/packages/e2/ae/7ec3a9d2d4351f99e37bcb06b6b6f954512646bfdbf9742e1de727865daf/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:79486287de1730dbaff3dbd124d0ca4d2ef7f9d29bf2544f1f93c09b5bcbbd12", size = 372179, upload-time = "2026-06-30T07:17:20.539Z" }, + { url = "https://files.pythonhosted.org/packages/d3/ac/9cee911dff2aaa9a5a8354f6610bf2e6a616de9197c5fff4f54f82585f1e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_31_riscv64.whl", hash = "sha256:808345f53cb952433ca2816f1604ff3515608a81784954f38d4452acfe8e61d5", size = 379993, upload-time = "2026-06-30T07:17:22.212Z" }, + { url = "https://files.pythonhosted.org/packages/83/6b/7c2a07ba88d1e9a936612f7a5d067467ed03d971d5a06f7d309dff044a7e/rpds_py-2026.6.3-cp315-cp315t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1967debc37f64f2c4dc90a7f563aec558b471966e12adcac4e1c4240496b6ebf", size = 398909, upload-time = "2026-06-30T07:17:23.66Z" }, + { url = "https://files.pythonhosted.org/packages/97/0b/776ffcb66783637b0031f6d58d6fb55913c8b5abf00aeecd46bf933fb477/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:f0840b5b17057f7fd918b76183a4b5a0635f43e14eb2ce60dce1d4ee4707ea00", size = 546584, upload-time = "2026-06-30T07:17:25.264Z" }, + { url = "https://files.pythonhosted.org/packages/55/33/ba3bc04d7092bd553c9b2b195624992d2cc4f3de1f380b7b93cbee67bd79/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:faa679d19a6696fd54259ad321251ad77a13e70e03dd834daa762a44fb6196ef", size = 614357, upload-time = "2026-06-30T07:17:26.888Z" }, + { url = "https://files.pythonhosted.org/packages/8b/71/14edf065f04630b1a8472f7653cad03f6c478bcf95ea0e6aed55451e33ea/rpds_py-2026.6.3-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:23a439f31ccbeff1574e24889128821d1f7917470e830cf6544dced1c662262a", size = 576533, upload-time = "2026-06-30T07:17:28.546Z" }, + { url = "https://files.pythonhosted.org/packages/ba/76/65002b08596c389105720a8c0d22298b8dc25a4baf89b2ce431343c8b1de/rpds_py-2026.6.3-cp315-cp315t-win32.whl", hash = "sha256:913ca42ccad3f8cc6e292b587ae8ae49c8c823e5dce51a736252fc7c7cdfa577", size = 201204, upload-time = "2026-06-30T07:17:30.193Z" }, + { url = "https://files.pythonhosted.org/packages/8c/97/d855d6b3c322d1f27e26f5241c42016b56cf01377ea8ed348285f54652f0/rpds_py-2026.6.3-cp315-cp315t-win_amd64.whl", hash = "sha256:ae3d4fe8c0b9213624fdce7279d70e3b148b682ca20719ebd193a23ebfa47324", size = 220719, upload-time = "2026-06-30T07:17:31.788Z" }, + { url = "https://files.pythonhosted.org/packages/b4/9c/f0d19ac587fd0e4ab6b72cda355e9c5a6166b01ef7e064e437aef8eb9fef/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:4cf2d36a2357e4d07bb5a4f98801265327b48256867816cfd2ceb001e9754a8f", size = 349791, upload-time = "2026-06-30T07:17:33.315Z" }, + { url = "https://files.pythonhosted.org/packages/38/c7/1d49d204c9fd2ee6c537601dc4c1ba921e03363ca576bfab94a00254ac9a/rpds_py-2026.6.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:30c6dc199b24a5e3e81d50da0f00858c5bbdb2617a750395687f4339c5818171", size = 352842, upload-time = "2026-06-30T07:17:34.897Z" }, + { url = "https://files.pythonhosted.org/packages/ac/e5/c0b5dc93cd0d4c06ce1f438907649514e2ea077bcd911e3154a51e96c38e/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9891e594296ab9dada6551c8e7b387b2721f27a67eecd528412e8906247a7b90", size = 382094, upload-time = "2026-06-30T07:17:36.514Z" }, + { url = "https://files.pythonhosted.org/packages/0d/54/ec0e907b4ca8d541112db352409bd15f871c9b243e0c92c9b5a46ae96f01/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b5c2dc92304aa48a4a60443b548bb12f12e119d4b72f314015e67b9e1be97fca", size = 388662, upload-time = "2026-06-30T07:17:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/d3/f4/921c22a4fd0f1c1ac13a3996ffbf0aa67951e2c8ad0d1d9574938a2932e8/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:127e08c0642d880cf32ca47ec2a4a77b901f7e2dd1ad9762adb13955d72ffcc9", size = 504896, upload-time = "2026-06-30T07:17:39.689Z" }, + { url = "https://files.pythonhosted.org/packages/0b/1b/a114b972cefa1ab1cdb3c7bb177cd3844a12826c507c722d3a73516dbbaf/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8bb68f03f395eb793220b45c097bd4d8c32944393da0fad8b999efac0868fc8c", size = 391545, upload-time = "2026-06-30T07:17:41.336Z" }, + { url = "https://files.pythonhosted.org/packages/4e/98/af9b3db77d47fcbe6c8c1f36e2c2147ec70292819e99c325f871584a1c11/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a3450b693fde92133e9f51060568a4c31fcca76d5e53bbd611e689ca446517e9", size = 380059, upload-time = "2026-06-30T07:17:42.857Z" }, + { url = "https://files.pythonhosted.org/packages/c9/ba/0efd8668b97c1d26a61566386c636a7a7a09829e474fdf807caa15a2c844/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:5e8d07bddee435a2ff6f1920e18feff28d0bc4533e42f4bf6927fbd073312c41", size = 393235, upload-time = "2026-06-30T07:17:44.637Z" }, + { url = "https://files.pythonhosted.org/packages/62/90/8c139ee9690f73b0829f32647de6f40d826f8f443af6fa72644f96351aac/rpds_py-2026.6.3-pp311-pypy311_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3a83ae6c67b7676b9878378547ca8e93ed77a580037bcbcd1d32f739e1e6089c", size = 413008, upload-time = "2026-06-30T07:17:46.225Z" }, + { url = "https://files.pythonhosted.org/packages/9c/97/0043896fdd7828ce09a1d9a8b06433714d0960fc4ff3fc4aa72b666b764e/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:2bfd04c19ddbd6640de0b51894d764bd2758854d5b75bd102d2ef10cb9c293a9", size = 558118, upload-time = "2026-06-30T07:17:47.759Z" }, + { url = "https://files.pythonhosted.org/packages/f6/40/02355f0e134f783a8f9814c4680a1bd311d37671577a5964ea838573ff37/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_i686.whl", hash = "sha256:ca6546b66be9dc4738b1b043d5ebd5488c66c578c5ff0fd0e8065313fe3afb76", size = 623138, upload-time = "2026-06-30T07:17:49.355Z" }, + { url = "https://files.pythonhosted.org/packages/10/85/48f0abdcef5cce4e034c7a5b0ceeceba0b01bf0d942824f4bb720afe2dec/rpds_py-2026.6.3-pp311-pypy311_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:8e65860d238379ed982fd9ba690579b5e95af2f4840f99c772816dbe573cb826", size = 586486, upload-time = "2026-06-30T07:17:51.141Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "sqlglot" +version = "30.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/cd/39a94f0f98076ee8e7c7c38fd4bba8d7845b0c629ff967057c64ef2c0989/sqlglot-30.14.0.tar.gz", hash = "sha256:df2ef5d2b8ca814313781f4ff35bf63e58f821ef517eeddbd523c19a61fa9bb9", size = 5944410, upload-time = "2026-07-27T11:23:30.698Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/33/ec/a729883ceda22dcd9117ce182f64d884bf494e72c4dfce00c2ad0a5978e1/sqlglot-30.14.0-py3-none-any.whl", hash = "sha256:fc768e24889d63a5e1237dea7ad305e5ffb4356a98b0bed828f89591ebcd3636", size = 719007, upload-time = "2026-07-27T11:23:28.637Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +]