From 30bce3d6ddd40fb7fcc52cf7ca29f390bd7c0be8 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Wed, 10 Dec 2025 18:00:35 -0800 Subject: [PATCH 1/9] First version of the spec --- core-spec/spec.md | 525 +++++++++++++++++++++++++++++ core-spec/spec.yaml | 222 ++++++++++++ examples/tpcds_semantic_model.yaml | 434 ++++++++++++++++++++++++ 3 files changed, 1181 insertions(+) create mode 100644 core-spec/spec.md create mode 100644 core-spec/spec.yaml create mode 100644 examples/tpcds_semantic_model.yaml diff --git a/core-spec/spec.md b/core-spec/spec.md new file mode 100644 index 00000000..2eca7686 --- /dev/null +++ b/core-spec/spec.md @@ -0,0 +1,525 @@ +# OSI - Core Metadata Specification + +**Version:** 1.0 + +## Goals + +- **Standardization**: Establish uniform language and structure for semantic model definitions, ensuring consistency and ease of interpretation across various tools and systems. +- **Extensibility**: Support domain-specific extensions while maintaining core compatibility. +- **Interoperability**: Enable exchange and reuse across different AI and BI applications. + +## Table of Contents + +1. [Enumerations](#enumerations) +2. [Semantic Model](#semantic-model) +3. [Datasets](#datasets) +4. [Relationships](#relationships) +5. [Fields](#fields) +6. [Metrics](#metrics) +7. [Examples](#examples) + +--- + +## Enumerations + +Standard enumeration values used throughout the specification. + +### Dialects + +Supported SQL and expression language dialects for metrics and field definitions. + +| Dialect | Description | +|---------|-------------| +| `ANSI_SQL` | Standard SQL dialect | +| `SNOWFLAKE` | Snowflake SQL | +| `MDX` | Multi-Dimensional Expressions | +| `TABLEAU` | Tableau calculations | + +### Vendors + +Supported vendors for custom extensions and integrations. + +| Vendor | Description | +|--------|-------------| +| `COMMON` | Common/standard extensions | +| `SNOWFLAKE` | Snowflake-specific attributes | +| `SALESFORCE` | Salesforce/Tableau-specific attributes | +| `DBT` | dbt-specific attributes | + +### Aggregation Types + +Standard aggregation functions for structured metric definitions. + +| Type | Description | +|------|-------------| +| `SUM` | Sum aggregation | +| `AVG` | Average aggregation | +| `COUNT` | Count aggregation | +| `COUNT_DISTINCT` | Distinct count aggregation | +| `MIN` | Minimum value | +| `MAX` | Maximum value | + +--- + +## Semantic Model + +The top-level container that represents a complete semantic model, including datasets, relationships, and metrics. + +### Schema + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Unique identifier for the semantic model | +| `description` | string | No | Human-readable description | +| `ai_context` | string/object | No | Additional context for AI tools (e.g., custom instructions) | +| `datasets` | array | Yes | Collection of logical datasets (fact and dimension tables) | +| `relationships` | array | No | Defines how logical datasets are connected | +| `metrics` | array | No | Quantifiable measures defined as aggregate expessions on fields from logical datsets | +| `custom_extensions` | array | No | Vendor-specific attributes for extensibility | + +### Example + +```yaml +semantic_model: + - name: sales_analytics + description: Sales and customer analytics model + ai_context: + instructions: "Use this model for sales analysis and customer insights" + datasets: [] + relationships: [] + metrics: [] + custom_extensions: + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' +``` + +--- + +## Datasets + +Logical datasets represent business entities or concepts (fact and dimension tables). They contain fields and define the structure of the data. + +### Schema + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Unique identifier for the dataset | +| `source` | string | Yes | Reference to underlying physical table/view (e.g., `database.schema.table`) or query | +| `primary_key` | array | No | Primary key columns that uniquely identify rows (single or composite) | +| `unique_keys` | array of arrays | No | Array of unique key definitions (each can be single or composite) | +| `description` | string | No | Human-readable description | +| `ai_context` | string/object | No | Additional context for AI tools (e.g., synonyms, common terms) | +| `fields` | array | No | Row-level attributes for grouping, filtering, and metric expressions | +| `custom_extensions` | array | No | Vendor-specific attributes | + +### Primary Key Examples + +```yaml +# Simple primary key +primary_key: [customer_id] + +# Composite primary key +primary_key: [order_id, line_number] +``` + +### Unique Keys Examples + +```yaml +# Multiple unique keys (each can be simple or composite) +unique_keys: + - [email] # Simple unique key + - [first_name, last_name] # Composite unique key +``` + +### Example + +```yaml +datasets: + - name: orders + source: sales.public.orders + primary_key: [order_id] + unique_keys: + - [order_id] + - [order_number] + description: Order transactions + ai_context: + synonyms: + - "purchases" + - "sales" + fields: [] + custom_extensions: + - vendor_name: DBT + data: '{"materialized": "table"}' +``` + +--- + +## Relationships + +Relationships define how logical datasets are connected through foreign key constraints. They support both simple and composite keys. + +### Schema + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Unique identifier for the relationship | +| `from` | string | Yes | The logical dataset on the many side of the relationship | +| `to` | string | Yes | The logical dataset on the one side of the relationship | +| `from_columns` | array | Yes | Array of column names in the "from" dataset (foreign key columns) | +| `to_columns` | array | Yes | Array of column names in the "to" dataset (primary or unique key columns) | +| `ai_context` | string/object | No | Additional context for AI tools | +| `custom_extensions` | array | No | Vendor-specific attributes | + +### Important Notes + +- The order of columns in `from_columns` must correspond to the order in `to_columns` +- Both arrays must have the same number of columns +- For simple relationships, use a single column: `[column1]` +- For composite relationships, use multiple columns: `[column1, column2]` + +### Examples + +**Simple Relationship:** + +```yaml +- name: orders_to_customers + from: orders + to: customers + from_columns: [customer_id] + to_columns: [id] +``` + +**Composite Relationship:** + +```yaml +# order_lines.product_id = products.id AND order_lines.variant_id = products.variant_id +- name: order_lines_to_products + from: order_lines + to: products + from_columns: [product_id, variant_id] + to_columns: [id, variant_id] +``` + +--- + +## Fields + +Fields represent row-level attributes that can be used for grouping, filtering, and in metric expressions. They can be simple column references or computed expressions. + +### Schema + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Unique identifier for the field within the dataset | +| `expression` | string | Yes | SQL expression that defines how to compute this field | +| `dimension` | object | No | Dimension metadata (e.g., `is_time` flag) | +| `label` | string | No | Label for categorization | +| `description` | string | No | Human-readable description | +| `ai_context` | string/object | No | Additional context for AI tools (e.g., synonyms) | +| `custom_extensions` | array | No | Vendor-specific attributes | + +### Dimension Object + +| Field | Type | Description | +|-------|------|-------------| +| `is_time` | boolean | Indicates if this is a time-based dimension for temporal filtering | + +### Examples + +**Simple Column Reference:** + +```yaml +- name: customer_id + expression: customer_id + description: Customer identifier + dimension: + is_time: false +``` + +**Computed Field:** + +```yaml +- name: full_name + expression: first_name || ' ' || last_name + description: Customer full name + ai_context: + synonyms: + - "name" + - "customer name" +``` + +**Time Dimension:** + +```yaml +- name: order_date + expression: order_date + dimension: + is_time: true + description: Date when order was placed + ai_context: + synonyms: + - "purchase date" + - "transaction date" +``` + +--- + +## Metrics + +Quantitative measures defined on business data, representing key calculations like sums, averages, ratios, etc. Metrics can be defined at the semantic model level and can span multiple datasets. + +### Schema + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | Yes | Unique identifier for the metric | +| `expression` | object | Yes | Expression definition with dialect support | +| `description` | string | No | Human-readable description of what the metric measures | +| `ai_context` | string/object | No | Additional context for AI tools (e.g., synonyms) | +| `custom_extensions` | array | No | Vendor-specific attributes | + +### Expression Object + +The expression object supports two formats: + +**Option 1: Full SQL Expression** + +```yaml +expression: + - dialect: ANSI_SQL # Default + expression: "SUM(sales) / COUNT(DISTINCT customer_id)" +``` + + +### Examples + +**Simple Aggregation:** + +```yaml +- name: total_revenue + expression: + - dialect: ANSI_SQL + aggregation_type: SUM + input_expr: order_amount + description: Total revenue across all orders + ai_context: + synonyms: + - "total sales" + - "revenue" +``` + +**Complex Calculation:** + +```yaml +- name: average_order_value + expression: + - dialect: ANSI_SQL + expression: SUM(orders.amount) / COUNT(DISTINCT orders.order_id) + description: Average value per order + ai_context: + synonyms: + - "AOV" + - "avg order size" +``` + +**Cross-Dataset Metric:** + +```yaml +- name: customer_lifetime_value + expression: + - dialect: ANSI_SQL + expression: SUM(orders.amount) / COUNT(DISTINCT customers.id) + description: Average lifetime value per customer + ai_context: + synonyms: + - "CLV" + - "LTV" +``` + +--- + +## Custom Extensions + +Custom extensions allow vendors to add platform-specific metadata without breaking core compatibility. Each extension includes a vendor name and arbitrary JSON data. + +### Schema + +```yaml +custom_extensions: + - vendor_name: string # Must be from vendors enum + data: string # JSON string containing vendor-specific data +``` + +### Examples + +**Snowflake Extension:** + +```yaml +- vendor_name: SNOWFLAKE + data: '{ + "warehouse": "ANALYTICS_WH", + "database": "PROD", + "schema": "PUBLIC" + }' +``` + +**Salesforce Extension:** + +```yaml +- vendor_name: SALESFORCE + data: '{ + "tableau_workbook_id": "sales_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily" + } + }' +``` + +**DBT Extension:** + +```yaml +- vendor_name: DBT + data: '{ + "project_name": "analytics", + "materialized": "table", + "tags": ["daily", "core"] + }' +``` + +--- + +## Complete Example + +Here's a complete semantic model example showing all components working together: + +```yaml +semantic_model: + - name: ecommerce_analytics + description: E-commerce sales and customer analytics + ai_context: + instructions: "Use this model for analyzing sales trends, customer behavior, and product performance" + + datasets: + - name: orders + source: sales.public.orders + primary_key: [order_id] + description: Customer orders + fields: + - name: order_id + expression: order_id + description: Order identifier + + - name: customer_id + expression: customer_id + description: Customer identifier + + - name: order_date + expression: order_date + dimension: + is_time: true + description: Order date + + - name: amount + expression: amount + description: Order amount + + - name: customers + source: sales.public.customers + primary_key: [id] + description: Customer information + fields: + - name: id + expression: id + description: Customer identifier + + - name: email + expression: email + description: Customer email + + relationships: + - name: orders_to_customers + from: orders + to: customers + from_columns: [customer_id] + to_columns: [id] + + metrics: + - name: total_revenue + expression: + - dialect: ANSI_SQL + aggregation_type: SUM + input_expr: orders.amount + description: Total revenue from all orders + ai_context: + synonyms: + - "total sales" + - "revenue" + + - name: customer_count + expression: + - dialect: ANSI_SQL + aggregation_type: COUNT_DISTINCT + input_expr: customers.id + description: Total number of customers + ai_context: + synonyms: + - "total customers" + - "customer base" + + custom_extensions: + - vendor_name: SNOWFLAKE + data: '{"warehouse": "ANALYTICS_WH"}' +``` + +--- + +## AI Context Structure + +The `ai_context` field can be either a simple string or a structured object with specific keys: + +**Simple String:** + +```yaml +ai_context: "orders, purchases, sales" +``` + +**Structured Object:** + +```yaml +ai_context: + instructions: "Use this for sales analysis" + synonyms: + - "orders" + - "purchases" + - "sales" + examples: + - "Show total sales last month" + - "What's the revenue by region?" +``` + +### Recommended AI Context Fields + +| Field | Type | Description | +|-------|------|-------------| +| `instructions` | string | Instructions for AI on how to use this entity | +| `synonyms` | array | Alternative names and terms | +| `examples` | array | Sample questions or use cases | + +--- + +## Version History + +- **1.0** (2024-12-11): Initial release + - Core semantic model structure + - Support for datasets, relationships, fields, and metrics + - Multi-dialect metric expressions + - Vendor extensibility framework + +--- + +## License + +See LICENSE file for details. + diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml new file mode 100644 index 00000000..772f80be --- /dev/null +++ b/core-spec/spec.yaml @@ -0,0 +1,222 @@ +# OSI - Core Metadata Spec (YAML Schema) +# Version: 1.0 +# +# Goals: +# - Standardization: Establish uniform language and structure for semantic model definitions +# - Extensibility: Support domain-specific extensions while maintaining core compatibility +# - Interoperability: Enable exchange and reuse across different AI and BI applications + +--- +# Enumerations +# Standard enums used throughout the specification + +# Supported SQL and expression language dialects +dialects: + - "ANSI_SQL" # Standard SQL dialect + - "SNOWFLAKE_SQL" # Snowflake + - "MDX" # Multi-Dimensional Expressions + - "TABLEAU" # Tableau + - "BIGQUERY_SQL" # Google BigQuery + - "REDSHIFT_SQL" # Amazon Redshift + - "DATABRICKS_SQL" # Databricks + - "POSTGRESQL" # PostgreSQL + - "MYSQL" # MySQL + + + +# Supported vendors for custom extensions +vendors: + - "COMMON" + - "SNOWFLAKE" + - "SALESFORCE" + - "DBT" + + +# Standard aggregation types for structured metrics +aggregation_types: + # Basic aggregations + - "SUM" + - "AVG" + - "COUNT" + - "COUNT_DISTINCT" + - "MIN" + - "MAX" + + +# Top-level semantic model definition +semantic_model: + # Required: Unique identifier for the semantic model + - name: string + + # Optional: Human-readable description of the semantic model + description: string + + # Optional: Additional context for AI tools (e.g., custom prompts, instructions) + ai_context: string + + # Required: Collection of logical datasets(fact and dimension tables) + # See Logical Dataset section below for detailed structure + datasets: [] + + # Optional: Defines how logical datasets are connected + # See Relationships section below for detailed structure + relationships:[] + + # Optional: + # These metrics can span one or more logical datasets and use relationships + # See Metrics section below for detailed structure + metrics: [] + + # Optional: Vendor-specific attributes for extensibility + # Allows vendors to add custom metadata without breaking core compatibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Logical Dataset Schema +# Represents business entities or concepts (fact and dimension tables) +# Fields are defined within the scope of a logical dataset +datasets: + # Required: Unique identifier for the logical dataset + - name: string + + # Required: Reference to the underlying physical table/view or query + # Format should be either database_name.schema_name.table_name or query + source: string + + # Optional: Primary key definition that uniquely identifies rows in this dataset + # Can be a single column or a composite of multiple columns + # This is the preferred unique identifier for this dataset and is used in relationships to determine many-to-one or one-to-one. + # Examples: + # - [customer_id] # Simple primary key + # - [order_id, line_number] # Composite primary key + primary_key: [] # Array of column names (single or composite) + + # Optional: Array of unique key definitions that uniquely identify rows in this dataset + # Each unique key can be a single column or a composite of multiple columns + # Used for determining relationship type of either many-to-one or one-to-one + # Examples: + # - [column1] # Simple key on one column + # - [column2, column3] # Composite key on two columns + unique_keys: + - [] # Array of column names (single or composite) + + # Optional: Human-readable description of the logical dataset + description: string + + # Optional: Additional context for AI tools (e.g., synonyms, common terms) + # Helps LLMs understand the business meaning and generate better queries + ai_context: string + + # Optional: Row-level calculations for grouping, filtering, and in metric expressions + # See Fields section below for detailed structure + fields: [] + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Relationship Schema +# Defines how logical datasets or semantic models are connected +# Represents foreign key relationships (many-to-one or one-to-one) +relationships: + # Required: Unique identifier for the relationship + - name: string + + # Required: The logical dataset on the many side of the relationship + # References a logical dataset name + from: string + + # Required: The logical dataset on the one side of the relationship + # References a logical dataset name + to: string + + # Required: Array of column names in the "from" dataset (foreign key columns) + # For simple relationships, use a single column: [column1] + # For composite relationships, use multiple columns: [column1, column2] + # The order of columns must correspond to the order in to_columns + # Examples: + # - [customer_id] # Simple foreign key + # - [order_id, line_number] # Composite foreign key + from_columns: [] # Array of column names + + # Required: Array of column names in the "to" dataset (primary or unique key columns) + # Must have the same number of columns as from_columns in corresponding order + # Examples: + # - [id] # Simple key + # - [order_id, line_number] # Composite key + to_columns: [] # Array of column names + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Fields Schema +# Represents row-level attributes that can be used for grouping, filtering, and metric expressions +fields: + # Required: Unique identifier for the field within the logical dataset + - name: string + + # Required: SQL expression that defines how to compute this field + # Can be a simple column reference or a complex scalar expression + expression: string + + # Optional: Dimension metadata + # Indicates this field can be used as a dimension for grouping/filtering + dimension: + # Optional: Indicates if this is a time-based dimension + # Used for time-series analysis and temporal filtering + is_time: boolean + + # Optional: Label for categorization (e.g., "filter") + label: string + + # Optional: Human-readable description of the field + description: string + + # Optional: Additional context for AI tools (e.g., synonyms, business terms) + # Helps LLMs understand the field meaning and generate better queries + ai_context: string + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Metrics Schema +# Quantitative measures defined on business data +# Represents key calculations like sums, averages, ratios, etc. +metrics: + # Required: Unique identifier for the metric within the logical dataset + - name: string + + # Required: Expression definition with dialect support + # Supports multiple SQL dialects for cross-platform compatibility + expression: + - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" + # Must be one of the following two options + # Option 1: Full expression with aggregate function + expression: string # e.g., "SUM(sales)", "AVG(revenue)" + + # Option 2: Structured + aggregation_type: string # Must be one of the values from 'aggregation_types' enum above + input_expr: string # Must be a scalar expressions e.g., "sales", "revenue", "quantity" + + # Optional: Human-readable description of the metric + # Should explain what the metric measures and how it's used + description: string + + # Optional: Additional context for AI tools (e.g., synonyms, business context) + # Helps LLMs understand the metric meaning and suggest it appropriately + ai_context: string + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml new file mode 100644 index 00000000..56bc5579 --- /dev/null +++ b/examples/tpcds_semantic_model.yaml @@ -0,0 +1,434 @@ +# TPC-DS Semantic Model Example +# This example demonstrates the OSI Core Metadata Spec using the TPC-DS benchmark schema +# TPC-DS is a decision support benchmark with a realistic retail business model + +semantic_model: + - name: tpcds_retail_model + description: TPC-DS retail semantic model for sales and customer analytics + ai_context: + prompt: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." + + datasets: + # Fact table: Store sales transactions + - name: store_sales + source: tpcds.public.store_sales + primary_key: [ss_item_sk, ss_ticket_number] # Composite primary key + unique_keys: + - [ss_item_sk, ss_ticket_number] # Composite key: item + ticket number uniquely identifies a line item + description: Fact table containing all store sales transactions + ai_context: + synonyms: + - "sales transactions" + - "store purchases" + - "retail sales" + - "POS data" + + fields: + - name: ss_sold_date_sk + expression: ss_sold_date_sk + description: Foreign key to date dimension + ai_context: + synonyms: + - "sale date" + - "transaction date" + + - name: ss_item_sk + expression: ss_item_sk + description: Foreign key to item dimension + ai_context: + synonyms: + - "product" + - "item" + + - name: ss_customer_sk + expression: ss_customer_sk + description: Foreign key to customer dimension + ai_context: + synonyms: + - "customer" + - "buyer" + + - name: ss_store_sk + expression: ss_store_sk + description: Foreign key to store dimension + ai_context: + synonyms: + - "store" + - "location" + + - name: ss_quantity + expression: ss_quantity + description: Quantity of items sold + ai_context: + synonyms: + - "units sold" + - "quantity" + + - name: ss_sales_price + expression: ss_sales_price + description: Sales price per unit + ai_context: + synonyms: + - "unit price" + - "price" + + - name: ss_ext_sales_price + expression: ss_ext_sales_price + description: Extended sales price (quantity * price) + ai_context: + synonyms: + - "total price" + - "line total" + + - name: ss_net_profit + expression: ss_net_profit + description: Net profit from the sale + ai_context: + synonyms: + - "profit" + - "margin" + + # Dimension table: Date + - name: date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] # Simple primary key + unique_keys: + - [d_date_sk] # Simple key: single column + description: Date dimension with calendar attributes + ai_context: + synonyms: + - "calendar" + - "dates" + - "time periods" + + fields: + - name: d_date_sk + expression: d_date_sk + description: Surrogate key for date + + - name: d_date + expression: d_date + description: Actual date value + dimension: + is_time: true + ai_context: + synonyms: + - "date" + - "calendar date" + + - name: d_year + expression: d_year + description: Year + dimension: + is_time: true + ai_context: + synonyms: + - "year" + + - name: d_quarter_name + expression: d_quarter_name + description: Quarter name (e.g., 2024Q1) + dimension: + is_time: true + ai_context: + synonyms: + - "quarter" + - "fiscal quarter" + + - name: d_month_name + expression: d_month_name + description: Month name + dimension: + is_time: true + ai_context: + synonyms: + - "month" + + # Dimension table: Customer + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] # Simple primary key + unique_keys: + - [c_customer_sk] # Simple key: single column + description: Customer dimension with demographic information + ai_context: + synonyms: + - "customers" + - "shoppers" + - "buyers" + + fields: + - name: c_customer_sk + expression: c_customer_sk + description: Surrogate key for customer + + - name: c_customer_id + expression: c_customer_id + description: Business key for customer + ai_context: + synonyms: + - "customer ID" + - "customer number" + + - name: c_first_name + expression: c_first_name + description: Customer first name + + - name: c_last_name + expression: c_last_name + description: Customer last name + + - name: customer_full_name + expression: c_first_name || ' ' || c_last_name + description: Customer full name (computed field) + ai_context: + synonyms: + - "full name" + - "customer name" + + - name: c_email_address + expression: c_email_address + description: Customer email address + ai_context: + synonyms: + - "email" + - "contact" + + # Dimension table: Item (Product) + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] # Simple primary key + unique_keys: + - [i_item_sk] # Simple key: single column + description: Item/Product dimension with product attributes + ai_context: + synonyms: + - "products" + - "items" + - "merchandise" + + fields: + - name: i_item_sk + expression: i_item_sk + description: Surrogate key for item + + - name: i_item_id + expression: i_item_id + description: Business key for item + ai_context: + synonyms: + - "item ID" + - "product ID" + - "SKU" + + - name: i_item_desc + expression: i_item_desc + description: Item description + ai_context: + synonyms: + - "product description" + - "item name" + + - name: i_brand + expression: i_brand + description: Brand name + ai_context: + synonyms: + - "brand" + - "manufacturer" + + - name: i_category + expression: i_category + description: Item category + ai_context: + synonyms: + - "product category" + - "department" + + - name: i_current_price + expression: i_current_price + description: Current price of the item + ai_context: + synonyms: + - "price" + - "list price" + + # Dimension table: Store + - name: store + source: tpcds.public.store + primary_key: [s_store_sk] # Simple primary key + unique_keys: + - [s_store_sk] # Simple key: single column + description: Store dimension with location and store attributes + ai_context: + synonyms: + - "stores" + - "retail locations" + - "branches" + + fields: + - name: s_store_sk + expression: s_store_sk + description: Surrogate key for store + + - name: s_store_id + expression: s_store_id + description: Business key for store + ai_context: + synonyms: + - "store ID" + - "store number" + + - name: s_store_name + expression: s_store_name + description: Store name + ai_context: + synonyms: + - "store name" + - "location name" + + - name: s_city + expression: s_city + description: City where store is located + ai_context: + synonyms: + - "city" + - "location" + + - name: s_state + expression: s_state + description: State where store is located + ai_context: + synonyms: + - "state" + - "region" + + - name: s_number_employees + expression: s_number_employees + description: Number of employees at the store + ai_context: + synonyms: + - "employee count" + - "staff size" + + # Relationships between datasets + relationships: + - name: store_sales_to_date + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] + ai_context: + synonyms: + - "sales date relationship" + - "when sale occurred" + + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] + ai_context: + synonyms: + - "customer purchase relationship" + - "who bought" + + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] + ai_context: + synonyms: + - "product sold relationship" + - "what was sold" + + - name: store_sales_to_store + from: store_sales + to: store + from_columns: [ss_store_sk] + to_columns: [s_store_sk] + ai_context: + synonyms: + - "store location relationship" + - "where sale occurred" + + # Semantic model-level metrics spanning multiple datasets + metrics: + - name: total_sales + expression: + - dialect: ANSI_SQL + aggregation_type: SUM + input_expr: store_sales.ss_ext_sales_price + description: Total sales revenue across all transactions + ai_context: + synonyms: + - "total revenue" + - "gross sales" + - "sales amount" + + - name: total_profit + expression: + - dialect: ANSI_SQL + aggregation_type: SUM + input_expr: store_sales.ss_net_profit + description: Total net profit from store sales + ai_context: + synonyms: + - "net profit" + - "total earnings" + - "profit" + + - name: customer_lifetime_value + expression: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) + description: Average lifetime sales value per customer + ai_context: + synonyms: + - "CLV" + - "LTV" + - "customer value" + - "lifetime revenue" + + - name: sales_by_brand + expression: + - dialect: ANSI_SQL + aggregation_type: SUM + input_expr: store_sales.ss_ext_sales_price + description: Total sales by brand (requires grouping by item.i_brand) + ai_context: + synonyms: + - "brand sales" + - "brand performance" + - "brand revenue" + + - name: store_productivity + expression: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) + description: Sales per employee across stores + ai_context: + synonyms: + - "sales per employee" + - "employee productivity" + - "revenue per employee" + + custom_extensions: + - vendor_name: SALESFORCE + data: '{ + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "1.0" + } + }' + + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' From b1c505a595d31fb0cfd77680dabc0a9833e41ce7 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Fri, 12 Dec 2025 10:59:06 -0800 Subject: [PATCH 2/9] first version of the spec --- core-spec/spec.md | 17 +---------------- core-spec/spec.yaml | 30 ++++-------------------------- 2 files changed, 5 insertions(+), 42 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index 2eca7686..40488389 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -46,21 +46,6 @@ Supported vendors for custom extensions and integrations. | `SALESFORCE` | Salesforce/Tableau-specific attributes | | `DBT` | dbt-specific attributes | -### Aggregation Types - -Standard aggregation functions for structured metric definitions. - -| Type | Description | -|------|-------------| -| `SUM` | Sum aggregation | -| `AVG` | Average aggregation | -| `COUNT` | Count aggregation | -| `COUNT_DISTINCT` | Distinct count aggregation | -| `MIN` | Minimum value | -| `MAX` | Maximum value | - ---- - ## Semantic Model The top-level container that represents a complete semantic model, including datasets, relationships, and metrics. @@ -266,7 +251,7 @@ Fields represent row-level attributes that can be used for grouping, filtering, ## Metrics -Quantitative measures defined on business data, representing key calculations like sums, averages, ratios, etc. Metrics can be defined at the semantic model level and can span multiple datasets. +Quantitative measures defined on business data, representing key calculations like sums, averages, ratios, etc. Metrics are defined at the semantic model level and can span multiple datasets. ### Schema diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 772f80be..a703b864 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -10,17 +10,11 @@ # Enumerations # Standard enums used throughout the specification -# Supported SQL and expression language dialects +# Supported expression language dialects dialects: - "ANSI_SQL" # Standard SQL dialect - - "SNOWFLAKE_SQL" # Snowflake + - "SNOWFLAKE" # Snowflake - "MDX" # Multi-Dimensional Expressions - - "TABLEAU" # Tableau - - "BIGQUERY_SQL" # Google BigQuery - - "REDSHIFT_SQL" # Amazon Redshift - - "DATABRICKS_SQL" # Databricks - - "POSTGRESQL" # PostgreSQL - - "MYSQL" # MySQL @@ -31,17 +25,6 @@ vendors: - "SALESFORCE" - "DBT" - -# Standard aggregation types for structured metrics -aggregation_types: - # Basic aggregations - - "SUM" - - "AVG" - - "COUNT" - - "COUNT_DISTINCT" - - "MIN" - - "MAX" - # Top-level semantic model definition semantic_model: @@ -196,18 +179,13 @@ metrics: # Required: Unique identifier for the metric within the logical dataset - name: string - # Required: Expression definition with dialect support + # Required: Expression definition # Supports multiple SQL dialects for cross-platform compatibility expression: + dialects: - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" - # Must be one of the following two options - # Option 1: Full expression with aggregate function expression: string # e.g., "SUM(sales)", "AVG(revenue)" - # Option 2: Structured - aggregation_type: string # Must be one of the values from 'aggregation_types' enum above - input_expr: string # Must be a scalar expressions e.g., "sales", "revenue", "quantity" - # Optional: Human-readable description of the metric # Should explain what the metric measures and how it's used description: string From 78a7fdf830574186006d52f83dab298eb7c3965e Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Fri, 12 Dec 2025 11:37:09 -0800 Subject: [PATCH 3/9] First version of the spec --- core-spec/spec.md | 22 ++++++++++------------ core-spec/spec.yaml | 12 +++++++----- examples/tpcds_semantic_model.yaml | 28 +++++++++++++++------------- 3 files changed, 32 insertions(+), 30 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index 40488389..85633d80 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -265,14 +265,13 @@ Quantitative measures defined on business data, representing key calculations li ### Expression Object -The expression object supports two formats: - -**Option 1: Full SQL Expression** +The expression object supports multiple dialects ```yaml expression: + dialects: - dialect: ANSI_SQL # Default - expression: "SUM(sales) / COUNT(DISTINCT customer_id)" + expression: "SUM(order.sales) / COUNT(DISTINCT order.customer_id)" ``` @@ -284,8 +283,7 @@ expression: - name: total_revenue expression: - dialect: ANSI_SQL - aggregation_type: SUM - input_expr: order_amount + expression: SUM(orders.amount) description: Total revenue across all orders ai_context: synonyms: @@ -433,9 +431,9 @@ semantic_model: metrics: - name: total_revenue expression: - - dialect: ANSI_SQL - aggregation_type: SUM - input_expr: orders.amount + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) description: Total revenue from all orders ai_context: synonyms: @@ -444,9 +442,9 @@ semantic_model: - name: customer_count expression: - - dialect: ANSI_SQL - aggregation_type: COUNT_DISTINCT - input_expr: customers.id + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT customers.id) description: Total number of customers ai_context: synonyms: diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index a703b864..5512c955 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -15,6 +15,7 @@ dialects: - "ANSI_SQL" # Standard SQL dialect - "SNOWFLAKE" # Snowflake - "MDX" # Multi-Dimensional Expressions + - "TABLEAU" # Tableau @@ -176,15 +177,16 @@ fields: # Quantitative measures defined on business data # Represents key calculations like sums, averages, ratios, etc. metrics: - # Required: Unique identifier for the metric within the logical dataset + # Required: Unique identifier for the metric - name: string - # Required: Expression definition + # Required: Expression definition with dialect support # Supports multiple SQL dialects for cross-platform compatibility + # Each metric can have expressions in different dialects for portability expression: - dialects: - - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" - expression: string # e.g., "SUM(sales)", "AVG(revenue)" + dialects: + - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" + expression: string # Full SQL expression with aggregate functions, e.g., "SUM(orders.sales)", "AVG(orders.amount)" # Optional: Human-readable description of the metric # Should explain what the metric measures and how it's used diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index 56bc5579..7d9418a8 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -357,9 +357,9 @@ semantic_model: metrics: - name: total_sales expression: - - dialect: ANSI_SQL - aggregation_type: SUM - input_expr: store_sales.ss_ext_sales_price + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) description: Total sales revenue across all transactions ai_context: synonyms: @@ -369,9 +369,9 @@ semantic_model: - name: total_profit expression: - - dialect: ANSI_SQL - aggregation_type: SUM - input_expr: store_sales.ss_net_profit + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_net_profit) description: Total net profit from store sales ai_context: synonyms: @@ -381,8 +381,9 @@ semantic_model: - name: customer_lifetime_value expression: - - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) description: Average lifetime sales value per customer ai_context: synonyms: @@ -393,9 +394,9 @@ semantic_model: - name: sales_by_brand expression: - - dialect: ANSI_SQL - aggregation_type: SUM - input_expr: store_sales.ss_ext_sales_price + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) description: Total sales by brand (requires grouping by item.i_brand) ai_context: synonyms: @@ -405,8 +406,9 @@ semantic_model: - name: store_productivity expression: - - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) description: Sales per employee across stores ai_context: synonyms: From 58bace67cfc9121518c180ec4239bc9d82978b92 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Fri, 12 Dec 2025 12:36:59 -0800 Subject: [PATCH 4/9] Add support for expression object with dialects for fields --- core-spec/spec.md | 100 +++++++++++++----- core-spec/spec.yaml | 9 +- examples/tpcds_semantic_model.yaml | 157 +++++++++++++++++++++++------ 3 files changed, 204 insertions(+), 62 deletions(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index 85633d80..cd898030 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -196,13 +196,31 @@ Fields represent row-level attributes that can be used for grouping, filtering, | Field | Type | Required | Description | |-------|------|----------|-------------| | `name` | string | Yes | Unique identifier for the field within the dataset | -| `expression` | string | Yes | SQL expression that defines how to compute this field | +| `expression` | object | Yes | Expression definition with dialect support | | `dimension` | object | No | Dimension metadata (e.g., `is_time` flag) | | `label` | string | No | Label for categorization | | `description` | string | No | Human-readable description | | `ai_context` | string/object | No | Additional context for AI tools (e.g., synonyms) | | `custom_extensions` | array | No | Vendor-specific attributes | +### Expression Object + +The expression object supports multiple SQL dialects for cross-platform compatibility. Each field can define expressions in different dialects. + +**Structure:** + +```yaml +expression: + dialects: + - dialect: ANSI_SQL # Must be one of the dialects enum values + expression: "customer_id" # Scalar SQL expression +``` + +**Key Points:** +- Use scalar SQL expressions (no aggregations) +- Can be simple column references (e.g., `customer_id`) or computed expressions (e.g., `first_name || ' ' || last_name`) +- Multiple dialect versions can be provided for the same field + ### Dimension Object | Field | Type | Description | @@ -215,7 +233,10 @@ Fields represent row-level attributes that can be used for grouping, filtering, ```yaml - name: customer_id - expression: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id description: Customer identifier dimension: is_time: false @@ -225,7 +246,10 @@ Fields represent row-level attributes that can be used for grouping, filtering, ```yaml - name: full_name - expression: first_name || ' ' || last_name + expression: + dialects: + - dialect: ANSI_SQL + expression: first_name || ' ' || last_name description: Customer full name ai_context: synonyms: @@ -237,7 +261,10 @@ Fields represent row-level attributes that can be used for grouping, filtering, ```yaml - name: order_date - expression: order_date + expression: + dialects: + - dialect: ANSI_SQL + expression: order_date dimension: is_time: true description: Date when order was placed @@ -247,6 +274,19 @@ Fields represent row-level attributes that can be used for grouping, filtering, - "transaction date" ``` +**Multi-Dialect Field:** + +```yaml +- name: email_normalized + expression: + dialects: + - dialect: ANSI_SQL + expression: LOWER(email) + - dialect: SNOWFLAKE + expression: LOWER(email)::VARCHAR + description: Normalized email address +``` + --- ## Metrics @@ -291,32 +331,17 @@ expression: - "revenue" ``` -**Complex Calculation:** - -```yaml -- name: average_order_value - expression: - - dialect: ANSI_SQL - expression: SUM(orders.amount) / COUNT(DISTINCT orders.order_id) - description: Average value per order - ai_context: - synonyms: - - "AOV" - - "avg order size" -``` - **Cross-Dataset Metric:** ```yaml -- name: customer_lifetime_value +- name: avg_orders expression: - dialect: ANSI_SQL expression: SUM(orders.amount) / COUNT(DISTINCT customers.id) - description: Average lifetime value per customer + description: Average orders ai_context: synonyms: - - "CLV" - - "LTV" + - "Order Average by customer" ``` --- @@ -391,21 +416,33 @@ semantic_model: description: Customer orders fields: - name: order_id - expression: order_id + expression: + dialects: + - dialect: ANSI_SQL + expression: order_id description: Order identifier - name: customer_id - expression: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id description: Customer identifier - name: order_date - expression: order_date + expression: + dialects: + - dialect: ANSI_SQL + expression: order_date dimension: is_time: true description: Order date - name: amount - expression: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount description: Order amount - name: customers @@ -414,11 +451,17 @@ semantic_model: description: Customer information fields: - name: id - expression: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id description: Customer identifier - name: email - expression: email + expression: + dialects: + - dialect: ANSI_SQL + expression: email description: Customer email relationships: @@ -499,6 +542,7 @@ ai_context: - Support for datasets, relationships, fields, and metrics - Multi-dialect metric expressions - Vendor extensibility framework + - Context for agents --- diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 5512c955..6c0f3794 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -146,9 +146,14 @@ fields: # Required: Unique identifier for the field within the logical dataset - name: string - # Required: SQL expression that defines how to compute this field + # Required: Expression definition with dialect support + # Supports multiple SQL dialects for cross-platform compatibility + # Each field can have expressions in different dialects for portability # Can be a simple column reference or a complex scalar expression - expression: string + expression: + dialects: + - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" + expression: string # SQL scalar expression, e.g., "customer_id", "first_name || ' ' || last_name", "UPPER(email)" # Optional: Dimension metadata # Indicates this field can be used as a dimension for grouping/filtering diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index 7d9418a8..a1cb3cb3 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -6,7 +6,7 @@ semantic_model: - name: tpcds_retail_model description: TPC-DS retail semantic model for sales and customer analytics ai_context: - prompt: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." + instructions: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." datasets: # Fact table: Store sales transactions @@ -25,7 +25,10 @@ semantic_model: fields: - name: ss_sold_date_sk - expression: ss_sold_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sold_date_sk description: Foreign key to date dimension ai_context: synonyms: @@ -33,7 +36,10 @@ semantic_model: - "transaction date" - name: ss_item_sk - expression: ss_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_item_sk description: Foreign key to item dimension ai_context: synonyms: @@ -41,7 +47,10 @@ semantic_model: - "item" - name: ss_customer_sk - expression: ss_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_customer_sk description: Foreign key to customer dimension ai_context: synonyms: @@ -49,7 +58,10 @@ semantic_model: - "buyer" - name: ss_store_sk - expression: ss_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_store_sk description: Foreign key to store dimension ai_context: synonyms: @@ -57,7 +69,10 @@ semantic_model: - "location" - name: ss_quantity - expression: ss_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_quantity description: Quantity of items sold ai_context: synonyms: @@ -65,7 +80,10 @@ semantic_model: - "quantity" - name: ss_sales_price - expression: ss_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sales_price description: Sales price per unit ai_context: synonyms: @@ -73,7 +91,10 @@ semantic_model: - "price" - name: ss_ext_sales_price - expression: ss_ext_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ext_sales_price description: Extended sales price (quantity * price) ai_context: synonyms: @@ -81,7 +102,10 @@ semantic_model: - "line total" - name: ss_net_profit - expression: ss_net_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_net_profit description: Net profit from the sale ai_context: synonyms: @@ -103,11 +127,17 @@ semantic_model: fields: - name: d_date_sk - expression: d_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date_sk description: Surrogate key for date - name: d_date - expression: d_date + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date description: Actual date value dimension: is_time: true @@ -117,7 +147,10 @@ semantic_model: - "calendar date" - name: d_year - expression: d_year + expression: + dialects: + - dialect: ANSI_SQL + expression: d_year description: Year dimension: is_time: true @@ -126,7 +159,10 @@ semantic_model: - "year" - name: d_quarter_name - expression: d_quarter_name + expression: + dialects: + - dialect: ANSI_SQL + expression: d_quarter_name description: Quarter name (e.g., 2024Q1) dimension: is_time: true @@ -136,7 +172,10 @@ semantic_model: - "fiscal quarter" - name: d_month_name - expression: d_month_name + expression: + dialects: + - dialect: ANSI_SQL + expression: d_month_name description: Month name dimension: is_time: true @@ -159,11 +198,17 @@ semantic_model: fields: - name: c_customer_sk - expression: c_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_sk description: Surrogate key for customer - name: c_customer_id - expression: c_customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_id description: Business key for customer ai_context: synonyms: @@ -171,15 +216,24 @@ semantic_model: - "customer number" - name: c_first_name - expression: c_first_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name description: Customer first name - name: c_last_name - expression: c_last_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_last_name description: Customer last name - name: customer_full_name - expression: c_first_name || ' ' || c_last_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name || ' ' || c_last_name description: Customer full name (computed field) ai_context: synonyms: @@ -187,7 +241,10 @@ semantic_model: - "customer name" - name: c_email_address - expression: c_email_address + expression: + dialects: + - dialect: ANSI_SQL + expression: c_email_address description: Customer email address ai_context: synonyms: @@ -209,11 +266,17 @@ semantic_model: fields: - name: i_item_sk - expression: i_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_sk description: Surrogate key for item - name: i_item_id - expression: i_item_id + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_id description: Business key for item ai_context: synonyms: @@ -222,7 +285,10 @@ semantic_model: - "SKU" - name: i_item_desc - expression: i_item_desc + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_desc description: Item description ai_context: synonyms: @@ -230,7 +296,10 @@ semantic_model: - "item name" - name: i_brand - expression: i_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: i_brand description: Brand name ai_context: synonyms: @@ -238,7 +307,10 @@ semantic_model: - "manufacturer" - name: i_category - expression: i_category + expression: + dialects: + - dialect: ANSI_SQL + expression: i_category description: Item category ai_context: synonyms: @@ -246,7 +318,10 @@ semantic_model: - "department" - name: i_current_price - expression: i_current_price + expression: + dialects: + - dialect: ANSI_SQL + expression: i_current_price description: Current price of the item ai_context: synonyms: @@ -268,11 +343,17 @@ semantic_model: fields: - name: s_store_sk - expression: s_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_sk description: Surrogate key for store - name: s_store_id - expression: s_store_id + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_id description: Business key for store ai_context: synonyms: @@ -280,7 +361,10 @@ semantic_model: - "store number" - name: s_store_name - expression: s_store_name + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_name description: Store name ai_context: synonyms: @@ -288,7 +372,10 @@ semantic_model: - "location name" - name: s_city - expression: s_city + expression: + dialects: + - dialect: ANSI_SQL + expression: s_city description: City where store is located ai_context: synonyms: @@ -296,7 +383,10 @@ semantic_model: - "location" - name: s_state - expression: s_state + expression: + dialects: + - dialect: ANSI_SQL + expression: s_state description: State where store is located ai_context: synonyms: @@ -304,7 +394,10 @@ semantic_model: - "region" - name: s_number_employees - expression: s_number_employees + expression: + dialects: + - dialect: ANSI_SQL + expression: s_number_employees description: Number of employees at the store ai_context: synonyms: From 4152b07b5b431021c82dfe59968167196de3ac08 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Fri, 12 Dec 2025 12:52:51 -0800 Subject: [PATCH 5/9] First version of the spec --- core-spec/spec.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-spec/spec.md b/core-spec/spec.md index cd898030..83576fb8 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -229,7 +229,7 @@ expression: ### Examples -**Simple Column Reference:** +**Simple Column Reference for a Dimension:** ```yaml - name: customer_id From 36e0d434b1dde105a94cd5440a69e587e52e630f Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Fri, 12 Dec 2025 12:56:55 -0800 Subject: [PATCH 6/9] tpcds example --- examples/tpcds_semantic_model.yaml | 46 +++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index a1cb3cb3..511a79d9 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -30,6 +30,8 @@ semantic_model: - dialect: ANSI_SQL expression: ss_sold_date_sk description: Foreign key to date dimension + dimension: + is_time: false ai_context: synonyms: - "sale date" @@ -41,6 +43,8 @@ semantic_model: - dialect: ANSI_SQL expression: ss_item_sk description: Foreign key to item dimension + dimension: + is_time: false ai_context: synonyms: - "product" @@ -52,6 +56,8 @@ semantic_model: - dialect: ANSI_SQL expression: ss_customer_sk description: Foreign key to customer dimension + dimension: + is_time: false ai_context: synonyms: - "customer" @@ -63,6 +69,8 @@ semantic_model: - dialect: ANSI_SQL expression: ss_store_sk description: Foreign key to store dimension + dimension: + is_time: false ai_context: synonyms: - "store" @@ -132,6 +140,8 @@ semantic_model: - dialect: ANSI_SQL expression: d_date_sk description: Surrogate key for date + dimension: + is_time: false - name: d_date expression: @@ -203,6 +213,8 @@ semantic_model: - dialect: ANSI_SQL expression: c_customer_sk description: Surrogate key for customer + dimension: + is_time: false - name: c_customer_id expression: @@ -210,6 +222,8 @@ semantic_model: - dialect: ANSI_SQL expression: c_customer_id description: Business key for customer + dimension: + is_time: false ai_context: synonyms: - "customer ID" @@ -221,6 +235,8 @@ semantic_model: - dialect: ANSI_SQL expression: c_first_name description: Customer first name + dimension: + is_time: false - name: c_last_name expression: @@ -228,6 +244,8 @@ semantic_model: - dialect: ANSI_SQL expression: c_last_name description: Customer last name + dimension: + is_time: false - name: customer_full_name expression: @@ -235,6 +253,8 @@ semantic_model: - dialect: ANSI_SQL expression: c_first_name || ' ' || c_last_name description: Customer full name (computed field) + dimension: + is_time: false ai_context: synonyms: - "full name" @@ -246,6 +266,8 @@ semantic_model: - dialect: ANSI_SQL expression: c_email_address description: Customer email address + dimension: + is_time: false ai_context: synonyms: - "email" @@ -271,6 +293,8 @@ semantic_model: - dialect: ANSI_SQL expression: i_item_sk description: Surrogate key for item + dimension: + is_time: false - name: i_item_id expression: @@ -278,6 +302,8 @@ semantic_model: - dialect: ANSI_SQL expression: i_item_id description: Business key for item + dimension: + is_time: false ai_context: synonyms: - "item ID" @@ -290,6 +316,8 @@ semantic_model: - dialect: ANSI_SQL expression: i_item_desc description: Item description + dimension: + is_time: false ai_context: synonyms: - "product description" @@ -301,6 +329,8 @@ semantic_model: - dialect: ANSI_SQL expression: i_brand description: Brand name + dimension: + is_time: false ai_context: synonyms: - "brand" @@ -312,6 +342,8 @@ semantic_model: - dialect: ANSI_SQL expression: i_category description: Item category + dimension: + is_time: false ai_context: synonyms: - "product category" @@ -323,6 +355,8 @@ semantic_model: - dialect: ANSI_SQL expression: i_current_price description: Current price of the item + dimension: + is_time: false ai_context: synonyms: - "price" @@ -333,7 +367,7 @@ semantic_model: source: tpcds.public.store primary_key: [s_store_sk] # Simple primary key unique_keys: - - [s_store_sk] # Simple key: single column + - [s_store_id] # Simple key: single column description: Store dimension with location and store attributes ai_context: synonyms: @@ -348,6 +382,8 @@ semantic_model: - dialect: ANSI_SQL expression: s_store_sk description: Surrogate key for store + dimension: + is_time: false - name: s_store_id expression: @@ -355,6 +391,8 @@ semantic_model: - dialect: ANSI_SQL expression: s_store_id description: Business key for store + dimension: + is_time: false ai_context: synonyms: - "store ID" @@ -366,6 +404,8 @@ semantic_model: - dialect: ANSI_SQL expression: s_store_name description: Store name + dimension: + is_time: false ai_context: synonyms: - "store name" @@ -377,6 +417,8 @@ semantic_model: - dialect: ANSI_SQL expression: s_city description: City where store is located + dimension: + is_time: false ai_context: synonyms: - "city" @@ -388,6 +430,8 @@ semantic_model: - dialect: ANSI_SQL expression: s_state description: State where store is located + dimension: + is_time: false ai_context: synonyms: - "state" From 3a10952d3ca20cbd37db1b3af9f107ba433ca1b7 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Fri, 12 Dec 2025 15:07:04 -0800 Subject: [PATCH 7/9] Update core-spec/spec.yaml Co-authored-by: Hossein Ahmadi --- core-spec/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 6c0f3794..35bfb74e 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -47,7 +47,7 @@ semantic_model: relationships:[] # Optional: - # These metrics can span one or more logical datasets and use relationships + # These metrics can span one or more logical datasets and use relationships # See Metrics section below for detailed structure metrics: [] From 2403a619e9a52f3b741f84584836b804616e7f98 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Fri, 12 Dec 2025 15:07:18 -0800 Subject: [PATCH 8/9] Update core-spec/spec.yaml Co-authored-by: Hossein Ahmadi --- core-spec/spec.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 35bfb74e..76e6ed01 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -38,7 +38,7 @@ semantic_model: # Optional: Additional context for AI tools (e.g., custom prompts, instructions) ai_context: string - # Required: Collection of logical datasets(fact and dimension tables) + # Required: Collection of logical datasets (fact and dimension tables) # See Logical Dataset section below for detailed structure datasets: [] From 14f3b13d693f627a21f3ec0cb28cefc3adbbd6f3 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Fri, 12 Dec 2025 15:24:24 -0800 Subject: [PATCH 9/9] examples for unique --- core-spec/spec.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 76e6ed01..3671ca06 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -73,7 +73,10 @@ datasets: # Can be a single column or a composite of multiple columns # This is the preferred unique identifier for this dataset and is used in relationships to determine many-to-one or one-to-one. # Examples: + # primary_key: # - [customer_id] # Simple primary key + # + # primary_key: # - [order_id, line_number] # Composite primary key primary_key: [] # Array of column names (single or composite) @@ -81,8 +84,13 @@ datasets: # Each unique key can be a single column or a composite of multiple columns # Used for determining relationship type of either many-to-one or one-to-one # Examples: - # - [column1] # Simple key on one column - # - [column2, column3] # Composite key on two columns + # unique_keys: + # - [column1] + # - [column2, column3] + # + # unique_keys: + # - [column1, column2] + # - [column3, column4] unique_keys: - [] # Array of column names (single or composite)