From 5059f2d3f389c267e710fc874c0931ec7c99bba0 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Wed, 10 Dec 2025 18:00:35 -0800 Subject: [PATCH 1/4] code --- core-spec/spec.yaml | 227 +++++++++++++ examples/tpcds_semantic_model.yaml | 507 +++++++++++++++++++++++++++++ 2 files changed, 734 insertions(+) create mode 100644 core-spec/spec.yaml create mode 100644 examples/tpcds_semantic_model.yaml diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml new file mode 100644 index 00000000..ce4ef97e --- /dev/null +++ b/core-spec/spec.yaml @@ -0,0 +1,227 @@ +# OSI - Core Metadata Spec (YAML Schema) +# Version: 1.0 +# +# Goals: +# - Standardization: Establish uniform language and structure for semantic model definitions +# - Extensibility: Support domain-specific extensions while maintaining core compatibility +# - Interoperability: Enable exchange and reuse across different AI and BI applications + +--- +# Enumerations +# Standard enums used throughout the specification + +# Supported SQL and expression language dialects +dialects: + - "ANSI SQL" # Standard SQL dialect + - "Snowflake SQL" # Snowflake + - "MDX" # Multi-Dimensional Expressions + - "Tableau" # Tableau + - "BigQuery SQL" # Google BigQuery + - "Redshift SQL" # Amazon Redshift + - "Databricks SQL" # Databricks + - "Azure Synapse SQL" # Azure Synapse + - "PostgreSQL" # PostgreSQL + - "MySQL" # MySQL + - "Oracle SQL" # Oracle + - "SQL Server" # Microsoft SQL Server + + + +# Supported vendors for custom extensions +vendors: + - "common" + - "snowflake" + - "tableau" + - "dbt" + + +# Standard aggregation types for structured metrics +aggregation_types: + # Basic aggregations + - "SUM" + - "AVG" + - "COUNT" + - "COUNT_DISTINCT" + - "MIN" + - "MAX" + + +# Top-level semantic model definition +semantic_model: + # Required: Unique identifier for the semantic model + - name: string + + # Optional: Human-readable description of the semantic model + description: string + + # Optional: Additional context for AI tools (e.g., custom prompts, instructions) + ai_context: string + + # Required: Collection of logical datasets(fact and dimension tables) + # See Logical Dataset section below for detailed structure + logical_datasets: + - name: string + base_table: string + primary_key: string + unique_keys: [] + description: string + fields: [] + ai_context: { + string: string + } + custom_extensions: {} + + # Required: Defines how logical datasets or semantic models are connected + # Represents foreign key relationships between datasets + relationships: + - name: string + from: string + to: string + from_column: string + to_column: string + custom_extensions: {} + + # Optional: metrics + # These metrics can span one or more logical datasets and use relationships + # See Metrics section below for detailed structure + metrics: + - name: string + expr: [] + description: string + ai_context: string + custom_extensions: + - vendor_name: string + data: string + + # Optional: Vendor-specific attributes for extensibility + # Allows vendors to add custom metadata without breaking core compatibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Logical Dataset Schema +# Represents business entities or concepts (fact and dimension tables) +# Fields are defined within the scope of a logical dataset +datasets: + # Required: Unique identifier for the logical dataset + - name: string + + # Required: Reference to the underlying physical table or view + # Format should include database.schema.table or similar qualified name + base_table: string + + # Optional: Array of column names that uniquely identify rows in this dataset + # Used for join optimization and data integrity validation + unique_keys: + - string + + # Optional: Human-readable description of the logical dataset + description: string + + # Optional: Additional context for AI tools (e.g., synonyms, common terms) + # Helps LLMs understand the business meaning and generate better queries + ai_context: string + + # Optional: Row-level calculations for grouping, filtering, and metric expressions + # See Fields section below for detailed structure + fields: [] + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Relationship Schema +# Defines how logical datasets or semantic models are connected +# Represents foreign key relationships (many-to-one or one-to-one) +relationships: + # Required: Unique identifier for the relationship + - name: string + + # Required: The logical dataset on the many side of the relationship + # References a logical dataset name + from: string + + # Required: The logical dataset on the one side of the relationship + # References a logical dataset name + to: string + + # Required: Column name in the "from" dataset (foreign key) + from_column: string + + # Required: Column name in the "to" dataset (primary key) + to_column: string + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Fields Schema +# Represents row-level attributes that can be used for grouping, filtering, and metric expressions +fields: + # Required: Unique identifier for the field within the logical dataset + - name: string + + # Required: SQL expression that defines how to compute this field + # Can be a simple column reference or a complex expression + expr: string + + # Optional: Dimension metadata + # Indicates this field can be used as a dimension for grouping/filtering + dimension: + # Optional: Indicates if this is a time-based dimension + # Used for time-series analysis and temporal filtering + is_time: boolean + + # Optional: Label for categorization (e.g., "filter") + label: string + + # Optional: Human-readable description of the field + description: string + + # Optional: Additional context for AI tools (e.g., synonyms, business terms) + # Helps LLMs understand the field meaning and generate better queries + ai_context: string + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Metrics Schema +# Quantitative measures defined on business data +# Represents key calculations like sums, averages, ratios, etc. +metrics: + # Required: Unique identifier for the metric within the logical dataset + - name: string + + # Required: Expression definition with dialect support + # Supports multiple SQL dialects for cross-platform compatibility + expr: + # Option 1: Full expression with dialect specification + - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI SQL" + expression: string # e.g., "SUM(sales)", "AVG(revenue)" + + # Option 2: Structured aggregation (for simple metrics) + # This provides a more standardized way to represent common aggregations + - dialect: string # Must be one of the values from 'dialects' enum above + aggregation_type: string # Must be one of the values from 'aggregation_types' enum above + input_expr: string # e.g., "sales", "revenue", "quantity" + + # Optional: Human-readable description of the metric + # Should explain what the metric measures and how it's used + description: string + + # Optional: Additional context for AI tools (e.g., synonyms, business context) + # Helps LLMs understand the metric meaning and suggest it appropriately + ai_context: string + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml new file mode 100644 index 00000000..0b5ebdd6 --- /dev/null +++ b/examples/tpcds_semantic_model.yaml @@ -0,0 +1,507 @@ +# TPC-DS Semantic Model Example +# This example demonstrates the OSI Core Metadata Spec using the TPC-DS benchmark schema +# TPC-DS is a decision support benchmark with a realistic retail business model + +semantic_model: + - name: tpcds_retail_model + description: TPC-DS retail semantic model for sales and customer analytics + ai_context: + prompt: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." + + logical_datasets: + # Fact table: Store sales transactions + - name: store_sales + base_table: tpcds.public.store_sales + unique_keys: + - ss_item_sk + - ss_ticket_number + description: Fact table containing all store sales transactions + ai_context: + synonyms: + - "sales transactions" + - "store purchases" + - "retail sales" + - "POS data" + - "point of sale" + + fields: + - name: ss_sold_date_sk + expr: ss_sold_date_sk + description: Foreign key to date dimension + ai_context: + synonyms: + - "sale date" + - "transaction date" + - "purchase date" + + - name: ss_item_sk + expr: ss_item_sk + description: Foreign key to item dimension + ai_context: + synonyms: + - "product" + - "item" + - "SKU" + - "product key" + + - name: ss_customer_sk + expr: ss_customer_sk + description: Foreign key to customer dimension + ai_context: + synonyms: + - "customer" + - "buyer" + - "purchaser" + + - name: ss_store_sk + expr: ss_store_sk + description: Foreign key to store dimension + ai_context: + synonyms: + - "store location" + - "retail location" + - "store" + + - name: ss_quantity + expr: ss_quantity + description: Quantity of items sold + ai_context: + synonyms: + - "units sold" + - "quantity purchased" + - "item count" + - "units" + + - name: ss_sales_price + expr: ss_sales_price + description: Sales price per unit + ai_context: + synonyms: + - "unit price" + - "price per item" + - "item price" + + - name: ss_ext_sales_price + expr: ss_ext_sales_price + description: Extended sales price (quantity * price) + ai_context: + synonyms: + - "total price" + - "line item total" + - "extended price" + - "total amount" + + - name: ss_net_profit + expr: ss_net_profit + description: Net profit from the sale + ai_context: + synonyms: + - "profit" + - "margin" + - "earnings" + - "net earnings" + + # Dimension table: Date + - name: date_dim + base_table: tpcds.public.date_dim + unique_keys: + - d_date_sk + description: Date dimension with calendar attributes + ai_context: + synonyms: + - "calendar" + - "dates" + - "time periods" + - "date table" + + fields: + - name: d_date_sk + expr: d_date_sk + description: Surrogate key for date + dimension: + is_time: true + + - name: d_date + expr: d_date + description: Actual date value + dimension: + is_time: true + ai_context: + synonyms: + - "calendar date" + - "date" + + - name: d_year + expr: d_year + description: Year + dimension: + is_time: true + ai_context: + synonyms: + - "year" + - "calendar year" + + - name: d_quarter_name + expr: d_quarter_name + description: Quarter name (e.g., 2024Q1) + dimension: + is_time: true + ai_context: + synonyms: + - "quarter" + - "fiscal quarter" + - "Q1, Q2, Q3, Q4" + + - name: d_month_name + expr: d_month_name + description: Month name + dimension: + is_time: true + ai_context: + synonyms: + - "month" + - "calendar month" + + - name: d_day_name + expr: d_day_name + description: Day of week name + dimension: + is_time: true + ai_context: + synonyms: + - "weekday" + - "day of week" + - "day name" + + # Dimension table: Customer + - name: customer + base_table: tpcds.public.customer + unique_keys: + - c_customer_sk + description: Customer dimension with demographic information + ai_context: + synonyms: + - "customers" + - "shoppers" + - "buyers" + - "consumer demographics" + - "purchasers" + + fields: + - name: c_customer_sk + expr: c_customer_sk + description: Surrogate key for customer + + - name: c_customer_id + expr: c_customer_id + description: Business key for customer + ai_context: + synonyms: + - "customer ID" + - "customer number" + - "customer identifier" + + - name: c_first_name + expr: c_first_name + description: Customer first name + + - name: c_last_name + expr: c_last_name + description: Customer last name + + - name: c_birth_year + expr: c_birth_year + description: Customer birth year + ai_context: + synonyms: + - "age" + - "birth year" + - "year of birth" + + - name: c_email_address + expr: c_email_address + description: Customer email address + ai_context: + synonyms: + - "email" + - "contact" + - "email address" + + - name: customer_full_name + expr: c_first_name || ' ' || c_last_name + description: Customer full name (computed field) + ai_context: + synonyms: + - "full name" + - "customer name" + - "name" + + # Dimension table: Item (Product) + - name: item + base_table: tpcds.public.item + unique_keys: + - i_item_sk + description: Item/Product dimension with product attributes + ai_context: + synonyms: + - "products" + - "items" + - "merchandise" + - "SKUs" + - "inventory" + - "catalog" + + fields: + - name: i_item_sk + expr: i_item_sk + description: Surrogate key for item + + - name: i_item_id + expr: i_item_id + description: Business key for item + ai_context: + synonyms: + - "item ID" + - "product ID" + - "SKU" + - "product code" + + - name: i_item_desc + expr: i_item_desc + description: Item description + ai_context: + synonyms: + - "product description" + - "item name" + - "product name" + + - name: i_brand + expr: i_brand + description: Brand name + ai_context: + synonyms: + - "brand" + - "manufacturer" + - "brand name" + + - name: i_category + expr: i_category + description: Item category + ai_context: + synonyms: + - "product category" + - "department" + - "category" + + - name: i_class + expr: i_class + description: Item class + ai_context: + synonyms: + - "product class" + - "product type" + - "class" + + - name: i_current_price + expr: i_current_price + description: Current price of the item + ai_context: + synonyms: + - "price" + - "MSRP" + - "list price" + - "current price" + + # Dimension table: Store + - name: store + base_table: tpcds.public.store + unique_keys: + - s_store_sk + description: Store dimension with location and store attributes + ai_context: + synonyms: + - "stores" + - "retail locations" + - "outlets" + - "branches" + - "locations" + + fields: + - name: s_store_sk + expr: s_store_sk + description: Surrogate key for store + + - name: s_store_id + expr: s_store_id + description: Business key for store + ai_context: + synonyms: + - "store ID" + - "store number" + - "location ID" + + - name: s_store_name + expr: s_store_name + description: Store name + ai_context: + synonyms: + - "store name" + - "location name" + - "branch name" + + - name: s_number_employees + expr: s_number_employees + description: Number of employees at the store + ai_context: + synonyms: + - "employee count" + - "staff size" + - "number of employees" + + - name: s_city + expr: s_city + description: City where store is located + ai_context: + synonyms: + - "city" + - "location" + - "store city" + + - name: s_state + expr: s_state + description: State where store is located + ai_context: + synonyms: + - "state" + - "region" + - "province" + + - name: s_zip + expr: s_zip + description: ZIP code of store location + ai_context: + synonyms: + - "zip code" + - "postal code" + - "ZIP" + + # Relationships between logical datasets + relationships: + - name: store_sales_to_date + from: store_sales + to: date_dim + from_column: ss_sold_date_sk + to_column: d_date_sk + ai_context: + synonyms: + - "sales date relationship" + - "when sale occurred" + + - name: store_sales_to_customer + from: store_sales + to: customer + from_column: ss_customer_sk + to_column: c_customer_sk + ai_context: + synonyms: + - "customer purchase relationship" + - "who bought" + - "purchaser relationship" + + - name: store_sales_to_item + from: store_sales + to: item + from_column: ss_item_sk + to_column: i_item_sk + ai_context: + synonyms: + - "product sold relationship" + - "what was sold" + - "item purchase relationship" + + - name: store_sales_to_store + from: store_sales + to: store + from_column: ss_store_sk + to_column: s_store_sk + ai_context: + synonyms: + - "store location relationship" + - "where sale occurred" + - "sales location relationship" + + # Semantic model-level metrics spanning multiple datasets + metrics: + - name: customer_lifetime_value + expr: + - dialect: ANSI SQL + expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) + description: Average lifetime sales value per customer + ai_context: + synonyms: + - "CLV" + - "LTV" + - "customer value" + - "lifetime revenue" + - "customer worth" + - "average customer value" + + - name: sales_by_brand + expr: + - dialect: ANSI SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales grouped by brand (requires grouping by item.i_brand) + ai_context: + synonyms: + - "brand sales" + - "brand performance" + - "brand revenue" + - "sales per brand" + + - name: year_over_year_growth + expr: + - dialect: ANSI SQL + expression: | + (SUM(CASE WHEN date_dim.d_year = YEAR(CURRENT_DATE) THEN store_sales.ss_ext_sales_price ELSE 0 END) - + SUM(CASE WHEN date_dim.d_year = YEAR(CURRENT_DATE) - 1 THEN store_sales.ss_ext_sales_price ELSE 0 END)) / + NULLIF(SUM(CASE WHEN date_dim.d_year = YEAR(CURRENT_DATE) - 1 THEN store_sales.ss_ext_sales_price ELSE 0 END), 0) * 100 + description: Year-over-year sales growth percentage + ai_context: + synonyms: + - "YoY growth" + - "year over year" + - "annual growth" + - "sales growth" + - "yearly growth" + + - name: store_sales_per_employee + expr: + - dialect: ANSI SQL + expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) + description: Sales per employee across stores + ai_context: + synonyms: + - "employee productivity" + - "sales per staff" + - "efficiency metric" + - "revenue per employee" + + - name: top_selling_categories + expr: + - dialect: ANSI SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales by product category (requires grouping by item.i_category) + ai_context: + synonyms: + - "category sales" + - "category performance" + - "department sales" + - "sales by category" + + custom_extensions: + - vendor_name: snowflake + data: '{"warehouse": "TPCDS_WH", "database": "TPCDS", "schema": "PUBLIC"}' + + - vendor_name: dbt + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' + From 54efd81e432a25120291245c1b5dd4d835cf8dab Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Wed, 10 Dec 2025 22:19:08 -0800 Subject: [PATCH 2/4] spec changes --- core-spec/spec.yaml | 49 +++++++++++++++--------------- examples/tpcds_semantic_model.yaml | 18 +++++------ 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index ce4ef97e..2833af6f 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -12,27 +12,24 @@ # Supported SQL and expression language dialects dialects: - - "ANSI SQL" # Standard SQL dialect - - "Snowflake SQL" # Snowflake + - "ANSI_SQL" # Standard SQL dialect + - "SNOWFLAKE_SQL" # Snowflake - "MDX" # Multi-Dimensional Expressions - - "Tableau" # Tableau - - "BigQuery SQL" # Google BigQuery - - "Redshift SQL" # Amazon Redshift - - "Databricks SQL" # Databricks - - "Azure Synapse SQL" # Azure Synapse - - "PostgreSQL" # PostgreSQL - - "MySQL" # MySQL - - "Oracle SQL" # Oracle - - "SQL Server" # Microsoft SQL Server + - "TABLEAU" # Tableau + - "BIGQUERY_SQL" # Google BigQuery + - "REDSHIFT_SQL" # Amazon Redshift + - "DATABRICKS_SQL" # Databricks + - "POSTGRESQL" # PostgreSQL + - "MYSQL" # MySQL # Supported vendors for custom extensions vendors: - - "common" - - "snowflake" - - "tableau" - - "dbt" + - "COMMON" + - "SNOWFLAKE" + - "SALESFORCE" + - "DBT" # Standard aggregation types for structured metrics @@ -69,7 +66,9 @@ semantic_model: ai_context: { string: string } - custom_extensions: {} + custom_extensions: + - vendor_name: string + data: string # Required: Defines how logical datasets or semantic models are connected # Represents foreign key relationships between datasets @@ -79,14 +78,18 @@ semantic_model: to: string from_column: string to_column: string - custom_extensions: {} + custom_extensions: + - vendor_name: string + data: string # Optional: metrics # These metrics can span one or more logical datasets and use relationships # See Metrics section below for detailed structure metrics: - name: string - expr: [] + expression: + - dialect: string + expression: string description: string ai_context: string custom_extensions: @@ -203,15 +206,13 @@ metrics: # Required: Expression definition with dialect support # Supports multiple SQL dialects for cross-platform compatibility expr: - # Option 1: Full expression with dialect specification - - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI SQL" + - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" + # Option 1: Full expression with aggregate function expression: string # e.g., "SUM(sales)", "AVG(revenue)" - # Option 2: Structured aggregation (for simple metrics) - # This provides a more standardized way to represent common aggregations - - dialect: string # Must be one of the values from 'dialects' enum above + # Option 2: Structured aggregation aggregation_type: string # Must be one of the values from 'aggregation_types' enum above - input_expr: string # e.g., "sales", "revenue", "quantity" + input_expr: string # Must be a scalar expressions e.g., "sales", "revenue", "quantity" # Optional: Human-readable description of the metric # Should explain what the metric measures and how it's used diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index 0b5ebdd6..0e7e0721 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -434,7 +434,7 @@ semantic_model: metrics: - name: customer_lifetime_value expr: - - dialect: ANSI SQL + - dialect: ANSI_SQL expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) description: Average lifetime sales value per customer ai_context: @@ -448,7 +448,7 @@ semantic_model: - name: sales_by_brand expr: - - dialect: ANSI SQL + - dialect: ANSI_SQL expression: SUM(store_sales.ss_ext_sales_price) description: Total sales grouped by brand (requires grouping by item.i_brand) ai_context: @@ -460,7 +460,7 @@ semantic_model: - name: year_over_year_growth expr: - - dialect: ANSI SQL + - dialect: ANSI_SQL expression: | (SUM(CASE WHEN date_dim.d_year = YEAR(CURRENT_DATE) THEN store_sales.ss_ext_sales_price ELSE 0 END) - SUM(CASE WHEN date_dim.d_year = YEAR(CURRENT_DATE) - 1 THEN store_sales.ss_ext_sales_price ELSE 0 END)) / @@ -476,7 +476,7 @@ semantic_model: - name: store_sales_per_employee expr: - - dialect: ANSI SQL + - dialect: ANSI_SQL expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) description: Sales per employee across stores ai_context: @@ -488,7 +488,7 @@ semantic_model: - name: top_selling_categories expr: - - dialect: ANSI SQL + - dialect: ANSI_SQL expression: SUM(store_sales.ss_ext_sales_price) description: Total sales by product category (requires grouping by item.i_category) ai_context: @@ -499,9 +499,9 @@ semantic_model: - "sales by category" custom_extensions: - - vendor_name: snowflake - data: '{"warehouse": "TPCDS_WH", "database": "TPCDS", "schema": "PUBLIC"}' + - vendor_name: SNOWFLAKE + data: '{"custom_instructions": "TPCDS model", "question_categorization": "TPCDS"}' - - vendor_name: dbt + - vendor_name: DBT data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' - + From 7ac668298bdbe2b19797bd82faa15e376524c074 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Wed, 10 Dec 2025 23:21:17 -0800 Subject: [PATCH 3/4] spec --- core-spec/spec.yaml | 102 +++++----- examples/tpcds_semantic_model.yaml | 289 ++++++++++------------------- 2 files changed, 150 insertions(+), 241 deletions(-) diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 2833af6f..772f80be 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -56,45 +56,16 @@ semantic_model: # Required: Collection of logical datasets(fact and dimension tables) # See Logical Dataset section below for detailed structure - logical_datasets: - - name: string - base_table: string - primary_key: string - unique_keys: [] - description: string - fields: [] - ai_context: { - string: string - } - custom_extensions: - - vendor_name: string - data: string - - # Required: Defines how logical datasets or semantic models are connected - # Represents foreign key relationships between datasets - relationships: - - name: string - from: string - to: string - from_column: string - to_column: string - custom_extensions: - - vendor_name: string - data: string - - # Optional: metrics + datasets: [] + + # Optional: Defines how logical datasets are connected + # See Relationships section below for detailed structure + relationships:[] + + # Optional: # These metrics can span one or more logical datasets and use relationships # See Metrics section below for detailed structure - metrics: - - name: string - expression: - - dialect: string - expression: string - description: string - ai_context: string - custom_extensions: - - vendor_name: string - data: string + metrics: [] # Optional: Vendor-specific attributes for extensibility # Allows vendors to add custom metadata without breaking core compatibility @@ -110,14 +81,26 @@ datasets: # Required: Unique identifier for the logical dataset - name: string - # Required: Reference to the underlying physical table or view - # Format should include database.schema.table or similar qualified name - base_table: string - - # Optional: Array of column names that uniquely identify rows in this dataset - # Used for join optimization and data integrity validation + # Required: Reference to the underlying physical table/view or query + # Format should be either database_name.schema_name.table_name or query + source: string + + # Optional: Primary key definition that uniquely identifies rows in this dataset + # Can be a single column or a composite of multiple columns + # This is the preferred unique identifier for this dataset and is used in relationships to determine many-to-one or one-to-one. + # Examples: + # - [customer_id] # Simple primary key + # - [order_id, line_number] # Composite primary key + primary_key: [] # Array of column names (single or composite) + + # Optional: Array of unique key definitions that uniquely identify rows in this dataset + # Each unique key can be a single column or a composite of multiple columns + # Used for determining relationship type of either many-to-one or one-to-one + # Examples: + # - [column1] # Simple key on one column + # - [column2, column3] # Composite key on two columns unique_keys: - - string + - [] # Array of column names (single or composite) # Optional: Human-readable description of the logical dataset description: string @@ -126,7 +109,7 @@ datasets: # Helps LLMs understand the business meaning and generate better queries ai_context: string - # Optional: Row-level calculations for grouping, filtering, and metric expressions + # Optional: Row-level calculations for grouping, filtering, and in metric expressions # See Fields section below for detailed structure fields: [] @@ -151,11 +134,21 @@ relationships: # References a logical dataset name to: string - # Required: Column name in the "from" dataset (foreign key) - from_column: string - - # Required: Column name in the "to" dataset (primary key) - to_column: string + # Required: Array of column names in the "from" dataset (foreign key columns) + # For simple relationships, use a single column: [column1] + # For composite relationships, use multiple columns: [column1, column2] + # The order of columns must correspond to the order in to_columns + # Examples: + # - [customer_id] # Simple foreign key + # - [order_id, line_number] # Composite foreign key + from_columns: [] # Array of column names + + # Required: Array of column names in the "to" dataset (primary or unique key columns) + # Must have the same number of columns as from_columns in corresponding order + # Examples: + # - [id] # Simple key + # - [order_id, line_number] # Composite key + to_columns: [] # Array of column names # Optional: Vendor-specific attributes for extensibility custom_extensions: @@ -170,8 +163,8 @@ fields: - name: string # Required: SQL expression that defines how to compute this field - # Can be a simple column reference or a complex expression - expr: string + # Can be a simple column reference or a complex scalar expression + expression: string # Optional: Dimension metadata # Indicates this field can be used as a dimension for grouping/filtering @@ -205,12 +198,13 @@ metrics: # Required: Expression definition with dialect support # Supports multiple SQL dialects for cross-platform compatibility - expr: + expression: - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" + # Must be one of the following two options # Option 1: Full expression with aggregate function expression: string # e.g., "SUM(sales)", "AVG(revenue)" - # Option 2: Structured aggregation + # Option 2: Structured aggregation_type: string # Must be one of the values from 'aggregation_types' enum above input_expr: string # Must be a scalar expressions e.g., "sales", "revenue", "quantity" diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index 0e7e0721..a0262476 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -8,13 +8,13 @@ semantic_model: ai_context: prompt: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." - logical_datasets: + datasets: # Fact table: Store sales transactions - name: store_sales - base_table: tpcds.public.store_sales + source: tpcds.public.store_sales + primary_key: [ss_item_sk, ss_ticket_number] # Composite primary key unique_keys: - - ss_item_sk - - ss_ticket_number + - [ss_item_sk, ss_ticket_number] # Composite key: item + ticket number uniquely identifies a line item description: Fact table containing all store sales transactions ai_context: synonyms: @@ -22,127 +22,111 @@ semantic_model: - "store purchases" - "retail sales" - "POS data" - - "point of sale" fields: - name: ss_sold_date_sk - expr: ss_sold_date_sk + expression: ss_sold_date_sk description: Foreign key to date dimension ai_context: synonyms: - "sale date" - "transaction date" - - "purchase date" - name: ss_item_sk - expr: ss_item_sk + expression: ss_item_sk description: Foreign key to item dimension ai_context: synonyms: - "product" - "item" - - "SKU" - - "product key" - name: ss_customer_sk - expr: ss_customer_sk + expression: ss_customer_sk description: Foreign key to customer dimension ai_context: synonyms: - "customer" - "buyer" - - "purchaser" - name: ss_store_sk - expr: ss_store_sk + expression: ss_store_sk description: Foreign key to store dimension ai_context: synonyms: - - "store location" - - "retail location" - "store" + - "location" - name: ss_quantity - expr: ss_quantity + expression: ss_quantity description: Quantity of items sold ai_context: synonyms: - "units sold" - - "quantity purchased" - - "item count" - - "units" + - "quantity" - name: ss_sales_price - expr: ss_sales_price + expression: ss_sales_price description: Sales price per unit ai_context: synonyms: - "unit price" - - "price per item" - - "item price" + - "price" - name: ss_ext_sales_price - expr: ss_ext_sales_price + expression: ss_ext_sales_price description: Extended sales price (quantity * price) ai_context: synonyms: - "total price" - - "line item total" - - "extended price" - - "total amount" + - "line total" - name: ss_net_profit - expr: ss_net_profit + expression: ss_net_profit description: Net profit from the sale ai_context: synonyms: - "profit" - "margin" - - "earnings" - - "net earnings" # Dimension table: Date - name: date_dim - base_table: tpcds.public.date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] # Simple primary key unique_keys: - - d_date_sk + - [d_date_sk] # Simple key: single column description: Date dimension with calendar attributes ai_context: synonyms: - "calendar" - "dates" - "time periods" - - "date table" fields: - name: d_date_sk - expr: d_date_sk + expression: d_date_sk description: Surrogate key for date - dimension: - is_time: true - name: d_date - expr: d_date + expression: d_date description: Actual date value dimension: is_time: true ai_context: synonyms: - - "calendar date" - "date" + - "calendar date" - name: d_year - expr: d_year + expression: d_year description: Year dimension: is_time: true ai_context: synonyms: - "year" - - "calendar year" - name: d_quarter_name - expr: d_quarter_name + expression: d_quarter_name description: Quarter name (e.g., 2024Q1) dimension: is_time: true @@ -150,248 +134,190 @@ semantic_model: synonyms: - "quarter" - "fiscal quarter" - - "Q1, Q2, Q3, Q4" - name: d_month_name - expr: d_month_name + expression: d_month_name description: Month name dimension: is_time: true ai_context: synonyms: - "month" - - "calendar month" - - - name: d_day_name - expr: d_day_name - description: Day of week name - dimension: - is_time: true - ai_context: - synonyms: - - "weekday" - - "day of week" - - "day name" # Dimension table: Customer - name: customer - base_table: tpcds.public.customer + source: tpcds.public.customer + primary_key: [c_customer_sk] # Simple primary key unique_keys: - - c_customer_sk + - [c_customer_sk] # Simple key: single column description: Customer dimension with demographic information ai_context: synonyms: - "customers" - "shoppers" - "buyers" - - "consumer demographics" - - "purchasers" fields: - name: c_customer_sk - expr: c_customer_sk + expression: c_customer_sk description: Surrogate key for customer - name: c_customer_id - expr: c_customer_id + expression: c_customer_id description: Business key for customer ai_context: synonyms: - "customer ID" - "customer number" - - "customer identifier" - name: c_first_name - expr: c_first_name + expression: c_first_name description: Customer first name - name: c_last_name - expr: c_last_name + expression: c_last_name description: Customer last name - - name: c_birth_year - expr: c_birth_year - description: Customer birth year + - name: customer_full_name + expression: c_first_name || ' ' || c_last_name + description: Customer full name (computed field) ai_context: synonyms: - - "age" - - "birth year" - - "year of birth" + - "full name" + - "customer name" - name: c_email_address - expr: c_email_address + expression: c_email_address description: Customer email address ai_context: synonyms: - "email" - "contact" - - "email address" - - - name: customer_full_name - expr: c_first_name || ' ' || c_last_name - description: Customer full name (computed field) - ai_context: - synonyms: - - "full name" - - "customer name" - - "name" # Dimension table: Item (Product) - name: item - base_table: tpcds.public.item + source: tpcds.public.item + primary_key: [i_item_sk] # Simple primary key unique_keys: - - i_item_sk + - [i_item_sk] # Simple key: single column description: Item/Product dimension with product attributes ai_context: synonyms: - "products" - "items" - "merchandise" - - "SKUs" - - "inventory" - - "catalog" fields: - name: i_item_sk - expr: i_item_sk + expression: i_item_sk description: Surrogate key for item - name: i_item_id - expr: i_item_id + expression: i_item_id description: Business key for item ai_context: synonyms: - "item ID" - "product ID" - "SKU" - - "product code" - name: i_item_desc - expr: i_item_desc + expression: i_item_desc description: Item description ai_context: synonyms: - "product description" - "item name" - - "product name" - name: i_brand - expr: i_brand + expression: i_brand description: Brand name ai_context: synonyms: - "brand" - "manufacturer" - - "brand name" - name: i_category - expr: i_category + expression: i_category description: Item category ai_context: synonyms: - "product category" - "department" - - "category" - - - name: i_class - expr: i_class - description: Item class - ai_context: - synonyms: - - "product class" - - "product type" - - "class" - name: i_current_price - expr: i_current_price + expression: i_current_price description: Current price of the item ai_context: synonyms: - "price" - - "MSRP" - "list price" - - "current price" # Dimension table: Store - name: store - base_table: tpcds.public.store + source: tpcds.public.store + primary_key: [s_store_sk] # Simple primary key unique_keys: - - s_store_sk + - [s_store_sk] # Simple key: single column description: Store dimension with location and store attributes ai_context: synonyms: - "stores" - "retail locations" - - "outlets" - "branches" - - "locations" fields: - name: s_store_sk - expr: s_store_sk + expression: s_store_sk description: Surrogate key for store - name: s_store_id - expr: s_store_id + expression: s_store_id description: Business key for store ai_context: synonyms: - "store ID" - "store number" - - "location ID" - name: s_store_name - expr: s_store_name + expression: s_store_name description: Store name ai_context: synonyms: - "store name" - "location name" - - "branch name" - - - name: s_number_employees - expr: s_number_employees - description: Number of employees at the store - ai_context: - synonyms: - - "employee count" - - "staff size" - - "number of employees" - name: s_city - expr: s_city + expression: s_city description: City where store is located ai_context: synonyms: - "city" - "location" - - "store city" - name: s_state - expr: s_state + expression: s_state description: State where store is located ai_context: synonyms: - "state" - "region" - - "province" - - name: s_zip - expr: s_zip - description: ZIP code of store location + - name: s_number_employees + expression: s_number_employees + description: Number of employees at the store ai_context: synonyms: - - "zip code" - - "postal code" - - "ZIP" + - "employee count" + - "staff size" - # Relationships between logical datasets + # Relationships between datasets relationships: - name: store_sales_to_date from: store_sales to: date_dim - from_column: ss_sold_date_sk - to_column: d_date_sk + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] ai_context: synonyms: - "sales date relationship" @@ -400,40 +326,61 @@ semantic_model: - name: store_sales_to_customer from: store_sales to: customer - from_column: ss_customer_sk - to_column: c_customer_sk + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] ai_context: synonyms: - "customer purchase relationship" - "who bought" - - "purchaser relationship" - name: store_sales_to_item from: store_sales to: item - from_column: ss_item_sk - to_column: i_item_sk + from_columns: [ss_item_sk] + to_columns: [i_item_sk] ai_context: synonyms: - "product sold relationship" - "what was sold" - - "item purchase relationship" - name: store_sales_to_store from: store_sales to: store - from_column: ss_store_sk - to_column: s_store_sk + from_columns: [ss_store_sk] + to_columns: [s_store_sk] ai_context: synonyms: - "store location relationship" - "where sale occurred" - - "sales location relationship" # Semantic model-level metrics spanning multiple datasets metrics: + - name: total_sales + expression: + - dialect: ANSI_SQL + aggregation_type: SUM + input_expr: store_sales.ss_ext_sales_price + description: Total sales revenue across all transactions + ai_context: + synonyms: + - "total revenue" + - "gross sales" + - "sales amount" + + - name: total_profit + expression: + - dialect: ANSI_SQL + aggregation_type: SUM + input_expr: store_sales.ss_net_profit + description: Total net profit from store sales + ai_context: + synonyms: + - "net profit" + - "total earnings" + - "profit" + - name: customer_lifetime_value - expr: + expression: - dialect: ANSI_SQL expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) description: Average lifetime sales value per customer @@ -443,65 +390,33 @@ semantic_model: - "LTV" - "customer value" - "lifetime revenue" - - "customer worth" - - "average customer value" - name: sales_by_brand - expr: + expression: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) - description: Total sales grouped by brand (requires grouping by item.i_brand) + aggregation_type: SUM + input_expr: store_sales.ss_ext_sales_price + description: Total sales by brand (requires grouping by item.i_brand) ai_context: synonyms: - "brand sales" - "brand performance" - "brand revenue" - - "sales per brand" - - name: year_over_year_growth - expr: - - dialect: ANSI_SQL - expression: | - (SUM(CASE WHEN date_dim.d_year = YEAR(CURRENT_DATE) THEN store_sales.ss_ext_sales_price ELSE 0 END) - - SUM(CASE WHEN date_dim.d_year = YEAR(CURRENT_DATE) - 1 THEN store_sales.ss_ext_sales_price ELSE 0 END)) / - NULLIF(SUM(CASE WHEN date_dim.d_year = YEAR(CURRENT_DATE) - 1 THEN store_sales.ss_ext_sales_price ELSE 0 END), 0) * 100 - description: Year-over-year sales growth percentage - ai_context: - synonyms: - - "YoY growth" - - "year over year" - - "annual growth" - - "sales growth" - - "yearly growth" - - - name: store_sales_per_employee - expr: + - name: store_productivity + expression: - dialect: ANSI_SQL expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) description: Sales per employee across stores ai_context: synonyms: + - "sales per employee" - "employee productivity" - - "sales per staff" - - "efficiency metric" - "revenue per employee" - - name: top_selling_categories - expr: - - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) - description: Total sales by product category (requires grouping by item.i_category) - ai_context: - synonyms: - - "category sales" - - "category performance" - - "department sales" - - "sales by category" - custom_extensions: - vendor_name: SNOWFLAKE - data: '{"custom_instructions": "TPCDS model", "question_categorization": "TPCDS"}' + data: '{"warehouse": "TPCDS_WH", "database": "TPCDS", "schema": "PUBLIC"}' - vendor_name: DBT data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' - From 25696230ad64011c128ff2fd5686007b9736be14 Mon Sep 17 00:00:00 2001 From: Khushboo Bhatia Date: Wed, 10 Dec 2025 23:32:04 -0800 Subject: [PATCH 4/4] exmaple --- examples/tpcds_semantic_model.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index a0262476..56bc5579 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -415,8 +415,20 @@ semantic_model: - "revenue per employee" custom_extensions: - - vendor_name: SNOWFLAKE - data: '{"warehouse": "TPCDS_WH", "database": "TPCDS", "schema": "PUBLIC"}' + - vendor_name: SALESFORCE + data: '{ + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "1.0" + } + }' - vendor_name: DBT data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}'