diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml new file mode 100644 index 00000000..772f80be --- /dev/null +++ b/core-spec/spec.yaml @@ -0,0 +1,222 @@ +# OSI - Core Metadata Spec (YAML Schema) +# Version: 1.0 +# +# Goals: +# - Standardization: Establish uniform language and structure for semantic model definitions +# - Extensibility: Support domain-specific extensions while maintaining core compatibility +# - Interoperability: Enable exchange and reuse across different AI and BI applications + +--- +# Enumerations +# Standard enums used throughout the specification + +# Supported SQL and expression language dialects +dialects: + - "ANSI_SQL" # Standard SQL dialect + - "SNOWFLAKE_SQL" # Snowflake + - "MDX" # Multi-Dimensional Expressions + - "TABLEAU" # Tableau + - "BIGQUERY_SQL" # Google BigQuery + - "REDSHIFT_SQL" # Amazon Redshift + - "DATABRICKS_SQL" # Databricks + - "POSTGRESQL" # PostgreSQL + - "MYSQL" # MySQL + + + +# Supported vendors for custom extensions +vendors: + - "COMMON" + - "SNOWFLAKE" + - "SALESFORCE" + - "DBT" + + +# Standard aggregation types for structured metrics +aggregation_types: + # Basic aggregations + - "SUM" + - "AVG" + - "COUNT" + - "COUNT_DISTINCT" + - "MIN" + - "MAX" + + +# Top-level semantic model definition +semantic_model: + # Required: Unique identifier for the semantic model + - name: string + + # Optional: Human-readable description of the semantic model + description: string + + # Optional: Additional context for AI tools (e.g., custom prompts, instructions) + ai_context: string + + # Required: Collection of logical datasets(fact and dimension tables) + # See Logical Dataset section below for detailed structure + datasets: [] + + # Optional: Defines how logical datasets are connected + # See Relationships section below for detailed structure + relationships:[] + + # Optional: + # These metrics can span one or more logical datasets and use relationships + # See Metrics section below for detailed structure + metrics: [] + + # Optional: Vendor-specific attributes for extensibility + # Allows vendors to add custom metadata without breaking core compatibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Logical Dataset Schema +# Represents business entities or concepts (fact and dimension tables) +# Fields are defined within the scope of a logical dataset +datasets: + # Required: Unique identifier for the logical dataset + - name: string + + # Required: Reference to the underlying physical table/view or query + # Format should be either database_name.schema_name.table_name or query + source: string + + # Optional: Primary key definition that uniquely identifies rows in this dataset + # Can be a single column or a composite of multiple columns + # This is the preferred unique identifier for this dataset and is used in relationships to determine many-to-one or one-to-one. + # Examples: + # - [customer_id] # Simple primary key + # - [order_id, line_number] # Composite primary key + primary_key: [] # Array of column names (single or composite) + + # Optional: Array of unique key definitions that uniquely identify rows in this dataset + # Each unique key can be a single column or a composite of multiple columns + # Used for determining relationship type of either many-to-one or one-to-one + # Examples: + # - [column1] # Simple key on one column + # - [column2, column3] # Composite key on two columns + unique_keys: + - [] # Array of column names (single or composite) + + # Optional: Human-readable description of the logical dataset + description: string + + # Optional: Additional context for AI tools (e.g., synonyms, common terms) + # Helps LLMs understand the business meaning and generate better queries + ai_context: string + + # Optional: Row-level calculations for grouping, filtering, and in metric expressions + # See Fields section below for detailed structure + fields: [] + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Relationship Schema +# Defines how logical datasets or semantic models are connected +# Represents foreign key relationships (many-to-one or one-to-one) +relationships: + # Required: Unique identifier for the relationship + - name: string + + # Required: The logical dataset on the many side of the relationship + # References a logical dataset name + from: string + + # Required: The logical dataset on the one side of the relationship + # References a logical dataset name + to: string + + # Required: Array of column names in the "from" dataset (foreign key columns) + # For simple relationships, use a single column: [column1] + # For composite relationships, use multiple columns: [column1, column2] + # The order of columns must correspond to the order in to_columns + # Examples: + # - [customer_id] # Simple foreign key + # - [order_id, line_number] # Composite foreign key + from_columns: [] # Array of column names + + # Required: Array of column names in the "to" dataset (primary or unique key columns) + # Must have the same number of columns as from_columns in corresponding order + # Examples: + # - [id] # Simple key + # - [order_id, line_number] # Composite key + to_columns: [] # Array of column names + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Fields Schema +# Represents row-level attributes that can be used for grouping, filtering, and metric expressions +fields: + # Required: Unique identifier for the field within the logical dataset + - name: string + + # Required: SQL expression that defines how to compute this field + # Can be a simple column reference or a complex scalar expression + expression: string + + # Optional: Dimension metadata + # Indicates this field can be used as a dimension for grouping/filtering + dimension: + # Optional: Indicates if this is a time-based dimension + # Used for time-series analysis and temporal filtering + is_time: boolean + + # Optional: Label for categorization (e.g., "filter") + label: string + + # Optional: Human-readable description of the field + description: string + + # Optional: Additional context for AI tools (e.g., synonyms, business terms) + # Helps LLMs understand the field meaning and generate better queries + ai_context: string + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string + +--- +# Metrics Schema +# Quantitative measures defined on business data +# Represents key calculations like sums, averages, ratios, etc. +metrics: + # Required: Unique identifier for the metric within the logical dataset + - name: string + + # Required: Expression definition with dialect support + # Supports multiple SQL dialects for cross-platform compatibility + expression: + - dialect: string # Must be one of the values from 'dialects' enum above, Default: "ANSI_SQL" + # Must be one of the following two options + # Option 1: Full expression with aggregate function + expression: string # e.g., "SUM(sales)", "AVG(revenue)" + + # Option 2: Structured + aggregation_type: string # Must be one of the values from 'aggregation_types' enum above + input_expr: string # Must be a scalar expressions e.g., "sales", "revenue", "quantity" + + # Optional: Human-readable description of the metric + # Should explain what the metric measures and how it's used + description: string + + # Optional: Additional context for AI tools (e.g., synonyms, business context) + # Helps LLMs understand the metric meaning and suggest it appropriately + ai_context: string + + # Optional: Vendor-specific attributes for extensibility + custom_extensions: + - vendor_name: string # Must be one of the values from 'vendors' enum above + data: string diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml new file mode 100644 index 00000000..56bc5579 --- /dev/null +++ b/examples/tpcds_semantic_model.yaml @@ -0,0 +1,434 @@ +# TPC-DS Semantic Model Example +# This example demonstrates the OSI Core Metadata Spec using the TPC-DS benchmark schema +# TPC-DS is a decision support benchmark with a realistic retail business model + +semantic_model: + - name: tpcds_retail_model + description: TPC-DS retail semantic model for sales and customer analytics + ai_context: + prompt: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." + + datasets: + # Fact table: Store sales transactions + - name: store_sales + source: tpcds.public.store_sales + primary_key: [ss_item_sk, ss_ticket_number] # Composite primary key + unique_keys: + - [ss_item_sk, ss_ticket_number] # Composite key: item + ticket number uniquely identifies a line item + description: Fact table containing all store sales transactions + ai_context: + synonyms: + - "sales transactions" + - "store purchases" + - "retail sales" + - "POS data" + + fields: + - name: ss_sold_date_sk + expression: ss_sold_date_sk + description: Foreign key to date dimension + ai_context: + synonyms: + - "sale date" + - "transaction date" + + - name: ss_item_sk + expression: ss_item_sk + description: Foreign key to item dimension + ai_context: + synonyms: + - "product" + - "item" + + - name: ss_customer_sk + expression: ss_customer_sk + description: Foreign key to customer dimension + ai_context: + synonyms: + - "customer" + - "buyer" + + - name: ss_store_sk + expression: ss_store_sk + description: Foreign key to store dimension + ai_context: + synonyms: + - "store" + - "location" + + - name: ss_quantity + expression: ss_quantity + description: Quantity of items sold + ai_context: + synonyms: + - "units sold" + - "quantity" + + - name: ss_sales_price + expression: ss_sales_price + description: Sales price per unit + ai_context: + synonyms: + - "unit price" + - "price" + + - name: ss_ext_sales_price + expression: ss_ext_sales_price + description: Extended sales price (quantity * price) + ai_context: + synonyms: + - "total price" + - "line total" + + - name: ss_net_profit + expression: ss_net_profit + description: Net profit from the sale + ai_context: + synonyms: + - "profit" + - "margin" + + # Dimension table: Date + - name: date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] # Simple primary key + unique_keys: + - [d_date_sk] # Simple key: single column + description: Date dimension with calendar attributes + ai_context: + synonyms: + - "calendar" + - "dates" + - "time periods" + + fields: + - name: d_date_sk + expression: d_date_sk + description: Surrogate key for date + + - name: d_date + expression: d_date + description: Actual date value + dimension: + is_time: true + ai_context: + synonyms: + - "date" + - "calendar date" + + - name: d_year + expression: d_year + description: Year + dimension: + is_time: true + ai_context: + synonyms: + - "year" + + - name: d_quarter_name + expression: d_quarter_name + description: Quarter name (e.g., 2024Q1) + dimension: + is_time: true + ai_context: + synonyms: + - "quarter" + - "fiscal quarter" + + - name: d_month_name + expression: d_month_name + description: Month name + dimension: + is_time: true + ai_context: + synonyms: + - "month" + + # Dimension table: Customer + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] # Simple primary key + unique_keys: + - [c_customer_sk] # Simple key: single column + description: Customer dimension with demographic information + ai_context: + synonyms: + - "customers" + - "shoppers" + - "buyers" + + fields: + - name: c_customer_sk + expression: c_customer_sk + description: Surrogate key for customer + + - name: c_customer_id + expression: c_customer_id + description: Business key for customer + ai_context: + synonyms: + - "customer ID" + - "customer number" + + - name: c_first_name + expression: c_first_name + description: Customer first name + + - name: c_last_name + expression: c_last_name + description: Customer last name + + - name: customer_full_name + expression: c_first_name || ' ' || c_last_name + description: Customer full name (computed field) + ai_context: + synonyms: + - "full name" + - "customer name" + + - name: c_email_address + expression: c_email_address + description: Customer email address + ai_context: + synonyms: + - "email" + - "contact" + + # Dimension table: Item (Product) + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] # Simple primary key + unique_keys: + - [i_item_sk] # Simple key: single column + description: Item/Product dimension with product attributes + ai_context: + synonyms: + - "products" + - "items" + - "merchandise" + + fields: + - name: i_item_sk + expression: i_item_sk + description: Surrogate key for item + + - name: i_item_id + expression: i_item_id + description: Business key for item + ai_context: + synonyms: + - "item ID" + - "product ID" + - "SKU" + + - name: i_item_desc + expression: i_item_desc + description: Item description + ai_context: + synonyms: + - "product description" + - "item name" + + - name: i_brand + expression: i_brand + description: Brand name + ai_context: + synonyms: + - "brand" + - "manufacturer" + + - name: i_category + expression: i_category + description: Item category + ai_context: + synonyms: + - "product category" + - "department" + + - name: i_current_price + expression: i_current_price + description: Current price of the item + ai_context: + synonyms: + - "price" + - "list price" + + # Dimension table: Store + - name: store + source: tpcds.public.store + primary_key: [s_store_sk] # Simple primary key + unique_keys: + - [s_store_sk] # Simple key: single column + description: Store dimension with location and store attributes + ai_context: + synonyms: + - "stores" + - "retail locations" + - "branches" + + fields: + - name: s_store_sk + expression: s_store_sk + description: Surrogate key for store + + - name: s_store_id + expression: s_store_id + description: Business key for store + ai_context: + synonyms: + - "store ID" + - "store number" + + - name: s_store_name + expression: s_store_name + description: Store name + ai_context: + synonyms: + - "store name" + - "location name" + + - name: s_city + expression: s_city + description: City where store is located + ai_context: + synonyms: + - "city" + - "location" + + - name: s_state + expression: s_state + description: State where store is located + ai_context: + synonyms: + - "state" + - "region" + + - name: s_number_employees + expression: s_number_employees + description: Number of employees at the store + ai_context: + synonyms: + - "employee count" + - "staff size" + + # Relationships between datasets + relationships: + - name: store_sales_to_date + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] + ai_context: + synonyms: + - "sales date relationship" + - "when sale occurred" + + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] + ai_context: + synonyms: + - "customer purchase relationship" + - "who bought" + + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] + ai_context: + synonyms: + - "product sold relationship" + - "what was sold" + + - name: store_sales_to_store + from: store_sales + to: store + from_columns: [ss_store_sk] + to_columns: [s_store_sk] + ai_context: + synonyms: + - "store location relationship" + - "where sale occurred" + + # Semantic model-level metrics spanning multiple datasets + metrics: + - name: total_sales + expression: + - dialect: ANSI_SQL + aggregation_type: SUM + input_expr: store_sales.ss_ext_sales_price + description: Total sales revenue across all transactions + ai_context: + synonyms: + - "total revenue" + - "gross sales" + - "sales amount" + + - name: total_profit + expression: + - dialect: ANSI_SQL + aggregation_type: SUM + input_expr: store_sales.ss_net_profit + description: Total net profit from store sales + ai_context: + synonyms: + - "net profit" + - "total earnings" + - "profit" + + - name: customer_lifetime_value + expression: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) + description: Average lifetime sales value per customer + ai_context: + synonyms: + - "CLV" + - "LTV" + - "customer value" + - "lifetime revenue" + + - name: sales_by_brand + expression: + - dialect: ANSI_SQL + aggregation_type: SUM + input_expr: store_sales.ss_ext_sales_price + description: Total sales by brand (requires grouping by item.i_brand) + ai_context: + synonyms: + - "brand sales" + - "brand performance" + - "brand revenue" + + - name: store_productivity + expression: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) + description: Sales per employee across stores + ai_context: + synonyms: + - "sales per employee" + - "employee productivity" + - "revenue per employee" + + custom_extensions: + - vendor_name: SALESFORCE + data: '{ + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "1.0" + } + }' + + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}'